From 793e067fe9d29278603fe64c126d3834eb88866f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 13:55:02 -0700 Subject: [PATCH 01/31] feat(storage)!: replace DuckLake telemetry Move telemetry ingestion to a durable WAL-backed repository that commits indexed hot segments and open Parquet files. Keep DuckDB as the analytical query and rollup engine, and SQLite as control-plane storage. BREAKING CHANGE: existing DuckLake telemetry data is not migrated. Deployments must start with a clean storage.data_dir. --- README.md | 5 +- THIRD_PARTY_NOTICES | 2468 ++++++++++++----- cmd/bench/main.go | 82 +- cmd/bench/metrics_report.go | 90 +- cmd/bench/metrics_report_test.go | 62 +- cmd/bench/verdict.go | 6 +- cmd/bench/verdict_test.go | 8 +- cmd/fanout/main.go | 36 +- cmd/storage-poc/main.go | 525 ++++ docs/storage-architecture-options.md | 412 +++ docs/storage-poc.md | 99 + experiments/storage-poc-chdb/go.mod | 23 + experiments/storage-poc-chdb/go.sum | 24 + experiments/storage-poc-chdb/main.go | 250 ++ fanout.example.yaml | 2 +- go.mod | 8 +- go.sum | 4 + internal/api/health.go | 11 +- internal/api/health_test.go | 8 +- internal/config/config.go | 37 +- internal/config/config_test.go | 30 +- internal/config/sizing.go | 4 +- internal/config/sizing_test.go | 6 +- internal/ingest/attrs_test.go | 4 +- internal/ingest/http_test.go | 26 +- internal/ingest/server.go | 40 +- internal/ingest/server_test.go | 8 +- internal/lake/writer.go | 615 ---- internal/lake/writer_test.go | 171 -- internal/metrics/metrics.go | 81 +- internal/metrics/metrics_test.go | 54 +- internal/observability/logs.go | 125 +- internal/observability/namespace_test.go | 2 +- .../performance_benchmark_test.go | 3 - .../observability/performance_rollup_test.go | 5 +- internal/observability/service.go | 9 +- internal/observability/service_test.go | 54 +- internal/observability/trace.go | 102 +- internal/query/duck.go | 453 +-- internal/query/duck_test.go | 233 +- internal/query/duck_wal_test.go | 232 -- internal/query/edge_backlog_test.go | 39 +- internal/query/hourprune_experiment_test.go | 114 - internal/query/retry_test.go | 228 -- internal/query/schema.go | 7 +- internal/query/sql.go | 17 +- internal/query/views.go | 62 +- internal/query/views_test.go | 3 - .../{lake => query}/writegate/write_gate.go | 12 +- .../writegate/write_gate_test.go | 21 +- internal/storagebench/data.go | 45 + internal/telemetry/parquet.go | 330 +++ internal/telemetry/rows.go | 87 + internal/telemetry/segment/signal_store.go | 582 ++++ internal/telemetry/segment/span_columnar.go | 266 ++ internal/telemetry/segment/span_store.go | 1102 ++++++++ internal/telemetry/segment/span_store_test.go | 168 ++ internal/telemetry/store/compaction.go | 177 ++ internal/telemetry/store/repository.go | 401 +++ internal/telemetry/store/repository_test.go | 176 ++ internal/telemetry/store/writer.go | 152 + .../docs/reference/settings/storage.mdx | 12 +- 62 files changed, 7207 insertions(+), 3211 deletions(-) create mode 100644 cmd/storage-poc/main.go create mode 100644 docs/storage-architecture-options.md create mode 100644 docs/storage-poc.md create mode 100644 experiments/storage-poc-chdb/go.mod create mode 100644 experiments/storage-poc-chdb/go.sum create mode 100644 experiments/storage-poc-chdb/main.go delete mode 100644 internal/lake/writer.go delete mode 100644 internal/lake/writer_test.go delete mode 100644 internal/query/duck_wal_test.go delete mode 100644 internal/query/hourprune_experiment_test.go delete mode 100644 internal/query/retry_test.go rename internal/{lake => query}/writegate/write_gate.go (78%) rename internal/{lake => query}/writegate/write_gate_test.go (90%) create mode 100644 internal/storagebench/data.go create mode 100644 internal/telemetry/parquet.go create mode 100644 internal/telemetry/rows.go create mode 100644 internal/telemetry/segment/signal_store.go create mode 100644 internal/telemetry/segment/span_columnar.go create mode 100644 internal/telemetry/segment/span_store.go create mode 100644 internal/telemetry/segment/span_store_test.go create mode 100644 internal/telemetry/store/compaction.go create mode 100644 internal/telemetry/store/repository.go create mode 100644 internal/telemetry/store/repository_test.go create mode 100644 internal/telemetry/store/writer.go diff --git a/README.md b/README.md index 4b150020..f957d506 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ executable, including the React client. ![Fanout architecture](docs/diagrams/architecture.svg) -Telemetry lands over OTLP/gRPC or OTLP/HTTP, is batched into DuckLake/Parquet, +Telemetry lands over OTLP/gRPC or OTLP/HTTP, is durably committed to indexed +hot segments and open Parquet files, and is read back through a DuckDB query kernel that also maintains service, endpoint, and edge rollups. The browser client, an in-process agent, and any external MCP host all reach the same typed observability contract rather than @@ -53,7 +54,7 @@ what separates it from its neighbours. | If you use | Where Fanout differs | | --- | --- | | **Grafana with Loki, Tempo, and Mimir** | That stack keeps a service and a query language per signal, plus object storage underneath. Fanout keeps one process, one data directory, and one typed contract across all three signals, at the cost of the horizontal scale those components are built for. | -| **SigNoz** | Both are OTLP-native and self-hosted. SigNoz composes a collector, ClickHouse, and query services; Fanout compiles ingest, storage, query, alerting, and the browser client into one binary, with DuckLake/Parquet on local disk instead of a database cluster. | +| **SigNoz** | Both are OTLP-native and self-hosted. SigNoz composes a collector, ClickHouse, and query services; Fanout compiles ingest, indexed storage, DuckDB queries, alerting, and the browser client into one binary, with open Parquet on local disk instead of a database cluster. | | **Jaeger** | Jaeger covers traces and expects a storage backend you run separately. Fanout ingests traces, logs, and metrics into the same store, with nothing else to deploy. | | **Prometheus with Grafana** | Prometheus pulls metrics and is excellent at them. Fanout accepts pushed OTLP for all three signals and is built around investigating a specific incident rather than maintaining long-range metric series. | | **Datadog**, **Honeycomb**, **Grafana Cloud** | Those are managed services: someone else runs the storage, the scaling, and the upgrades, and your telemetry leaves your network to get there. Fanout is a binary you run, on data that stays on your disk. | diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index e318bb06..9a0532b8 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -17,7 +17,10 @@ COMPONENT INVENTORY - Go: github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260826145851-49e71f2b2d21 - Go: github.com/agext/levenshtein v1.2.3 - Go: github.com/alexedwards/scs/v2 v2.9.0 +- Go: github.com/andybalholm/brotli v1.2.2 - Go: github.com/antlr4-go/antlr/v4 v4.13.1 +- Go: github.com/apache/arrow-go/v18 v18.7.0 +- Go: github.com/apache/thrift v0.24.0 - Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 - Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 - Go: github.com/beorn7/perks v1.0.1 @@ -35,11 +38,15 @@ COMPONENT INVENTORY - Go: github.com/go-jose/go-jose/v4 v4.1.4 - Go: github.com/go-openapi/inflect v1.0.0 - Go: github.com/go-viper/mapstructure/v2 v2.5.0 +- Go: github.com/goccy/go-json v0.10.6 +- Go: github.com/google/flatbuffers v25.12.19+incompatible - Go: github.com/google/go-cmp v0.7.0 - Go: github.com/google/jsonschema-go v0.4.3 - Go: github.com/google/uuid v1.6.0 - Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 - Go: github.com/hashicorp/hcl/v2 v2.24.0 +- Go: github.com/klauspost/compress v1.19.2 +- Go: github.com/klauspost/cpuid/v2 v2.4.0 - Go: github.com/knadh/koanf/maps v0.1.3 - Go: github.com/knadh/koanf/parsers/yaml v1.1.1 - Go: github.com/knadh/koanf/providers/confmap v1.0.1 @@ -53,6 +60,7 @@ COMPONENT INVENTORY - Go: github.com/modelcontextprotocol/go-sdk v1.7.0 - Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - Go: github.com/ncruces/go-strftime v1.0.0 +- Go: github.com/pierrec/lz4/v4 v4.1.29 - Go: github.com/prometheus/client_golang v1.24.1 - Go: github.com/prometheus/client_model v0.6.2 - Go: github.com/prometheus/common v0.70.1 @@ -65,6 +73,7 @@ COMPONENT INVENTORY - Go: github.com/yosida95/uritemplate/v3 v3.0.2 - Go: github.com/zclconf/go-cty v1.19.0 - Go: github.com/zclconf/go-cty-yaml v1.2.0 +- Go: github.com/zeebo/xxh3 v1.1.0 - Go: go.opentelemetry.io/proto/otlp v1.11.0 - Go: go.yaml.in/yaml/v3 v3.0.5 - Go: golang.org/x/crypto v0.55.0 @@ -352,6 +361,7 @@ LICENSE AND NOTICE TEXTS - Go: github.com/agext/levenshtein v1.2.3 / LICENSE - Go: github.com/go-jose/go-jose/v4 v4.1.4 / LICENSE - Go: github.com/go-openapi/inflect v1.0.0 / LICENSE +- Go: github.com/google/flatbuffers v25.12.19+incompatible / LICENSE - Go: github.com/prometheus/client_golang v1.24.1 / LICENSE - Go: github.com/prometheus/client_model v0.6.2 / LICENSE - Go: github.com/prometheus/common v0.70.1 / LICENSE @@ -869,6 +879,30 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- Applies to ------------------------------------------------------------- +- Go: github.com/andybalholm/brotli v1.2.2 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + --- Applies to ------------------------------------------------------------- - Go: github.com/antlr4-go/antlr/v4 v4.13.1 / LICENSE ---------------------------------------------------------------------------- @@ -903,210 +937,7 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- Applies to ------------------------------------------------------------- -- Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2017 Martin Atkins - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ---------- - -Unicode table generation programs are under a separate copyright and license: - -Copyright (c) 2014 Couchbase, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the -License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -either express or implied. See the License for the specific language governing permissions -and limitations under the License. - ---------- - -Grapheme break data is provided as part of the Unicode character database, -copright 2016 Unicode, Inc, which is provided with the following license: - -Unicode Data Files include all data files under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, -http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and -http://www.unicode.org/utility/trac/browser/. - -Unicode Data Files do not include PDF online code charts under the -directory http://www.unicode.org/Public/. - -Software includes any source code published in the Unicode Standard -or under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, -http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and -http://www.unicode.org/utility/trac/browser/. - -NOTICE TO USER: Carefully read the following legal agreement. -BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S -DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), -YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE -TERMS AND CONDITIONS OF THIS AGREEMENT. -IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE -THE DATA FILES OR SOFTWARE. - -COPYRIGHT AND PERMISSION NOTICE - -Copyright © 1991-2017 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in http://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2017 Martin Atkins - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/beorn7/perks v1.0.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright (C) 2013 Blake Mizerany - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/bmatcuk/doublestar v1.3.4 / LICENSE ----------------------------------------------------------------------------- - -The MIT License (MIT) - -Copyright (c) 2014 Bob Matcuk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/cespare/xxhash/v2 v2.3.0 / LICENSE.txt ----------------------------------------------------------------------------- - -Copyright (c) 2016 Caleb Spare - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/coreos/go-oidc/v3 v3.20.0 / LICENSE -- Go: github.com/zclconf/go-cty-yaml v1.2.0 / LICENSE +- Go: github.com/apache/arrow-go/v18 v18.7.0 / LICENSE.txt ---------------------------------------------------------------------------- Apache License @@ -1289,7 +1120,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -1297,7 +1128,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1311,172 +1142,19 @@ Apache License See the License for the specific language governing permissions and limitations under the License. ---- Applies to ------------------------------------------------------------- -- Go: github.com/coreos/go-oidc/v3 v3.20.0 / NOTICE ----------------------------------------------------------------------------- - -CoreOS Project -Copyright 2014 CoreOS, Inc - -This product includes software developed at CoreOS, Inc. -(http://www.coreos.com/). - ---- Applies to ------------------------------------------------------------- -- Go: github.com/duckdb/duckdb-go-bindings v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 / LICENSE ----------------------------------------------------------------------------- +-------------------------------------------------------------------------------- -Copyright 2018-2026 Stichting DuckDB Foundation +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS): -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * arrow/flight/cookie_middleware.go -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +Copyright (c) 2009 The Go Authors. All rights reserved. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/duckdb/duckdb-go/v2 v2.10505.0 / LICENSE ----------------------------------------------------------------------------- - -Copyright 2019-2024 Marc Boeker -Copyright 2025-2026 Stichting DuckDB Foundation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/dustin/go-humanize v1.0.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2005-2008 Dustin Sallings - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - - ---- Applies to ------------------------------------------------------------- -- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright © 2012 The Go Authors. All rights reserved. -Copyright © fsnotify Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Google Inc. nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/go-openapi/inflect v1.0.0 / NOTICE ----------------------------------------------------------------------------- - -Copyright 2015-2025 go-swagger maintainers - -// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers -// SPDX-License-Identifier: Apache-2.0 - -This software library, github.com/go-openapi/jsonpointer, includes software developed -by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this software except in compliance with the License. - -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0. - -This software is copied from, derived from, and inspired by other original software products. -It ships with copies of other software which license terms are recalled below. - -The original software was authored by Chris Farmiloe at https://bitbucket.org/pkg/inflect under a MIT License. - -ghttps://bitbucket.org/pkg/inflect -=========================== - -Copyright (c) 2011 Chris Farmiloe - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE -- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE ----------------------------------------------------------------------------- - -The MIT License (MIT) - -Copyright (c) 2013 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/google/go-cmp v0.7.0 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2017 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. @@ -1500,455 +1178,1772 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/google/jsonschema-go v0.4.3 / LICENSE ----------------------------------------------------------------------------- - -MIT License - -Copyright (c) 2025 JSON Schema Go Project Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +-------------------------------------------------------------------------------- -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +This project includes code from the LLVM project: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +* arrow/compute/internal/kernels/_lib/types.h ---- Applies to ------------------------------------------------------------- -- Go: github.com/google/uuid v1.6.0 / LICENSE ----------------------------------------------------------------------------- +Apache License v2.0 with LLVM Exceptions. +See https://llvm.org/LICENSE.txt for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -Copyright (c) 2009,2014 Google Inc. All rights reserved. +-------------------------------------------------------------------------------- -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +This project includes code from the brotli project (https://github.com/google/brotli): - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +* parquet/compress/brotli.go -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Copyright: 2013 Google Inc. All Rights Reserved +Distributed under MIT License. --- Applies to ------------------------------------------------------------- -- Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 / LICENSE +- Go: github.com/apache/arrow-go/v18 v18.7.0 / NOTICE.txt ---------------------------------------------------------------------------- -Copyright (c) 2015, Gengo, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name of Gengo, Inc. nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. +Apache Arrow Go +Copyright 2016-2025 The Apache Software Foundation -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). --- Applies to ------------------------------------------------------------- -- Go: github.com/hashicorp/hcl/v2 v2.24.0 / LICENSE +- Go: github.com/apache/thrift v0.24.0 / LICENSE ---------------------------------------------------------------------------- -Copyright (c) 2014 HashiCorp, Inc. - -Mozilla Public License, version 2.0 +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -1. Definitions + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1.1. “Contributor” + 1. Definitions. - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -1.2. “Contributor Version” + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -1.3. “Contribution” + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. - means Covered Software of a particular Contributor. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -1.4. “Covered Software” + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -1.5. “Incompatible With Secondary Licenses” - means + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -1.6. “Executable Form” + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. - means any form of the work other than Source Code Form. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -1.7. “Larger Work” + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and -1.8. “License” + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and - means this document. + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and -1.9. “Licensable” + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. -1.10. “Modifications” + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. - means any of the following: + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. - b. any new file in Source Code Form that contains any Covered Software. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. -1.11. “Patent Claims” of a Contributor + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. + END OF TERMS AND CONDITIONS -1.12. “Secondary License” + APPENDIX: How to apply the Apache License to your work. - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -1.13. “Source Code Form” + Copyright [yyyy] [name of copyright owner] - means the form of the work preferred for making modifications. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -1.14. “You” (or “Your”) + http://www.apache.org/licenses/LICENSE-2.0 - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) + +-------------------------------------------------- +For lib/cpp/src/thrift/windows/SocketPair.cpp + +/* socketpair.c + * Copyright 2007 by Nathan C. Myers ; some rights reserved. + * This code is Free Software. It may be copied freely, in original or + * modified form, subject only to the restrictions that (1) the author is + * relieved from all responsibilities for any use for any purpose, and (2) + * this copyright notice must be retained, unchanged, in its entirety. If + * for any reason the author might be held responsible for any consequences + * of copying or use, license is withheld. + */ + + +-------------------------------------------------- +For lib/py/compat/win32/stdint.h + +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + + +-------------------------------------------------- +Codegen template in t_html_generator.h + +* Bootstrap v2.0.3 +* +* Copyright 2012 Twitter, Inc +* Licensed under the Apache License v2.0 +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Designed and built with all the love in the world @twitter by @mdo and @fat. + +--------------------------------------------------- +For t_cl_generator.cc + + * Copyright (c) 2008- Patrick Collison + * Copyright (c) 2006- Facebook + +--------------------------------------------------- + +--------------------------------------------------- +For compiler/cpp/src/thrift/generate/sha256.h + +SHA-256 implementation by Brad Conte (brad AT bradconte.com). +Source: https://github.com/B-Con/crypto-algorithms +The author has placed this code in the public domain (no copyright claimed). +No algorithmic changes were made; the file was adapted to a C++ header-only +form for inclusion in the Thrift compiler. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/apache/thrift v0.24.0 / NOTICE +---------------------------------------------------------------------------- + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation -2. License Grants and Conditions +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -2.1. Grants +--- Applies to ------------------------------------------------------------- +- Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 / LICENSE +---------------------------------------------------------------------------- - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: +Copyright (c) 2017 Martin Atkins - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -2.2. Effective Date +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. +--------- -2.3. Limitations on Grant Scope +Unicode table generation programs are under a separate copyright and license: - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: +Copyright (c) 2014 Couchbase, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain a copy of the License at - a. for any code that a Contributor has removed from Covered Software; or + http://www.apache.org/licenses/LICENSE-2.0 - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or +Unless required by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the specific language governing permissions +and limitations under the License. - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. +--------- - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). +Grapheme break data is provided as part of the Unicode character database, +copright 2016 Unicode, Inc, which is provided with the following license: -2.4. Subsequent Licenses +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +Unicode Data Files do not include PDF online code charts under the +directory http://www.unicode.org/Public/. + +Software includes any source code published in the Unicode Standard +or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +NOTICE TO USER: Carefully read the following legal agreement. +BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S +DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), +YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. +IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE +THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2017 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2017 Martin Atkins + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/beorn7/perks v1.0.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (C) 2013 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/bmatcuk/doublestar v1.3.4 / LICENSE +---------------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Bob Matcuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/cespare/xxhash/v2 v2.3.0 / LICENSE.txt +---------------------------------------------------------------------------- + +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/coreos/go-oidc/v3 v3.20.0 / LICENSE +- Go: github.com/zclconf/go-cty-yaml v1.2.0 / LICENSE +---------------------------------------------------------------------------- + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/coreos/go-oidc/v3 v3.20.0 / NOTICE +---------------------------------------------------------------------------- + +CoreOS Project +Copyright 2014 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). + +--- Applies to ------------------------------------------------------------- +- Go: github.com/duckdb/duckdb-go-bindings v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright 2018-2026 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/duckdb/duckdb-go/v2 v2.10505.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright 2019-2024 Marc Boeker +Copyright 2025-2026 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/dustin/go-humanize v1.0.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2005-2008 Dustin Sallings + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +--- Applies to ------------------------------------------------------------- +- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/go-openapi/inflect v1.0.0 / NOTICE +---------------------------------------------------------------------------- + +Copyright 2015-2025 go-swagger maintainers + +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 + +This software library, github.com/go-openapi/jsonpointer, includes software developed +by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this software except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0. + +This software is copied from, derived from, and inspired by other original software products. +It ships with copies of other software which license terms are recalled below. + +The original software was authored by Chris Farmiloe at https://bitbucket.org/pkg/inflect under a MIT License. + +ghttps://bitbucket.org/pkg/inflect +=========================== + +Copyright (c) 2011 Chris Farmiloe + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE +- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE +---------------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/goccy/go-json v0.10.6 / LICENSE +---------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2020 Masaaki Goshima + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/google/go-cmp v0.7.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2017 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/google/jsonschema-go v0.4.3 / LICENSE +---------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2025 JSON Schema Go Project Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/google/uuid v1.6.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2015, Gengo, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/hashicorp/hcl/v2 v2.24.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2014 HashiCorp, Inc. + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. -2.5. Representation +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. +You may add additional accurate notices of copyright ownership. -2.6. Fair Use +Exhibit B - “Incompatible With Secondary Licenses” Notice - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. -2.7. Conditions +--- Applies to ------------------------------------------------------------- +- Go: github.com/klauspost/compress v1.19.2 / LICENSE +---------------------------------------------------------------------------- - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -3. Responsibilities + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -3.1. Distribution of Source Form +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. +------------------ -3.2. Distribution of Executable Form +Files: gzhttp/* - If You distribute Covered Software in Executable Form then: + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. + 1. Definitions. -3.3. Distribution of a Larger Work + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -3.4. Notices + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. -3.5. Application of Additional Terms + END OF TERMS AND CONDITIONS - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. + APPENDIX: How to apply the Apache License to your work. -4. Inability to Comply Due to Statute or Regulation + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. + Copyright 2016-2017 The New York Times Company -5. Termination + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. + http://www.apache.org/licenses/LICENSE-2.0 -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. +------------------ -6. Disclaimer of Warranty +Files: s2/cmd/internal/readahead/* - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. +The MIT License (MIT) -7. Limitation of Liability +Copyright (c) 2015 Klaus Post - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -8. Litigation +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -9. Miscellaneous +--------------------- +Files: snappy/* +Files: internal/snapref/* - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -10. Versions of the License + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -10.1. New Versions +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. +----------------- -10.2. Effect of New Versions +Files: s2/cmd/internal/filepathx/* - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. +Copyright 2016 The filepathx Authors -10.3. Modified Versions +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Exhibit A - Source Code Form License Notice +--- Applies to ------------------------------------------------------------- +- Go: github.com/klauspost/cpuid/v2 v2.4.0 / LICENSE +---------------------------------------------------------------------------- - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. +The MIT License (MIT) -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +Copyright (c) 2015 Klaus Post -You may add additional accurate notices of copyright ownership. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -Exhibit B - “Incompatible With Secondary Licenses” Notice +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. --- Applies to ------------------------------------------------------------- - Go: github.com/knadh/koanf/maps v0.1.3 / LICENSE @@ -2330,6 +3325,38 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- Applies to ------------------------------------------------------------- +- Go: github.com/pierrec/lz4/v4 v4.1.29 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2015, Pierre Curto +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of xxHash nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + --- Applies to ------------------------------------------------------------- - Go: github.com/prometheus/client_golang v1.24.1 / NOTICE ---------------------------------------------------------------------------- @@ -2633,6 +3660,39 @@ license. See LICENSE.libyaml for more information. Modifications for cty interfacing copyright 2019 Martin Atkins, and distributed under the same license terms. +--- Applies to ------------------------------------------------------------- +- Go: github.com/zeebo/xxh3 v1.1.0 / LICENSE +---------------------------------------------------------------------------- + +BSD 2-Clause License + +Copyright (c) 2012-2014, Yann Collet +Copyright (c) 2019, Jeff Wendling +All rights reserved. + +xxHash Library + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + --- Applies to ------------------------------------------------------------- - Go: go.yaml.in/yaml/v3 v3.0.5 / LICENSE - Go: gopkg.in/yaml.v3 v3.0.1 / LICENSE diff --git a/cmd/bench/main.go b/cmd/bench/main.go index 882e9427..e167a81a 100644 --- a/cmd/bench/main.go +++ b/cmd/bench/main.go @@ -910,43 +910,43 @@ type latencyReport struct { } type serverReport struct { - BaselineAvailable bool `json:"baseline_available"` - ProcessStartTime float64 `json:"process_start_time_seconds"` - ProcessRestarted bool `json:"process_restarted"` - IngestRowsStart float64 `json:"ingest_rows_start"` - IngestRowsEnd float64 `json:"ingest_rows_end"` - IngestRowsDelta float64 `json:"ingest_rows_delta"` - RowsDroppedStart float64 `json:"rows_dropped_start"` - RowsDroppedEnd float64 `json:"rows_dropped_end"` - RowsDroppedDelta float64 `json:"rows_dropped_delta"` - LakePartitionsStart float64 `json:"lake_partitions_start"` - LakePartitions float64 `json:"lake_partitions"` - LakePartitionsDelta float64 `json:"lake_partitions_delta"` - LakeSizeBytesStart float64 `json:"lake_size_bytes_start"` - LakeSizeBytes float64 `json:"lake_size_bytes"` - LakeSizeBytesDelta float64 `json:"lake_size_bytes_delta"` - LakeGrowthBytesPerSec float64 `json:"lake_growth_bytes_per_sec"` - IngestQueueDepth float64 `json:"ingest_queue_depth"` - AvgRollupMs float64 `json:"avg_rollup_ms"` - AvgFlushMs float64 `json:"avg_flush_ms"` - AvgQueryMs float64 `json:"avg_query_ms"` - CPUSecondsStart float64 `json:"cpu_seconds_start"` - CPUSecondsEnd float64 `json:"cpu_seconds_end"` - CPUSecondsDelta float64 `json:"cpu_seconds_delta"` - CPUCores float64 `json:"cpu_cores"` - RSSBytes float64 `json:"rss_bytes"` - HeapAllocBytes float64 `json:"heap_alloc_bytes"` - AllocBytesStart float64 `json:"alloc_bytes_start"` - AllocBytesEnd float64 `json:"alloc_bytes_end"` - AllocBytesDelta float64 `json:"alloc_bytes_delta"` - AllocBytesPerSec float64 `json:"alloc_bytes_per_sec"` - GCPauseSecondsStart float64 `json:"gc_pause_seconds_start"` - GCPauseSecondsEnd float64 `json:"gc_pause_seconds_end"` - GCPauseSecondsDelta float64 `json:"gc_pause_seconds_delta"` - WriteGateWaitMs map[string]distributionReport `json:"write_gate_wait_ms,omitempty"` - WriteGateHoldMs map[string]distributionReport `json:"write_gate_hold_ms,omitempty"` - DuckLakeOperations map[string]backgroundOperationReport `json:"ducklake_operations,omitempty"` - Rollups map[string]rollupReport `json:"rollups,omitempty"` + BaselineAvailable bool `json:"baseline_available"` + ProcessStartTime float64 `json:"process_start_time_seconds"` + ProcessRestarted bool `json:"process_restarted"` + IngestRowsStart float64 `json:"ingest_rows_start"` + IngestRowsEnd float64 `json:"ingest_rows_end"` + IngestRowsDelta float64 `json:"ingest_rows_delta"` + RowsDroppedStart float64 `json:"rows_dropped_start"` + RowsDroppedEnd float64 `json:"rows_dropped_end"` + RowsDroppedDelta float64 `json:"rows_dropped_delta"` + ParquetFilesStart float64 `json:"parquet_partitions_start"` + ParquetFiles float64 `json:"parquet_partitions"` + ParquetFilesDelta float64 `json:"parquet_partitions_delta"` + ParquetSizeBytesStart float64 `json:"parquet_size_bytes_start"` + ParquetSizeBytes float64 `json:"parquet_size_bytes"` + ParquetSizeBytesDelta float64 `json:"parquet_size_bytes_delta"` + ParquetGrowthBytesPerSec float64 `json:"parquet_growth_bytes_per_sec"` + IngestQueueDepth float64 `json:"ingest_queue_depth"` + AvgRollupMs float64 `json:"avg_rollup_ms"` + AvgFlushMs float64 `json:"avg_flush_ms"` + AvgQueryMs float64 `json:"avg_query_ms"` + CPUSecondsStart float64 `json:"cpu_seconds_start"` + CPUSecondsEnd float64 `json:"cpu_seconds_end"` + CPUSecondsDelta float64 `json:"cpu_seconds_delta"` + CPUCores float64 `json:"cpu_cores"` + RSSBytes float64 `json:"rss_bytes"` + HeapAllocBytes float64 `json:"heap_alloc_bytes"` + AllocBytesStart float64 `json:"alloc_bytes_start"` + AllocBytesEnd float64 `json:"alloc_bytes_end"` + AllocBytesDelta float64 `json:"alloc_bytes_delta"` + AllocBytesPerSec float64 `json:"alloc_bytes_per_sec"` + GCPauseSecondsStart float64 `json:"gc_pause_seconds_start"` + GCPauseSecondsEnd float64 `json:"gc_pause_seconds_end"` + GCPauseSecondsDelta float64 `json:"gc_pause_seconds_delta"` + WriteGateWaitMs map[string]distributionReport `json:"write_gate_wait_ms,omitempty"` + WriteGateHoldMs map[string]distributionReport `json:"write_gate_hold_ms,omitempty"` + TelemetryOperations map[string]backgroundOperationReport `json:"telemetry_operations,omitempty"` + Rollups map[string]rollupReport `json:"rollups,omitempty"` } func printReport(r report) { @@ -982,11 +982,11 @@ func printReport(r report) { s := r.Server fmt.Printf("server (Δ over run):\n") fmt.Printf(" rows accepted=%.0f dropped=%.0f\n", s.IngestRowsDelta, s.RowsDroppedDelta) - fmt.Printf(" lake_partitions=%.0f lake_size=%.1fMB ingest_queue_depth=%.0f\n", - s.LakePartitions, s.LakeSizeBytes/(1<<20), s.IngestQueueDepth) + fmt.Printf(" parquet_partitions=%.0f parquet_size=%.1fMB ingest_queue_depth=%.0f\n", + s.ParquetFiles, s.ParquetSizeBytes/(1<<20), s.IngestQueueDepth) fmt.Printf(" avg rollup=%.1fms flush=%.1fms query=%.1fms\n", s.AvgRollupMs, s.AvgFlushMs, s.AvgQueryMs) - fmt.Printf(" cpu=%.2f core(s) rss=%.1fMB alloc=%.1fMB/s lake_growth=%.1fMB\n", - s.CPUCores, s.RSSBytes/(1<<20), s.AllocBytesPerSec/(1<<20), s.LakeSizeBytesDelta/(1<<20)) + fmt.Printf(" cpu=%.2f core(s) rss=%.1fMB alloc=%.1fMB/s parquet_growth=%.1fMB\n", + s.CPUCores, s.RSSBytes/(1<<20), s.AllocBytesPerSec/(1<<20), s.ParquetSizeBytesDelta/(1<<20)) } if r.Passed { fmt.Printf("verdict PASS\n") diff --git a/cmd/bench/metrics_report.go b/cmd/bench/metrics_report.go index 7763548b..de3aa05d 100644 --- a/cmd/bench/metrics_report.go +++ b/cmd/bench/metrics_report.go @@ -292,8 +292,8 @@ func serverDelta(base, final *metricSnapshot, durationSeconds float64) *serverRe } return round2(value / durationSeconds) } - lakePartitionsStart := base.total("fanout_lake_partitions") - lakeSizeStart := base.total("fanout_lake_size_bytes") + lakePartitionsStart := base.total("fanout_parquet_files") + lakeSizeStart := base.total("fanout_parquet_size_bytes") cpuSeconds := delta("process_cpu_seconds_total") allocBytes := delta("go_memstats_alloc_bytes_total") // process_start_time_seconds is constant for the life of a process, so a @@ -302,43 +302,43 @@ func serverDelta(base, final *metricSnapshot, durationSeconds float64) *serverRe startTimeBefore := base.total("process_start_time_seconds") startTimeAfter := final.total("process_start_time_seconds") return &serverReport{ - BaselineAvailable: baselineAvailable, - ProcessStartTime: startTimeAfter, - ProcessRestarted: baselineAvailable && startTimeBefore > 0 && startTimeAfter != startTimeBefore, - IngestRowsStart: base.total("fanout_ingest_rows_total"), - IngestRowsEnd: final.total("fanout_ingest_rows_total"), - IngestRowsDelta: delta("fanout_ingest_rows_total"), - RowsDroppedStart: base.total("fanout_rows_dropped_total"), - RowsDroppedEnd: final.total("fanout_rows_dropped_total"), - RowsDroppedDelta: delta("fanout_rows_dropped_total"), - LakePartitionsStart: lakePartitionsStart, - LakePartitions: final.total("fanout_lake_partitions"), - LakePartitionsDelta: final.total("fanout_lake_partitions") - lakePartitionsStart, - LakeSizeBytesStart: lakeSizeStart, - LakeSizeBytes: final.total("fanout_lake_size_bytes"), - LakeSizeBytesDelta: final.total("fanout_lake_size_bytes") - lakeSizeStart, - LakeGrowthBytesPerSec: rate(final.total("fanout_lake_size_bytes") - lakeSizeStart), - IngestQueueDepth: final.total("fanout_ingest_queue_depth"), - AvgRollupMs: averageDurationMs(base, final, "fanout_rollup_duration_seconds"), - AvgFlushMs: averageDurationMs(base, final, "fanout_flush_duration_seconds"), - AvgQueryMs: averageDurationMs(base, final, "fanout_query_duration_seconds"), - CPUSecondsStart: round4(base.total("process_cpu_seconds_total")), - CPUSecondsEnd: round4(final.total("process_cpu_seconds_total")), - CPUSecondsDelta: round4(cpuSeconds), - CPUCores: perSecond(cpuSeconds, durationSeconds), - RSSBytes: final.total("process_resident_memory_bytes"), - HeapAllocBytes: final.total("go_memstats_heap_alloc_bytes"), - AllocBytesStart: base.total("go_memstats_alloc_bytes_total"), - AllocBytesEnd: final.total("go_memstats_alloc_bytes_total"), - AllocBytesDelta: allocBytes, - AllocBytesPerSec: rate(allocBytes), - GCPauseSecondsStart: round4(base.total("go_gc_duration_seconds_sum")), - GCPauseSecondsEnd: round4(final.total("go_gc_duration_seconds_sum")), - GCPauseSecondsDelta: round4(delta("go_gc_duration_seconds_sum")), - WriteGateWaitMs: histogramReports(base, final, "fanout_write_gate_wait_seconds", "operation"), - WriteGateHoldMs: histogramReports(base, final, "fanout_write_gate_hold_seconds", "operation"), - DuckLakeOperations: backgroundReports(base, final), - Rollups: rollupReports(base, final), + BaselineAvailable: baselineAvailable, + ProcessStartTime: startTimeAfter, + ProcessRestarted: baselineAvailable && startTimeBefore > 0 && startTimeAfter != startTimeBefore, + IngestRowsStart: base.total("fanout_ingest_rows_total"), + IngestRowsEnd: final.total("fanout_ingest_rows_total"), + IngestRowsDelta: delta("fanout_ingest_rows_total"), + RowsDroppedStart: base.total("fanout_rows_dropped_total"), + RowsDroppedEnd: final.total("fanout_rows_dropped_total"), + RowsDroppedDelta: delta("fanout_rows_dropped_total"), + ParquetFilesStart: lakePartitionsStart, + ParquetFiles: final.total("fanout_parquet_files"), + ParquetFilesDelta: final.total("fanout_parquet_files") - lakePartitionsStart, + ParquetSizeBytesStart: lakeSizeStart, + ParquetSizeBytes: final.total("fanout_parquet_size_bytes"), + ParquetSizeBytesDelta: final.total("fanout_parquet_size_bytes") - lakeSizeStart, + ParquetGrowthBytesPerSec: rate(final.total("fanout_parquet_size_bytes") - lakeSizeStart), + IngestQueueDepth: final.total("fanout_ingest_queue_depth"), + AvgRollupMs: averageDurationMs(base, final, "fanout_rollup_duration_seconds"), + AvgFlushMs: averageDurationMs(base, final, "fanout_flush_duration_seconds"), + AvgQueryMs: averageDurationMs(base, final, "fanout_query_duration_seconds"), + CPUSecondsStart: round4(base.total("process_cpu_seconds_total")), + CPUSecondsEnd: round4(final.total("process_cpu_seconds_total")), + CPUSecondsDelta: round4(cpuSeconds), + CPUCores: perSecond(cpuSeconds, durationSeconds), + RSSBytes: final.total("process_resident_memory_bytes"), + HeapAllocBytes: final.total("go_memstats_heap_alloc_bytes"), + AllocBytesStart: base.total("go_memstats_alloc_bytes_total"), + AllocBytesEnd: final.total("go_memstats_alloc_bytes_total"), + AllocBytesDelta: allocBytes, + AllocBytesPerSec: rate(allocBytes), + GCPauseSecondsStart: round4(base.total("go_gc_duration_seconds_sum")), + GCPauseSecondsEnd: round4(final.total("go_gc_duration_seconds_sum")), + GCPauseSecondsDelta: round4(delta("go_gc_duration_seconds_sum")), + WriteGateWaitMs: histogramReports(base, final, "fanout_write_gate_wait_seconds", "operation"), + WriteGateHoldMs: histogramReports(base, final, "fanout_write_gate_hold_seconds", "operation"), + TelemetryOperations: backgroundReports(base, final), + Rollups: rollupReports(base, final), } } @@ -425,10 +425,10 @@ func histogramDelta(base, final *metricSnapshot, name string, filters map[string func backgroundReports(base, final *metricSnapshot) map[string]backgroundOperationReport { operations := unionStrings( - base.labelValues("fanout_ducklake_operation_total", "operation", nil), - final.labelValues("fanout_ducklake_operation_total", "operation", nil), - base.labelValues("fanout_ducklake_operation_duration_seconds_count", "operation", nil), - final.labelValues("fanout_ducklake_operation_duration_seconds_count", "operation", nil), + base.labelValues("fanout_telemetry_operation_total", "operation", nil), + final.labelValues("fanout_telemetry_operation_total", "operation", nil), + base.labelValues("fanout_telemetry_operation_duration_seconds_count", "operation", nil), + final.labelValues("fanout_telemetry_operation_duration_seconds_count", "operation", nil), ) if len(operations) == 0 { return nil @@ -437,8 +437,8 @@ func backgroundReports(base, final *metricSnapshot) map[string]backgroundOperati for _, operation := range operations { filters := map[string]string{"operation": operation} reports[operation] = backgroundOperationReport{ - DurationMs: histogramDelta(base, final, "fanout_ducklake_operation_duration_seconds", filters), - Outcomes: counterOutcomes(base, final, "fanout_ducklake_operation_total", "result", filters), + DurationMs: histogramDelta(base, final, "fanout_telemetry_operation_duration_seconds", filters), + Outcomes: counterOutcomes(base, final, "fanout_telemetry_operation_total", "result", filters), } } return reports diff --git a/cmd/bench/metrics_report_test.go b/cmd/bench/metrics_report_test.go index b117d5e0..a87a4746 100644 --- a/cmd/bench/metrics_report_test.go +++ b/cmd/bench/metrics_report_test.go @@ -9,14 +9,14 @@ import ( func TestParseMetricSnapshotPreservesLabelsAndTotals(t *testing.T) { snapshot := mustMetricSnapshot(t, ` # HELP fanout_test_total test metric -fanout_test_total{operation="merge",detail="quoted\" value\\path"} 2 +fanout_test_total{operation="compaction",detail="quoted\" value\\path"} 2 fanout_test_total{operation="maintenance",detail="plain"} 3 `) if got := snapshot.total("fanout_test_total"); got != 5 { t.Fatalf("total = %v, want 5", got) } - if got := snapshot.filteredTotal("fanout_test_total", map[string]string{"operation": "merge"}); got != 2 { + if got := snapshot.filteredTotal("fanout_test_total", map[string]string{"operation": "compaction"}); got != 2 { t.Fatalf("merge total = %v, want 2", got) } if got := snapshot.Samples[0].Labels["detail"]; got != "quoted\" value\\path" { @@ -29,10 +29,10 @@ func TestServerDeltaReportsOperationAndRuntimeDistributions(t *testing.T) { fanout_ingest_rows_total{signal="spans"} 10 fanout_ingest_rows_total{signal="logs"} 5 fanout_rows_dropped_total{signal="spans"} 0 -fanout_lake_partitions{signal="spans"} 2 -fanout_lake_partitions{signal="logs"} 1 -fanout_lake_size_bytes{signal="spans"} 60 -fanout_lake_size_bytes{signal="logs"} 40 +fanout_parquet_files{signal="spans"} 2 +fanout_parquet_files{signal="logs"} 1 +fanout_parquet_size_bytes{signal="spans"} 60 +fanout_parquet_size_bytes{signal="logs"} 40 fanout_ingest_queue_depth{signal="spans"} 2 fanout_ingest_queue_depth{signal="logs"} 1 process_cpu_seconds_total 10 @@ -57,14 +57,14 @@ fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.100"} 1 fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="+Inf"} 1 fanout_write_gate_hold_seconds_sum{operation="ingest_spans"} 0.005 fanout_write_gate_hold_seconds_count{operation="ingest_spans"} 1 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="0.010"} 1 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="0.100"} 1 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="1.000"} 1 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="+Inf"} 1 -fanout_ducklake_operation_duration_seconds_sum{operation="merge"} 0.005 -fanout_ducklake_operation_duration_seconds_count{operation="merge"} 1 -fanout_ducklake_operation_total{operation="merge",result="success"} 1 -fanout_ducklake_operation_total{operation="merge",result="throttled"} 2 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.010"} 1 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.100"} 1 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="1.000"} 1 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="+Inf"} 1 +fanout_telemetry_operation_duration_seconds_sum{operation="compaction"} 0.005 +fanout_telemetry_operation_duration_seconds_count{operation="compaction"} 1 +fanout_telemetry_operation_total{operation="compaction",result="success"} 1 +fanout_telemetry_operation_total{operation="compaction",result="throttled"} 2 fanout_rollup_enabled{rollup="service"} 1 fanout_rollup_watermark_timestamp_seconds{rollup="service"} 100 fanout_rollup_source_timestamp_seconds{rollup="service"} 105 @@ -82,10 +82,10 @@ fanout_rollup_component_total{rollup="service",result="success"} 1 fanout_ingest_rows_total{signal="spans"} 30 fanout_ingest_rows_total{signal="logs"} 15 fanout_rows_dropped_total{signal="spans"} 0 -fanout_lake_partitions{signal="spans"} 3 -fanout_lake_partitions{signal="logs"} 2 -fanout_lake_size_bytes{signal="spans"} 100 -fanout_lake_size_bytes{signal="logs"} 80 +fanout_parquet_files{signal="spans"} 3 +fanout_parquet_files{signal="logs"} 2 +fanout_parquet_size_bytes{signal="spans"} 100 +fanout_parquet_size_bytes{signal="logs"} 80 fanout_ingest_queue_depth{signal="spans"} 0 fanout_ingest_queue_depth{signal="logs"} 1 process_cpu_seconds_total 14 @@ -110,15 +110,15 @@ fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.100"} 3 fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="+Inf"} 3 fanout_write_gate_hold_seconds_sum{operation="ingest_spans"} 0.035 fanout_write_gate_hold_seconds_count{operation="ingest_spans"} 3 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="0.010"} 1 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="0.100"} 2 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="1.000"} 3 -fanout_ducklake_operation_duration_seconds_bucket{operation="merge",le="+Inf"} 3 -fanout_ducklake_operation_duration_seconds_sum{operation="merge"} 0.125 -fanout_ducklake_operation_duration_seconds_count{operation="merge"} 3 -fanout_ducklake_operation_total{operation="merge",result="success"} 3 -fanout_ducklake_operation_total{operation="merge",result="throttled"} 5 -fanout_ducklake_operation_total{operation="merge",result="error"} 1 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.010"} 1 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.100"} 2 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="1.000"} 3 +fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="+Inf"} 3 +fanout_telemetry_operation_duration_seconds_sum{operation="compaction"} 0.125 +fanout_telemetry_operation_duration_seconds_count{operation="compaction"} 3 +fanout_telemetry_operation_total{operation="compaction",result="success"} 3 +fanout_telemetry_operation_total{operation="compaction",result="throttled"} 5 +fanout_telemetry_operation_total{operation="compaction",result="error"} 1 fanout_rollup_enabled{rollup="service"} 1 fanout_rollup_watermark_timestamp_seconds{rollup="service"} 200 fanout_rollup_source_timestamp_seconds{rollup="service"} 212 @@ -141,9 +141,9 @@ fanout_rollup_component_total{rollup="service",result="noop"} 2 assertFloat(t, "ingest rows start", report.IngestRowsStart, 15) assertFloat(t, "ingest rows end", report.IngestRowsEnd, 45) assertFloat(t, "ingest rows", report.IngestRowsDelta, 30) - assertFloat(t, "lake partitions delta", report.LakePartitionsDelta, 2) - assertFloat(t, "lake size delta", report.LakeSizeBytesDelta, 80) - assertFloat(t, "lake growth rate", report.LakeGrowthBytesPerSec, 10) + assertFloat(t, "lake partitions delta", report.ParquetFilesDelta, 2) + assertFloat(t, "lake size delta", report.ParquetSizeBytesDelta, 80) + assertFloat(t, "lake growth rate", report.ParquetGrowthBytesPerSec, 10) assertFloat(t, "average rollup", report.AvgRollupMs, 750) assertFloat(t, "average flush", report.AvgFlushMs, 200) assertFloat(t, "average query", report.AvgQueryMs, 500) @@ -170,7 +170,7 @@ fanout_rollup_component_total{rollup="service",result="noop"} 2 assertFloat(t, "wait p95", wait.P95Ms, 100) assertFloat(t, "hold mean", report.WriteGateHoldMs["ingest_spans"].MeanMs, 15) - merge := report.DuckLakeOperations["merge"] + merge := report.TelemetryOperations["compaction"] if merge.DurationMs.Count != 2 { t.Fatalf("merge duration count = %d, want 2", merge.DurationMs.Count) } diff --git a/cmd/bench/verdict.go b/cmd/bench/verdict.go index 2168016d..0f069f27 100644 --- a/cmd/bench/verdict.go +++ b/cmd/bench/verdict.go @@ -85,9 +85,9 @@ func evaluateReport(cfg config, report report, infrastructureFailures []string) fails = append(fails, fmt.Sprintf("%s rollup errors=%.0f", name, errors)) } } - for _, name := range sortedMapKeys(report.Server.DuckLakeOperations) { - if errors := report.Server.DuckLakeOperations[name].Outcomes["error"]; errors > 0 { - fails = append(fails, fmt.Sprintf("ducklake %s errors=%.0f", name, errors)) + for _, name := range sortedMapKeys(report.Server.TelemetryOperations) { + if errors := report.Server.TelemetryOperations[name].Outcomes["error"]; errors > 0 { + fails = append(fails, fmt.Sprintf("telemetry %s errors=%.0f", name, errors)) } } } diff --git a/cmd/bench/verdict_test.go b/cmd/bench/verdict_test.go index cfa29c53..d67ef9ed 100644 --- a/cmd/bench/verdict_test.go +++ b/cmd/bench/verdict_test.go @@ -152,18 +152,18 @@ func TestEvaluateReportFailsOnBackgroundWorkErrors(t *testing.T) { "service": {Outcomes: map[string]float64{"success": 30}}, "edge": {Outcomes: map[string]float64{"error": 12}}, }, - DuckLakeOperations: map[string]backgroundOperationReport{ + TelemetryOperations: map[string]backgroundOperationReport{ "maintenance": {Outcomes: map[string]float64{"error": 3}}, - "merge": {Outcomes: map[string]float64{"success": 9}}, + "compaction": {Outcomes: map[string]float64{"success": 9}}, }, }, }, nil) - for _, want := range []string{"edge rollup errors=12", "ducklake maintenance errors=3"} { + for _, want := range []string{"edge rollup errors=12", "telemetry maintenance errors=3"} { if !hasFailure(failures, want) { t.Fatalf("failures %q do not contain %q", failures, want) } } - if hasFailure(failures, "service rollup") || hasFailure(failures, "ducklake merge") { + if hasFailure(failures, "service rollup") || hasFailure(failures, "telemetry merge") { t.Errorf("healthy components reported as failures: %q", failures) } } diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index dca92d8f..5623446c 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -35,13 +35,14 @@ import ( "github.com/labstack/fanout/internal/dashboard" "github.com/labstack/fanout/internal/ingest" "github.com/labstack/fanout/internal/intelligence" - "github.com/labstack/fanout/internal/lake" "github.com/labstack/fanout/internal/mcp" appmetrics "github.com/labstack/fanout/internal/metrics" "github.com/labstack/fanout/internal/observability" "github.com/labstack/fanout/internal/query" "github.com/labstack/fanout/internal/settings" "github.com/labstack/fanout/internal/store" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/labstack/fanout/internal/ui" ) @@ -95,10 +96,10 @@ func main() { os.Exit(1) } - // Channels for ingest → lake writer - chSpans := make(chan lake.SpanRow, 10000) - chLogs := make(chan lake.LogRow, 10000) - chMetrics := make(chan lake.MetricRow, 10000) + // Channels for OTLP decoding → the single authoritative telemetry writer. + chSpans := make(chan telemetry.Span, 10000) + chLogs := make(chan telemetry.Log, 10000) + chMetrics := make(chan telemetry.Metric, 10000) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -109,18 +110,23 @@ func main() { // Error channel for goroutine failures errCh := make(chan error, 4) - // Start DuckDB + rollups - q, err := query.NewDuck(ctx, cfg) + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + slog.Error("telemetry store init failed", "err", err) + os.Exit(1) + } + defer repository.Close() + + // DuckDB is the SQL engine over open Parquet; hot indexed reads use the same + // repository directly through the typed observability kernel. + q, err := query.NewDuck(ctx, cfg, repository) if err != nil { slog.Error("duckdb init failed", "err", err) os.Exit(1) } defer q.Close() - // Start Lake Writer. Share the query layer's write gate so appender flushes - // serialize with rollup/maintenance commits when the pool holds >1 connection. - writer := lake.NewWriter(cfg, q.DB, chSpans, chLogs, chMetrics) - writer.UseWriteGate(q.WriteGate()) + writer := telemetrystore.NewWriter(repository, cfg.FlushInterval, cfg.FlushBatchSize, chSpans, chLogs, chMetrics) writerResult := make(chan error, 1) go func() { err := writer.Run(ctx) @@ -129,7 +135,7 @@ func main() { // failure cannot be lost in the close(done) -> goroutine-send scheduling gap. writerResult <- err if err != nil { - errCh <- fmt.Errorf("lake writer: %w", err) + errCh <- fmt.Errorf("telemetry writer: %w", err) } }() @@ -291,8 +297,8 @@ func main() { // Fanout owns telemetry semantics; agents and web clients consume this one // typed query kernel through deterministic HTTP or standard MCP tools. // Route both HTTP and MCP reads through Duck's retrying adapter. Passing the - // raw *sql.DB here bypassed the DuckLake maintenance-race protection. - queries := observability.New(q) + // raw *sql.DB here bypassed the Telemetry maintenance-race protection. + queries := observability.New(q, repository) api.NewObservabilityHandler(queries).Register(e.Group("/api/observability", api.RequireCapability(api.ReadTelemetry))) api.RegisterIntelligenceRoutes(e, detector) dashboards := dashboard.New(sqlite.DB) @@ -452,7 +458,7 @@ func main() { cancel() writer.Wait() if err := <-writerResult; err != nil { - slog.Error("lake writer stopped with unwritten telemetry", "err", err) + slog.Error("telemetry writer stopped with unwritten telemetry", "err", err) } httpCancel() // triggers graceful HTTP shutdown (5s timeout) } diff --git a/cmd/storage-poc/main.go b/cmd/storage-poc/main.go new file mode 100644 index 00000000..eb1d9321 --- /dev/null +++ b/cmd/storage-poc/main.go @@ -0,0 +1,525 @@ +// Command storage-poc compares a workload-specific immutable segment format +// with native DuckDB and Parquet on Fanout-shaped spans. It is an experiment, +// not a supported Fanout command. +package main + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "sync" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/labstack/fanout/internal/storagebench" + "github.com/labstack/fanout/internal/telemetry/segment" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" +) + +type result struct { + name string + writeRate float64 + rollupBuild time.Duration + maintenance time.Duration + diskBytes int64 + endpoint time.Duration + trace time.Duration + rawService time.Duration + recovery time.Duration + queryRowCount uint64 + mixedWrite float64 + mixedReadP95 time.Duration +} + +func main() { + rows := flag.Int("rows", 1_000_000, "number of synthetic spans") + batch := flag.Int("batch", 50_000, "rows per durable append") + repeats := flag.Int("repeats", 21, "query repetitions used for medians") + mixedRows := flag.Int("mixed-rows", 200_000, "additional rows written while trace reads run at 100 qps") + engine := flag.String("engine", "all", "engines to run: all, repository, custom, or duck") + keep := flag.String("keep", "", "keep artifacts in this directory instead of a temporary directory") + flag.Parse() + if *rows <= 0 || *batch <= 0 || *repeats <= 0 { + fmt.Fprintln(os.Stderr, "rows, batch, and repeats must be positive") + os.Exit(2) + } + + root := *keep + if root == "" { + var err error + root, err = os.MkdirTemp("", "fanout-storage-poc-") + if err != nil { + fatal(err) + } + defer os.RemoveAll(root) + } else if err := os.MkdirAll(root, 0o755); err != nil { + fatal(err) + } + + base := time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC).UnixNano() + targetTrace := storagebench.TraceID(uint64(*rows / 2 / 5)) + start := base + end := base + storagebench.DayNanos + fmt.Printf("Fanout storage POC: %d spans, %d-row commits, %s/%s, %d CPUs\n", *rows, *batch, runtime.GOOS, runtime.GOARCH, runtime.NumCPU()) + + var results []result + if *engine == "all" || *engine == "repository" { + repositoryResult, err := runRepository(filepath.Join(root, "repository"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) + if err != nil { + fatal(fmt.Errorf("production repository: %w", err)) + } + results = append(results, repositoryResult) + } + var custom result + if *engine == "all" || *engine == "custom" { + var err error + custom, err = runCustom(filepath.Join(root, "fanseg"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) + if err != nil { + fatal(fmt.Errorf("custom segments: %w", err)) + } + results = append(results, custom) + } + if *engine == "all" || *engine == "duck" { + duck, parquet, err := runDuck(filepath.Join(root, "duck.db"), filepath.Join(root, "parquet"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) + if err != nil { + fatal(fmt.Errorf("duckdb: %w", err)) + } + results = append(results, duck, parquet) + } + if len(results) == 0 { + fatal(fmt.Errorf("unknown engine %q", *engine)) + } + + fmt.Println() + fmt.Printf("%-20s %14s %13s %12s %12s %12s %12s %12s\n", "storage / execution", "write rows/s", "rollup build", "maintenance", "disk MiB", "endpoint", "trace", "raw service") + for _, r := range results { + rollup := "live" + if r.rollupBuild > 0 { + rollup = formatDuration(r.rollupBuild) + } + fmt.Printf("%-20s %14.0f %13s %12s %12.1f %12s %12s %12s\n", r.name, r.writeRate, rollup, formatDuration(r.maintenance), float64(r.diskBytes)/(1<<20), formatDuration(r.endpoint), formatDuration(r.trace), formatDuration(r.rawService)) + } + fmt.Println("\nMixed load: committed writes plus full trace reads at 100 qps") + fmt.Printf("%-20s %14s %14s\n", "storage / execution", "write rows/s", "trace p95") + for _, r := range results { + if r.mixedWrite == 0 { + continue + } + fmt.Printf("%-20s %14.0f %14s\n", r.name, r.mixedWrite, formatDuration(r.mixedReadP95)) + } + if custom.name != "" { + fmt.Printf("\nfanseg reopen/recovery: %s; trace rows: %d\n", formatDuration(custom.recovery), custom.queryRowCount) + } + if *keep != "" { + fmt.Printf("artifacts: %s\n", root) + } +} + +func runRepository(dir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, error) { + repository, err := telemetrystore.Open(dir) + if err != nil { + return result{}, err + } + defer repository.Close() + writeStart := time.Now() + for offset := 0; offset < total; offset += batch { + rows := storagebench.Rows(offset, min(batch, total-offset), total, base) + if err := repository.Commit(telemetrystore.Batch{ID: fmt.Sprintf("initial-%08d", offset), Spans: rows}); err != nil { + return result{}, err + } + } + writeElapsed := time.Since(writeStart) + var endpointSink []segment.Endpoint + endpoint := median(repeats, func() error { + endpointSink = repository.Spans.Endpoints("default", "service-00", start, end, 20) + return nil + }) + var traceSink []segment.Span + trace := median(repeats, func() (err error) { traceSink, err = repository.Spans.Trace(targetTrace); return err }) + var aggregateSink segment.Aggregate + raw := median(repeats, func() (err error) { + aggregateSink, err = repository.Spans.ScanService("default", "service-00", start, end) + return err + }) + if len(endpointSink) == 0 || aggregateSink.Calls == 0 { + return result{}, errors.New("production repository queries returned no rows") + } + disk, err := directoryBytes(dir) + if err != nil { + return result{}, err + } + mixedWrite, mixedP95, err := mixedLoad(mixedRows, + func() error { _, err := repository.Spans.Trace(targetTrace); return err }, + func() error { + finalTotal := total + mixedRows + for offset := 0; offset < mixedRows; offset += batch { + rows := storagebench.Rows(total+offset, min(batch, mixedRows-offset), finalTotal, base) + if err := repository.Commit(telemetrystore.Batch{ID: fmt.Sprintf("mixed-%08d", offset), Spans: rows}); err != nil { + return err + } + } + return nil + }) + if err != nil { + return result{}, err + } + return result{name: "Fanout + Parquet", writeRate: float64(total) / writeElapsed.Seconds(), diskBytes: disk, endpoint: endpoint, trace: trace, rawService: raw, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil +} + +func directoryBytes(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + } + return nil + }) + return total, err +} + +func runCustom(dir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, error) { + store, err := segment.Open(dir) + if err != nil { + return result{}, err + } + writeStart := time.Now() + for offset := 0; offset < total; offset += batch { + count := min(batch, total-offset) + if err := store.Append(storagebench.Rows(offset, count, total, base)); err != nil { + return result{}, err + } + } + writeElapsed := time.Since(writeStart) + maintenanceStart := time.Now() + if err := store.CompactOldest(store.SegmentCount()); err != nil { + return result{}, err + } + maintenance := time.Since(maintenanceStart) + disk, err := store.DiskBytes() + if err != nil { + return result{}, err + } + if err := store.Close(); err != nil { + return result{}, err + } + reopenStart := time.Now() + store, err = segment.Open(dir) + if err != nil { + return result{}, err + } + recovery := time.Since(reopenStart) + defer store.Close() + + var endpointSink []segment.Endpoint + endpoint := median(repeats, func() error { endpointSink = store.Endpoints("default", "service-00", start, end, 20); return nil }) + var traceSink []segment.Span + trace := median(repeats, func() (err error) { traceSink, err = store.Trace(targetTrace); return err }) + var aggSink segment.Aggregate + raw := median(repeats, func() (err error) { aggSink, err = store.ScanService("default", "service-00", start, end); return err }) + if len(endpointSink) == 0 || aggSink.Calls == 0 { + return result{}, fmt.Errorf("queries returned no rows") + } + mixedWrite, mixedP95, err := mixedCustom(store, total, mixedRows, batch, base, targetTrace) + if err != nil { + return result{}, err + } + return result{name: "fanseg + direct", writeRate: float64(total) / writeElapsed.Seconds(), maintenance: maintenance, diskBytes: disk, endpoint: endpoint, trace: trace, rawService: raw, recovery: recovery, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil +} + +func runDuck(dbPath, parquetDir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, result, error) { + connector, err := duckdb.NewConnector(dbPath, nil) + if err != nil { + return result{}, result{}, err + } + db := sql.OpenDB(connector) + db.SetMaxOpenConns(4) + defer db.Close() + if _, err := db.Exec(`CREATE TABLE spans ( + namespace VARCHAR, trace_id VARCHAR, span_id VARCHAR, parent_span_id VARCHAR, + service_name VARCHAR, name VARCHAR, kind VARCHAR, start_time TIMESTAMP_NS, + end_time TIMESTAMP_NS, start_ns BIGINT, end_ns BIGINT, duration_ms DOUBLE, + status_code VARCHAR, status_msg VARCHAR, resource JSON, attributes JSON, + events JSON, links JSON, trace_state VARCHAR, flags UINTEGER, + scope_name VARCHAR, scope_version VARCHAR, ingested_at TIMESTAMP_NS, + ingested_ns BIGINT, http_method VARCHAR, http_status_code VARCHAR, + http_route VARCHAR, db_system VARCHAR, rpc_method VARCHAR, rpc_service VARCHAR, + peer_service VARCHAR, service_version VARCHAR, deployment_env VARCHAR, + exception_type VARCHAR, exception_message VARCHAR + )`); err != nil { + return result{}, result{}, err + } + + writeStart := time.Now() + for offset := 0; offset < total; offset += batch { + rows := storagebench.Rows(offset, min(batch, total-offset), total, base) + if err := appendDuck(db, rows); err != nil { + return result{}, result{}, err + } + } + writeElapsed := time.Since(writeStart) + rollupStart := time.Now() + if _, err := db.Exec(`CREATE TABLE endpoint_rollup AS + SELECT start_ns - start_ns % 300000000000 AS bucket, namespace, service_name, http_method, http_route, + count(*)::UBIGINT AS calls, count(*) FILTER (WHERE status_code='ERROR')::UBIGINT AS errors, + sum(duration_ms) AS duration_ms, approx_quantile(duration_ms, 0.95) AS p95_ms + FROM spans GROUP BY ALL`); err != nil { + return result{}, result{}, err + } + rollupBuild := time.Since(rollupStart) + checkpointStart := time.Now() + if _, err := db.Exec("CHECKPOINT"); err != nil { + return result{}, result{}, err + } + checkpointElapsed := time.Since(checkpointStart) + info, err := os.Stat(dbPath) + if err != nil { + return result{}, result{}, err + } + + endpointQuery := `SELECT service_name, http_method, http_route, sum(calls) calls, sum(errors) errors, + sum(duration_ms) / sum(calls) average_ms, max(p95_ms) p95_ms + FROM endpoint_rollup WHERE namespace=? AND service_name=? AND bucket>=? AND bucket=? AND start_ns=? AND bucket=? AND start_ns= time.Second { + return fmt.Sprintf("%.2fs", value.Seconds()) + } + if value >= time.Millisecond { + return fmt.Sprintf("%.2fms", float64(value)/float64(time.Millisecond)) + } + return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) +} + +func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-poc:", err); os.Exit(1) } diff --git a/docs/storage-architecture-options.md b/docs/storage-architecture-options.md new file mode 100644 index 00000000..5e859946 --- /dev/null +++ b/docs/storage-architecture-options.md @@ -0,0 +1,412 @@ +# Fanout storage architecture options + +**Decision date:** August 2026 + +**Product constraint:** one distributable Fanout binary +**Workload:** high-volume OTLP spans, logs, and metrics with fast dashboards, +trace lookup, attribute filtering, retention, and ad-hoc SQL + +## Recommendation + +Use a hybrid architecture: + +1. **Fanout columnar segments** for hot telemetry. +2. **Direct Fanout execution** for known product queries. +3. **Parquet** as the durable open SQL copy, written in the same commit. +4. **DuckDB** for arbitrary SQL over Parquet. +5. **SQLite** for control-plane data only. +6. **Do not use DuckLake, Iceberg, or chDB initially.** + +```text +OTLP ingestion + │ + ▼ +Fanout hot columnar store (.fseg) + ├── trace and promoted-attribute indexes + ├── ingestion-time service/endpoint rollups + ├── direct dashboard and trace execution + └── atomic manifest + streaming compaction + │ same durable commit + ▼ + Parquet files + │ + ▼ + DuckDB ad-hoc SQL + +SQLite: users, configuration, sessions, alerts, and other control data +``` + +This is still a single-binary product. Fanout owns the high-performance hot +path; embedded DuckDB supplies a mature SQL engine without owning ingestion or +table lifecycle. + +## First: separate the layers + +Several technologies under consideration solve different problems and are not +direct substitutes. + +| Layer | Purpose | Candidates | +|---|---|---| +| Physical format | Encodes column values in files | Fanout segments, Parquet, MergeTree parts, DuckDB native pages | +| Table management | Tracks files, commits, snapshots, schema, and deletion | Fanout manifest, DuckLake, Iceberg v3 | +| Query execution | Plans and executes filters, joins, and aggregations | Fanout direct execution, DuckDB, ClickHouse through chDB | +| Control database | Stores small transactional product state | SQLite | + +Important distinctions: + +- **Parquet is a file format**, not a database or query engine. +- **Iceberg uses Parquet in this proposal** and adds table metadata and commit + semantics above it. +- **DuckLake is a table-management layer for DuckDB and Parquet.** +- **DuckDB is a query engine and native database.** It can query plain Parquet + without DuckLake. +- **chDB embeds ClickHouse.** Its primary format is ClickHouse MergeTree parts; + it can also read and write Parquet. +- **SQLite does not overlap with the telemetry engines.** It remains the right + database for Fanout's low-volume control plane. + +## Measured result + +The normalized POC used one million complete Fanout-shaped spans, 50,000-row +commits, live endpoint rollups, complete trace reads, and another 200,000 rows +under concurrent trace load at 100 queries per second. + +| Storage / execution | Write rows/s | Endpoint | Full trace | Raw scan | Mixed write | Mixed trace p95 | Active disk | Peak RSS | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| **Production repository: Fanout + Parquet** | **165,875** | **0.172 ms** | **0.517 ms** | 26.18 ms | **167,353/s** | **0.76 ms** | 56.6 MiB | Not isolated | +| **Fanout columnar + direct** | **520,879** | **0.224 ms** | **0.507 ms** | 26.98 ms | **528,668/s** | **1.33 ms** | 34.4 MiB | **197 MiB** | +| DuckDB native | 98,030 | 0.905 ms | 1.54 ms | **1.44 ms** | 85,514/s | 2.14 ms | 47.5 MiB | 1,693 MiB | +| Zstd Parquet + DuckDB | 94,824 effective | 1.35 ms | 10.53 ms | 3.88 ms | n/a | n/a | **21.8 MiB** | Included in DuckDB process | +| chDB MergeTree | 129,724 | 2.81 ms | 7.39 ms | 6.06 ms | 118,722/s | 9.61 ms | 38.1 MiB | 699 MiB | + +Maintenance measurements: + +| Operation | Time | +|---|---:| +| Fanout compressed-block compaction | **135 ms** | +| DuckDB endpoint-rollup build | 274 ms | +| Parquet export | 345 ms | +| chDB forced optimization | 4.13 s | + +The production-repository row includes the real atomic WAL + hot-segment + +Parquet commit path and was rerun on 2026-08-26. The isolated rows measure each +engine separately. These are development measurements from an Apple M3 Max, +not published capacity claims. The detailed methodology and reproduction commands are in +[storage-poc.md](storage-poc.md). + +## Options at a glance + +| Option | Writes | Product reads | Ad-hoc SQL | Open data | Complexity | Verdict | +|---|---|---|---|---|---|---| +| Fanout hot + Parquet cold + DuckDB | **Best** | **Best** | Strong | Yes for cold data | Medium | **Recommended** | +| DuckDB native | Medium | Strong | **Best** | Export required | Low | Good simpler alternative | +| DuckLake + DuckDB + Parquet | Medium | Strong | Strong | Yes | Medium-high | Remove from new design | +| Iceberg v3 + Parquet + DuckDB | Medium-low | Strong | Strong | **Best** | High | Add only for shared object storage | +| chDB + MergeTree | Strong | Good | Strong | Export required | Medium | Not selected | +| Fully custom database and SQL engine | Potentially best | Potentially best | Weak initially | No | **Extreme** | Do not build | + +## Option A: Fanout hot store + Parquet + DuckDB + +### Components + +- Fanout-owned immutable columnar hot segments. +- Atomic Fanout manifest and crash recovery. +- Trace, tenant, service, and other promoted indexes. +- Service and endpoint rollups created during ingestion. +- Streaming compaction that copies compressed blocks without decoding rows. +- Parquet files committed alongside each hot segment. +- DuckDB for ad-hoc SQL over cold files. +- SQLite for control data. + +### Benefits + +- Highest measured ingestion throughput. +- Lowest measured indexed-query latency. +- Lowest measured peak memory. +- No C++ call in the ingestion hot path. +- Fanout can optimize precisely for append-only telemetry and TTL retention. +- Parquet preserves interoperability for the complete retained dataset. +- DuckDB retains featureful SQL without controlling ingestion. + +### Costs and risks + +- Fanout owns file-format compatibility, checksums, recovery, retention, and + compaction correctness. +- Hot and cold data use different physical formats. +- Product queries spanning hot and cold data must merge two result streams. +- The current POC's broad scan is much slower than DuckDB. +- Further promoted-attribute indexes and long-run compaction tuning remain + workload-driven optimizations. + +### Decision + +**Recommended.** It wins the overall Fanout objective while delegating general +SQL and interoperable cold storage to established components. + +## Option B: DuckDB native tables + +### Components + +- DuckDB native database for raw telemetry and rollups. +- DuckDB for all reads and SQL. +- SQLite for control data. +- Optional Parquet export. + +### Benefits + +- Simplest analytical architecture. +- Excellent broad scans and small dashboard queries. +- Mature SQL, joins, window functions, extensions, and vectorized execution. +- No separate lakehouse catalog is required. + +### Costs and risks + +- Ingestion was approximately five times slower than the custom hot store in + the full-shape POC. +- Peak RSS was much higher in the isolated comparison. +- Scheduled rollup work remains outside ingestion. +- Native files are not an interoperable telemetry format. +- Export is required for other engines to consume the data. + +### Decision + +**Best fallback if owning a hot format becomes too expensive.** It is preferable +to a more complicated DuckLake or Iceberg deployment when everything remains +inside one Fanout process. + +## Option C: DuckLake + DuckDB + Parquet + +### Components + +- Parquet data files. +- DuckLake metadata and commits, currently backed by a SQLite catalog. +- DuckDB reads and writes. +- A separate SQLite database for Fanout control data. + +### Benefits + +- Transactional table semantics over Parquet. +- DuckDB-native integration. +- Schema evolution, snapshots, and managed file lifecycle. +- Parquet remains externally readable. + +### Costs and overlap + +- DuckLake and the proposed Fanout manifest both manage file commits, + compaction, retention, and visibility. +- Catalog writes require serialization in the current single-process design. +- More maintenance paths exist than with DuckDB native tables. +- It does not improve the measured hot-path advantage of custom segments. +- The SQLite DuckLake catalog is separate from Fanout's control SQLite and + should never be conflated with it. + +### Decision + +**Remove from the new architecture.** DuckLake makes sense when DuckDB owns the +authoritative Parquet table. In the recommended design, Fanout owns the hot +table lifecycle and DuckDB is a secondary SQL executor. + +## Option D: Iceberg v3 + Parquet + DuckDB + +### Components + +- Parquet data files. +- Iceberg v3 table metadata, manifests, snapshots, schema and partition + evolution, and row-level change mechanisms. +- An Iceberg catalog. +- DuckDB and potentially other engines as readers. +- `iceberg-go` for Fanout writes and metadata commits. + +### Benefits + +- Strongest open, multi-engine table contract. +- Appropriate for S3/R2 and large long-lived datasets. +- Supports snapshot history, time travel, schema evolution, and multiple + independent consumers. +- Avoids binding the cold table to DuckDB. + +### Costs and overlap + +- Iceberg does not replace Parquet or the query engine. +- Snapshot and manifest planning add work above direct Parquet reads. +- It introduces a catalog and a more complex commit protocol. +- Small-file management becomes a first-class operational responsibility. +- It solves multi-engine and object-storage coordination that a single-process + Fanout appliance does not initially have. + +### Add Iceberg when + +- S3 or R2 becomes primary durable storage. +- Multiple Fanout writers commit to the same table. +- Spark, Trino, Flink, or another external engine must share authoritative + tables. +- Snapshot history and time travel become product requirements. +- Cold data outlives individual Fanout installations. + +### Decision + +**Do not include initially.** Keep the Parquet layout compatible with a later +Iceberg adoption, but do not pay its catalog and metadata cost before those +requirements exist. + +## Option E: chDB + ClickHouse MergeTree + +### Components + +- Embedded ClickHouse through `chdb-go` bindings. +- MergeTree raw tables. +- Materialized views and AggregatingMergeTree rollups. +- Projections and data-skipping indexes. +- SQLite for control data. + +### Benefits + +- One analytical engine handles ingestion, raw storage, rollups, TTL, indexes, + and SQL. +- Strong ClickHouse feature set. +- Better measured ingestion and memory than DuckDB in some tests. +- No Fanout-owned analytical file format is required. + +### Costs and risks + +- It was substantially slower than the custom path for ingestion, endpoint + reads, full trace reads, mixed load, and maintenance. +- The Go package is a binding to a large C++ engine, not a native-Go database. +- The embedded library is extracted and dynamically loaded at runtime. +- Cold initialization and extracted-library size are meaningful appliance + concerns. +- The current Go result and bulk-ingestion APIs are less mature than DuckDB's + appender path. +- MergeTree files are engine-specific; Parquet export is required for open + storage. + +### Decision + +**Not selected.** It is a credible one-engine architecture, but the normalized +POC no longer justifies its footprint and binding complexity for Fanout. + +## Option F: fully custom database + +This would include a custom storage format, WAL, catalog, indexes, compaction, +query planner, vectorized execution engine, SQL parser, joins, memory manager, +and transaction system. + +### Potential benefit + +- Complete control and the theoretical maximum performance for Fanout-specific + operations. + +### Why not + +- The POC already demonstrates that custom **storage and fixed execution** + provide most of the useful advantage. +- Building general SQL would duplicate years of DuckDB work. +- Correct recovery, concurrency, query planning, joins, spilling, and schema + evolution would dominate product development. + +### Decision + +**Do not build a general database.** Build a Fanout storage engine and use +DuckDB where general SQL is valuable. + +## Why Parquet remains + +Parquet is the one lakehouse component retained in the initial architecture. + +It provides: + +- the smallest measured representation; +- an open, documented columnar format; +- direct DuckDB reads; +- compatibility with future Iceberg adoption; +- straightforward export and backup; +- independence from Fanout's hot-format evolution. + +Parquet should not be used for every small ingest flush. Fanout should first +write hot segments, then create reasonably sized Parquet files during aging or +cold compaction. + +## Proposed data lifecycle + +```text +1. Receive OTLP batch +2. Normalize and promote indexed attributes once +3. Append a crash-safe Fanout hot segment +4. Publish the segment through an atomic manifest +5. Answer dashboards and trace lookup directly +6. Stream-compact small hot segments +7. Age completed time partitions into Parquet +8. Atomically publish cold files and retire superseded hot segments +9. Query cold/ad-hoc data with DuckDB +10. Delete expired whole files through manifest commits +``` + +## Query routing + +| Query | Hot data | Cold data | +|---|---|---| +| Trace by ID | Fanout trace index | Parquet sidecar index, then DuckDB or direct reader | +| Service/endpoint dashboard | Fanout ingestion-time rollups | Parquet rollups through DuckDB | +| Promoted attribute filter | Fanout attribute index | DuckDB predicate pushdown | +| Log text search | Fanout text/token index | DuckDB scan initially; specialized cold index if required | +| Arbitrary SQL | Optional limited direct projection | DuckDB over Parquet | +| Export | Parquet writer | Existing Parquet files | + +## Single-binary implications + +| Choice | Distribution consequence | +|---|---| +| Fanout hot store | Native Go code inside the existing binary | +| DuckDB | Embedded native dependency already used by Fanout | +| Parquet | Library code; no separate server | +| SQLite | Embedded control database already used by Fanout | +| chDB | Adds and extracts a large ClickHouse native library | +| Iceberg | Adds Go metadata/catalog logic but still needs storage and execution | + +No external database daemon is required by the recommended design. + +## Production gates + +Do not replace the existing telemetry path until all gates pass: + +- [ ] Add complete log and metric columnar formats. +- [ ] Add promoted tenant and high-value attribute indexes. +- [ ] Add per-block and per-file checksums. +- [ ] Test torn writes and corruption at every commit boundary. +- [ ] Run continuous kill/restart recovery tests. +- [ ] Prove retention and compaction are safe under active readers. +- [ ] Bound memory during multi-day compaction. +- [ ] Add hot/cold query result merging. +- [ ] Benchmark on the target Linux 4-vCPU/8-GB host. +- [ ] Run a long concurrent ingest/query/retention soak. +- [ ] Validate upgrade and format-version handling. +- [ ] Benchmark realistic high-cardinality attributes and large exception data. + +## Final decision table + +| Component | Initial decision | Revisit when | +|---|---|---| +| Fanout hot columnar format | **Use** | If ownership cost exceeds its measured advantage | +| Fanout direct query paths | **Use** | Always retain benchmarks against DuckDB | +| Parquet cold format | **Use** | No expected replacement | +| DuckDB query engine | **Use** | If another embedded engine wins normalized SQL tests materially | +| SQLite control database | **Use** | No overlap with telemetry storage | +| DuckDB native telemetry tables | Do not use as primary | Fallback if the custom store fails production gates | +| DuckLake | **Remove** | If DuckDB again becomes authoritative over mutable Parquet tables | +| Iceberg v3 | Not initially | Shared object storage, multiple writers, or multi-engine tables | +| chDB | **Remove** | Only if its binding, footprint, and normalized results improve materially | +| Custom general SQL engine | **Do not build** | No planned revisit | + +## Bottom line + +Fanout does not need every lakehouse layer. + +The smallest architecture that satisfies the product is: + +```text +Fanout hot columnar store + Parquet cold files + DuckDB SQL + SQLite control +``` + +DuckLake and Iceberg overlap with lifecycle management that Fanout already must +own for the hot store. Iceberg remains a clean future option for shared object +storage; it is not a prerequisite for a fast, featureful single-binary Fanout. diff --git a/docs/storage-poc.md b/docs/storage-poc.md new file mode 100644 index 00000000..8f089e70 --- /dev/null +++ b/docs/storage-poc.md @@ -0,0 +1,99 @@ +# Fanout-native storage POC + +This experiment asks whether a storage path designed only for Fanout's +telemetry workload can outperform embedded general-purpose databases while +remaining crash-safe and retaining a path to ad-hoc SQL. + +It is isolated from the product and does not define a migration format. + +## Workload + +The shared generator emits the complete typed span produced by Fanout's OTLP +parser: 32 source fields, including all resource, attribute, event, link, +scope, HTTP, RPC, database, peer, deployment, status, and exception values. +DuckDB and chDB additionally persist three derived timestamp representations, +matching Fanout's current 35-column analytical shape. Each run uses one million +spans, 50,000-row durable commits, 50 services, 20 routes, 200 tenants, five +spans per trace, and 24 hours of event time. + +Queries return complete 35-column traces, endpoint rollups, and a raw service +aggregation. The mixed test writes another 200,000 committed rows while full +trace reads run at 100 queries per second. + +## Fanout segment design + +- immutable segments containing 2,048-row columnar blocks; +- every column compressed independently with Zstandard; +- block min/max event-time metadata; +- compact per-block trace indexes with full-ID verification; +- five-minute endpoint histograms built during ingestion; +- atomic manifest replacement with file and directory `fsync` ordering; +- orphan detection after a crash between segment and manifest publication; +- streaming compaction that copies compressed blocks without materializing + rows, then atomically replaces the input segments; +- direct execution for indexed Fanout operations. + +## Normalized result + +Collected on Darwin/arm64, Apple M3 Max, 14 logical CPUs. This is a development +comparison, not a published Fanout capacity claim. Peak RSS was measured in an +isolated process for each embedded engine. + +| Storage / execution | Write rows/s | Maintenance | Active disk | Endpoint | Full trace | Raw service | Mixed write | Mixed trace p95 | Peak RSS | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Fanout columnar + direct | **520,879** | **135 ms** | 34.4 MiB | **0.224 ms** | **0.507 ms** | 26.98 ms | **528,668/s** | **1.33 ms** | **197 MiB** | +| DuckDB native | 98,030 | 274 ms rollup + 4 ms checkpoint | 47.5 MiB | 0.905 ms | 1.54 ms | **1.44 ms** | 85,514/s | 2.14 ms | 1,693 MiB | +| Zstd Parquet + DuckDB | 94,824 effective | 345 ms export | **21.8 MiB** | 1.35 ms | 10.53 ms | 3.88 ms | n/a | n/a | included in DuckDB process | +| chDB MergeTree | 129,724 | 4.13 s optimize | 38.1 MiB active | 2.81 ms | 7.39 ms | 6.06 ms | 118,722/s | 9.61 ms | 699 MiB | + +The chDB directory occupied 106.7 MiB after forced merges because inactive and +engine-internal files remain present; the table's active parts occupied 38.1 +MiB. Its embedded-engine initialization took 412 ms in the measured run. + +Iceberg is not listed as an execution engine. Its data plane is Parquet; table +metadata, snapshots, manifests, deletion vectors, and planning would sit above +the Parquet/DuckDB result and add capabilities plus some overhead. + +## Interpretation + +The custom path wins Fanout's fixed ingestion, endpoint, trace, concurrency, +maintenance, and memory objectives. DuckDB remains about 19 times faster for +the broad raw aggregation, and Parquet remains about 37% smaller than the +custom durable format. + +This supports a hybrid architecture rather than a home-grown general SQL +database: + +- Fanout owns the hot WAL/manifest, columnar segments, indexes, retention, + compaction, and ingestion-time rollups; +- known product queries use direct vectorized execution; +- cold segments use Parquet when interoperability and density matter; +- an established vectorized SQL engine handles arbitrary scans over cold data; +- SQLite remains control/configuration storage only. + +Before production replacement, the POC still needs logs and metrics, promoted +attribute indexes, retention under active readers, corruption checksums, +bounded-memory multi-day compaction, Linux 4-vCPU/8-GB measurements, and a +long-running kill/restart soak. + +## Reproduction + +Custom, DuckDB, and Parquet: + +```sh +go run ./cmd/storage-poc \ + -rows 1000000 \ + -batch 50000 \ + -repeats 11 \ + -mixed-rows 200000 +``` + +Run one embedded engine in isolation with `-engine custom` or `-engine duck`. + +chDB is a nested experiment module so its embedded C++ library does not enter +Fanout's production dependency graph or binary: + +```sh +cd experiments/storage-poc-chdb +go run . -rows 1000000 -batch 50000 -repeats 11 -mixed-rows 200000 +``` diff --git a/experiments/storage-poc-chdb/go.mod b/experiments/storage-poc-chdb/go.mod new file mode 100644 index 00000000..640adf23 --- /dev/null +++ b/experiments/storage-poc-chdb/go.mod @@ -0,0 +1,23 @@ +module github.com/labstack/fanout/experiments/storage-poc-chdb + +go 1.27.0 + +require ( + github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 + github.com/chdb-io/chdb-go/v2 v2.1.0 + github.com/labstack/fanout v0.0.0 +) + +require ( + github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 // indirect + github.com/ebitengine/purego v0.8.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/sys v0.47.0 // indirect +) + +replace github.com/labstack/fanout => ../.. diff --git a/experiments/storage-poc-chdb/go.sum b/experiments/storage-poc-chdb/go.sum new file mode 100644 index 00000000..fddb0c32 --- /dev/null +++ b/experiments/storage-poc-chdb/go.sum @@ -0,0 +1,24 @@ +github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 h1:QiXQ1vZWcQCbRpciirWG/+F3KRXYnDLiFOVYkAJxCls= +github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1/go.mod h1:jB7U0oct7fDV+SbrDzh3oorQFAQ8YOM5QFXf273ZEEs= +github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 h1:qvkIS/fozvgJfd30MEPM1nVDSM1JMXQgqZfqp1Oh7aQ= +github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1/go.mod h1:tebe6DiYx113PoHD0WjWCWVY74QDmaPcCdWA+aNpWPM= +github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 h1:FBSXH0ChVm7fOEHUWpf3bNd2BKnp1cv9LhEn1eLcUdE= +github.com/chdb-io/chdb-go/lib/embedded v0.260700.1/go.mod h1:N9Dra/RfDuELfnT2TSMVBF+NzyDKL6AFiDqHwCWF7T0= +github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 h1:w1q1whc7LlBpJtzoMkokgnSsdRrFLAG2kgRkb1tQ7jM= +github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1/go.mod h1:LK3ORN5rYtQDUYKswMZKf09RT1qIdwlCGk/3/vB7aHE= +github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 h1:GoviqHKnVOJIfnfgiSYWiEqxciThy2XGmSHUdq1qvmI= +github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1/go.mod h1:vkAVOjzg+j6TwFWobfxvmU+pV8fD7dS9ibBdZc3Wf8Y= +github.com/chdb-io/chdb-go/v2 v2.1.0 h1:Nf/StmYfE90mePp0EzdWqCbqUv/TJUbVOrnea/x3PN0= +github.com/chdb-io/chdb-go/v2 v2.1.0/go.mod h1:tyiHoF8pWUfrD7ylseofFEnELnA+jocf/yElE3AHBaQ= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/experiments/storage-poc-chdb/main.go b/experiments/storage-poc-chdb/main.go new file mode 100644 index 00000000..8648c1ea --- /dev/null +++ b/experiments/storage-poc-chdb/main.go @@ -0,0 +1,250 @@ +package main + +import ( + "bytes" + "encoding/csv" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "sync" + "time" + + _ "github.com/chdb-io/chdb-go/lib/embedded" + "github.com/chdb-io/chdb-go/v2/chdb" + "github.com/labstack/fanout/internal/storagebench" + "github.com/labstack/fanout/internal/telemetry/segment" +) + +func main() { + rows := flag.Int("rows", 1_000_000, "number of synthetic spans") + batch := flag.Int("batch", 50_000, "rows per insert") + repeats := flag.Int("repeats", 11, "query repetitions") + mixedRows := flag.Int("mixed-rows", 200_000, "additional rows written while trace reads run at 100 qps") + dir := flag.String("dir", "", "session directory; temporary when empty") + flag.Parse() + root := *dir + if root == "" { + var err error + root, err = os.MkdirTemp("", "fanout-chdb-poc-") + if err != nil { + fatal(err) + } + defer os.RemoveAll(root) + } else if err := os.MkdirAll(root, 0o755); err != nil { + fatal(err) + } + base := time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) + targetTrace := storagebench.TraceID(uint64(*rows / 2 / 5)) + initStart := time.Now() + session, err := chdb.NewSession(filepath.Join(root, "engine")) + if err != nil { + fatal(err) + } + initElapsed := time.Since(initStart) + defer session.Close() + for _, statement := range []string{ + "CREATE DATABASE fanout", spansDDL, endpointDDL, endpointMVDDL, + fmt.Sprintf("SET max_threads=%d", runtime.NumCPU()), "SET date_time_input_format='best_effort'", + } { + query(session, statement, "Null") + } + + writeStart := time.Now() + insertRange(session, 0, *rows, *rows, *batch, base.UnixNano()) + writeElapsed := time.Since(writeStart) + maintenanceStart := time.Now() + query(session, "OPTIMIZE TABLE fanout.spans FINAL", "Null") + query(session, "OPTIMIZE TABLE fanout.endpoint_rollup FINAL", "Null") + maintenanceElapsed := time.Since(maintenanceStart) + activeBytes, err := strconv.ParseInt(queryText(session, "SELECT sum(bytes_on_disk) FROM system.parts WHERE active AND database='fanout'"), 10, 64) + if err != nil { + fatal(fmt.Errorf("parse active bytes: %w", err)) + } + + startLiteral, endLiteral := "2026-08-25 00:00:00", "2026-08-26 00:00:00" + endpointSQL := fmt.Sprintf(`SELECT service_name,http_method,http_route,sum(calls),sum(errors),sum(duration_sum)/sum(calls),max(p95_ms) + FROM fanout.endpoint_rollup WHERE namespace='default' AND service_name='service-00' + AND bucket>=toDateTime64('%s',9,'UTC') AND bucket=toDateTime64('%s',9,'UTC') AND start_time= time.Second { + return fmt.Sprintf("%.2fs", value.Seconds()) + } + if value >= time.Millisecond { + return fmt.Sprintf("%.2fms", float64(value)/float64(time.Millisecond)) + } + return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) +} + +func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-poc-chdb:", err); os.Exit(1) } + +const spansDDL = `CREATE TABLE fanout.spans ( + namespace String,trace_id String,span_id String,parent_span_id String,service_name LowCardinality(String),name LowCardinality(String),kind LowCardinality(String), + start_time DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4),end_time DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4), + start_unix_nano Int64 CODEC(DoubleDelta,LZ4),end_unix_nano Int64 CODEC(DoubleDelta,LZ4),duration_ms Float64 CODEC(Gorilla,LZ4), + status_code LowCardinality(String),status_msg String CODEC(ZSTD(1)),resource_json String CODEC(ZSTD(1)),attributes_json String CODEC(ZSTD(1)), + events_json String CODEC(ZSTD(1)),links_json String CODEC(ZSTD(1)),trace_state String,flags UInt32,scope_name LowCardinality(String),scope_version String, + ingested_at DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4),ingested_unix_nano Int64 CODEC(DoubleDelta,LZ4),http_method LowCardinality(String), + http_status_code LowCardinality(String),http_route LowCardinality(String),db_system LowCardinality(String),rpc_method LowCardinality(String), + rpc_service LowCardinality(String),peer_service LowCardinality(String),service_version LowCardinality(String),deployment_env LowCardinality(String), + exception_type LowCardinality(String),exception_message String CODEC(ZSTD(1)), + tenant_id LowCardinality(String) MATERIALIZED JSONExtractString(attributes_json,'tenant'), + PROJECTION by_trace INDEX trace_id TYPE basic,PROJECTION by_tenant INDEX tenant_id TYPE basic +) ENGINE=MergeTree PARTITION BY toDate(start_time) ORDER BY (namespace,start_time,service_name) SETTINGS old_parts_lifetime=0` + +const endpointDDL = `CREATE TABLE fanout.endpoint_rollup ( + namespace String,bucket DateTime64(9,'UTC'),service_name LowCardinality(String),http_method LowCardinality(String),http_route LowCardinality(String), + calls SimpleAggregateFunction(sum,UInt64),errors SimpleAggregateFunction(sum,UInt64),duration_sum SimpleAggregateFunction(sum,Float64), + p95_ms SimpleAggregateFunction(max,Float64) +) ENGINE=AggregatingMergeTree PARTITION BY toDate(bucket) ORDER BY (namespace,bucket,service_name,http_method,http_route) SETTINGS old_parts_lifetime=0` + +const endpointMVDDL = `CREATE MATERIALIZED VIEW fanout.endpoint_rollup_mv TO fanout.endpoint_rollup AS +SELECT namespace,toStartOfInterval(start_time,INTERVAL 5 MINUTE) AS bucket,service_name,http_method,http_route, + count() AS calls,countIf(status_code='ERROR') AS errors,sum(duration_ms) AS duration_sum,quantileTDigest(0.95)(duration_ms) AS p95_ms +FROM fanout.spans GROUP BY namespace,bucket,service_name,http_method,http_route` diff --git a/fanout.example.yaml b/fanout.example.yaml index 029df643..27441770 100644 --- a/fanout.example.yaml +++ b/fanout.example.yaml @@ -27,10 +27,10 @@ ingest: storage: data_dir: ./data # FANOUT_DATA_DIR retention_days: 30 # FANOUT_RETENTION_DAYS + hot_retention: 24h # FANOUT_HOT_RETENTION rollup_interval: 1m # FANOUT_ROLLUP_INTERVAL rollup_skip_to_latest: false # FANOUT_ROLLUP_SKIP_TO_LATEST maintenance_interval: 1h # FANOUT_MAINTENANCE_INTERVAL - merge_interval: 1m # FANOUT_MERGE_INTERVAL duckdb: # Empty/zero values let Fanout size these from the machine. memory: "" # FANOUT_DUCKDB_MEMORY diff --git a/go.mod b/go.mod index 7a011b13..887dab9b 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,13 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260826145851-49e71f2b2d21 github.com/alexedwards/scs/v2 v2.9.0 + github.com/apache/arrow-go/v18 v18.7.0 github.com/coreos/go-oidc/v3 v3.20.0 github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.19.2 github.com/knadh/koanf/parsers/yaml v1.1.1 github.com/knadh/koanf/providers/confmap v1.0.1 github.com/knadh/koanf/providers/file v1.2.1 @@ -22,6 +24,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/wneessen/go-mail v0.8.1 + github.com/zeebo/xxh3 v1.1.0 go.opentelemetry.io/proto/otlp v1.11.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sys v0.47.0 @@ -34,8 +37,9 @@ require ( require ( cel.dev/expr v0.25.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect - github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/apache/thrift v0.24.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -57,7 +61,6 @@ require ( github.com/google/jsonschema-go v0.4.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/hcl/v2 v2.24.0 // indirect - github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/knadh/koanf/maps v0.1.3 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -77,7 +80,6 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zclconf/go-cty v1.19.0 // indirect github.com/zclconf/go-cty-yaml v1.2.0 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect diff --git a/go.sum b/go.sum index 409875ed..7e10e4e5 100644 --- a/go.sum +++ b/go.sum @@ -141,10 +141,14 @@ github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfv github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/wneessen/go-mail v0.8.1 h1:tVcncj02/QySVFw3zr/kXOzZcuFQqBNT6K+Rbgm/pcM= github.com/wneessen/go-mail v0.8.1/go.mod h1:dWZ61zadzCIyvB4y1/YzC5O7MrbbzBfPkARmbosdf8w= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4= diff --git a/internal/api/health.go b/internal/api/health.go index 86e0d478..ca218c07 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -59,7 +59,7 @@ func (h *HealthHandler) Readiness(c *echo.Context) error { // Check DuckDB resp.Checks["duckdb"] = h.checkDuckDB() - resp.Checks["ducklake"] = h.checkDuckLake() + resp.Checks["telemetry"] = h.checkTelemetry() // Check data directory resp.Checks["data"] = h.checkDataDir() @@ -177,7 +177,7 @@ func maintenanceStaleThreshold(maintEvery time.Duration) time.Duration { return stale } -func (h *HealthHandler) checkDuckLake() CheckResult { +func (h *HealthHandler) checkTelemetry() CheckResult { if h.duck == nil { return CheckResult{ Status: "unhealthy", @@ -204,13 +204,14 @@ func (h *HealthHandler) checkDuckLake() CheckResult { LatencyMs: time.Since(start).Milliseconds(), } if err == sql.ErrNoRows { - res.Detail = "telemetry attached, no spans yet" + res.Detail = "telemetry repository ready, no spans yet" } return res } -// checkMaintenance surfaces the maintenance loop's own health (retention + -// DuckLake compaction). A failing pass reports "degraded", not "unhealthy": +// checkMaintenance surfaces the maintenance loop's own health (retention, +// compaction, and cache checkpointing). A failing pass reports "degraded", not +// "unhealthy": // ingest and queries still work while maintenance fails, and a restart // wouldn't fix it — pulling the instance from rotation would only hide the // signal that storage growth is no longer being reclaimed. A maintenance pass diff --git a/internal/api/health_test.go b/internal/api/health_test.go index ca3049c3..07f17c49 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -111,7 +111,7 @@ func TestReadinessReportsSizingResolvedByLoader(t *testing.T) { } } -func TestReadiness_HealthyDuckLakeAndRollups(t *testing.T) { +func TestReadiness_HealthyTelemetryAndRollups(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New: %v", err) @@ -151,12 +151,12 @@ func TestReadiness_HealthyDuckLakeAndRollups(t *testing.T) { } // The "data" check reads real host free space, which can legitimately be - // degraded on a low-disk CI/dev box. This test is about ducklake+rollups, so + // degraded on a low-disk CI/dev box. This test is about telemetry+rollups, so // tolerate a data-only degradation but require the actual subjects to be ok. if resp.Status != "ready" && resp.Status != "degraded" { t.Fatalf("status = %q, want ready or degraded", resp.Status) } - for _, key := range []string{"duckdb", "ducklake", "data", "rollups", "maintenance"} { + for _, key := range []string{"duckdb", "telemetry", "data", "rollups", "maintenance"} { if _, ok := resp.Checks[key]; !ok { t.Fatalf("missing %s check", key) } @@ -167,7 +167,7 @@ func TestReadiness_HealthyDuckLakeAndRollups(t *testing.T) { if resp.RuntimeSizing.GOMAXPROCS <= 0 { t.Fatalf("runtime sizing GOMAXPROCS = %d, want positive", resp.RuntimeSizing.GOMAXPROCS) } - for _, key := range []string{"duckdb", "ducklake", "rollups", "maintenance"} { + for _, key := range []string{"duckdb", "telemetry", "rollups", "maintenance"} { if got := resp.Checks[key].Status; got != "ok" { t.Fatalf("%s check = %q, want ok", key, got) } diff --git a/internal/config/config.go b/internal/config/config.go index 9eeb8f9a..0915d35f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,18 +32,12 @@ type Config struct { RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` - // MaintenanceInterval throttles the DuckLake maintenance cycle (retention - // deletes + compaction). Default 1h. Lower it to compact more - // aggressively, or for soak tests that need to observe file-count staying - // bounded within minutes rather than hours. + // HotRetention controls how long the custom indexed segments are retained. + // Older telemetry remains queryable in Parquet through DuckDB. + HotRetention time.Duration `koanf:"storage.hot_retention" env:"FANOUT_HOT_RETENTION" default:"24h"` + // MaintenanceInterval controls hot-segment pruning, Parquet retention and + // compaction, and query-cache checkpointing. MaintenanceInterval time.Duration `koanf:"storage.maintenance_interval" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` - // MergeInterval is the cadence for the cheap, frequent DuckLake file - // compaction pass (ducklake_merge_adjacent_files only — it consolidates the - // newest small parquet files and deletes nothing). Run often (default 1m) it - // keeps the queryable file count continuously low, which is what bounds - // rollup/query scan latency — WITHOUT the churn, deletion race, or catalog - // cost of the full hourly maintenance pass (expire + cleanup). 0 disables it. - MergeInterval time.Duration `koanf:"storage.merge_interval" env:"FANOUT_MERGE_INTERVAL" default:"1m"` // RollupSkipToLatest, set once at boot, advances every rollup watermark to the // current max ingested timestamp so existing data is treated as already-rolled-up // instead of aggregated as a backlog. Stands up a large pre-seeded historical @@ -78,15 +72,8 @@ type Config struct { // DuckDB's own default in place (one worker per core). Set it to leave // cores free for ingest on a query-heavy co-tenant host. DuckDBThreads int `koanf:"storage.duckdb.threads" env:"FANOUT_DUCKDB_THREADS"` - // DuckDBMaxConns caps the DuckDB connection pool. A value of 1 serializes - // everything through one handle; the machine-sized default lets read queries - // run concurrently with each other and with ingest flushes. Two things make >1 - // safe: the DuckLake SQLite catalog is opened in WAL mode (enableCatalogWAL), - // so readers don't collide with the single writer and a crashed writer can't - // leave the catalog permanently locked; and write commits are serialized by - // the shared write gate (Duck.WriteGate, wired into the writer via - // UseWriteGate in cmd/fanout/main.go, enforced at startup). Without the WAL - // mode, pool >1 fails with "database is locked". + // DuckDBMaxConns caps the DuckDB connection pool. Reads scan immutable Parquet + // concurrently; rollup-cache writes are serialized by the query write gate. // Zero means "size it from the machine" — the same spelling DuckDBThreads // uses for deferring to a default. Resolution happens in resolveSizing and // is reported in the startup configuration log. @@ -154,10 +141,6 @@ func (c Config) TelemetryParquetDir() string { return filepath.Join(c.TelemetryDir(), "parquet") } -func (c Config) TelemetryDuckLakePath() string { - return filepath.Join(c.TelemetryDir(), "ducklake.sqlite") -} - func (c Config) QueryDir() string { return filepath.Join(c.DataDir, "query") } @@ -211,12 +194,12 @@ func (c Config) Validate() error { if c.RetentionDays < 0 { return fmt.Errorf("storage.retention_days must be >= 0, got %d", c.RetentionDays) } + if c.HotRetention < 24*time.Hour { + return fmt.Errorf("storage.hot_retention must be at least 24h, got %s", c.HotRetention) + } if c.MaintenanceInterval < time.Second { return fmt.Errorf("storage.maintenance_interval must be at least 1s, got %s", c.MaintenanceInterval) } - if c.MergeInterval < 0 || (c.MergeInterval > 0 && c.MergeInterval < time.Second) { - return fmt.Errorf("storage.merge_interval must be 0s or at least 1s, got %s", c.MergeInterval) - } if c.DuckDBThreads < 0 { return fmt.Errorf("storage.duckdb.threads must be >= 0, got %d", c.DuckDBThreads) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 77e63974..a75decae 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -77,8 +77,6 @@ func TestLoadLayering(t *testing.T) { path := filepath.Join(t.TempDir(), "fanout.yaml") if err := os.WriteFile(path, []byte(`server: http_addr: ":1111" -storage: - merge_interval: 75s mcp: enabled: false metrics: @@ -100,7 +98,7 @@ auth: if cfg.HTTPAddr != ":2222" { t.Fatalf("HTTPAddr = %q, want environment override", cfg.HTTPAddr) } - if cfg.MergeInterval != 75*time.Second || cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { + if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { t.Fatalf("YAML values were not merged: %+v", cfg) } } @@ -109,14 +107,13 @@ func TestLoadTypedEnvironmentValues(t *testing.T) { cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_MCP_ENABLED=false", "FANOUT_METRICS_PUBLIC=true", - "FANOUT_MERGE_INTERVAL=75s", "FANOUT_SESSION_IDLE_TTL=10h", "FANOUT_SELF_SIGNUP=true", )}) if err != nil { t.Fatalf("Load: %v", err) } - if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.MergeInterval != 75*time.Second || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { + if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { t.Fatalf("environment values were not decoded: %+v", cfg) } } @@ -274,7 +271,7 @@ func TestConfigurationSchemaUsesUnitBearingDurations(t *testing.T) { strings.Contains(env, "_seconds") || strings.Contains(env, "_ms") { t.Errorf("elapsed-time setting uses a unit suffix: %s / %s", spec.key, spec.env) } - if spec.typ == durationType && !strings.HasSuffix(key, "_interval") && !strings.HasSuffix(key, "_ttl") { + if spec.typ == durationType && !strings.HasSuffix(key, "_interval") && !strings.HasSuffix(key, "_ttl") && !strings.HasSuffix(key, "_retention") { t.Errorf("duration setting %s must end in _interval or _ttl", spec.key) } } @@ -287,7 +284,6 @@ func TestLoadDurationIntervals(t *testing.T) { flush_interval: 30s storage: rollup_interval: 5m - merge_interval: 0s maintenance_interval: 1h alerts: evaluation_interval: 45s @@ -300,7 +296,7 @@ alerts: t.Fatalf("Load: %v", err) } if cfg.FlushInterval != 30*time.Second || cfg.RollupInterval != 5*time.Minute || - cfg.MergeInterval != 0 || cfg.MaintenanceInterval != time.Hour || + cfg.MaintenanceInterval != time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { t.Fatalf("duration values were not decoded: %+v", cfg) } @@ -310,7 +306,6 @@ alerts: cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_FLUSH_INTERVAL=30s", "FANOUT_ROLLUP_INTERVAL=5m", - "FANOUT_MERGE_INTERVAL=0s", "FANOUT_MAINTENANCE_INTERVAL=1h", "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", )}) @@ -318,7 +313,7 @@ alerts: t.Fatalf("Load: %v", err) } if cfg.FlushInterval != 30*time.Second || cfg.RollupInterval != 5*time.Minute || - cfg.MergeInterval != 0 || cfg.MaintenanceInterval != time.Hour || + cfg.MaintenanceInterval != time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { t.Fatalf("duration values were not decoded: %+v", cfg) } @@ -406,6 +401,7 @@ func TestLoadRejectsUnknownInputs(t *testing.T) { "FANOUT_FLUSH_SECONDS", "FANOUT_ROLLUP_EVERY_SECONDS", "FANOUT_MERGE_EVERY_SECONDS", + "FANOUT_MERGE_INTERVAL", "FANOUT_MAINTENANCE_EVERY_SECONDS", "FANOUT_ALERTS_EVALUATION_INTERVAL_SECONDS", "FANOUT_MCP_PUBLIC_URL", @@ -543,7 +539,6 @@ func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { }{ {"negative max connections is not auto", "FANOUT_DUCKDB_MAX_CONNECTIONS=-3", "max_connections"}, {"zero alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=0s", "evaluation_interval"}, - {"negative merge interval", "FANOUT_MERGE_INTERVAL=-5s", "merge_interval"}, {"negative maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=-5s", "maintenance_interval"}, {"negative DuckDB threads", "FANOUT_DUCKDB_THREADS=-4", "duckdb.threads"}, } { @@ -560,7 +555,6 @@ func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { }{ {"subsecond flush interval", "FANOUT_FLUSH_INTERVAL=999ms", "flush_interval"}, {"subsecond rollup interval", "FANOUT_ROLLUP_INTERVAL=999ms", "rollup_interval"}, - {"subsecond merge interval", "FANOUT_MERGE_INTERVAL=1ns", "merge_interval"}, {"subsecond maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=500ms", "maintenance_interval"}, {"subsecond alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=999ms", "evaluation_interval"}, } { @@ -626,8 +620,8 @@ func TestValidate(t *testing.T) { FlushBatchSize: 50000, RollupInterval: time.Minute, RetentionDays: 30, + HotRetention: 24 * time.Hour, MaintenanceInterval: time.Hour, - MergeInterval: time.Minute, DuckDBMaxConns: 4, AlertEvaluationInterval: 30 * time.Second, AlertHistoryDays: 7, @@ -664,8 +658,6 @@ func TestValidate(t *testing.T) { {"DataDir empty", func(c *Config) { c.DataDir = "" }}, {"MaintenanceInterval=0", func(c *Config) { c.MaintenanceInterval = 0 }}, {"MaintenanceInterval=999ms", func(c *Config) { c.MaintenanceInterval = 999 * time.Millisecond }}, - {"MergeInterval=-1s", func(c *Config) { c.MergeInterval = -time.Second }}, - {"MergeInterval=1ns", func(c *Config) { c.MergeInterval = time.Nanosecond }}, {"DuckDBThreads=-1", func(c *Config) { c.DuckDBThreads = -1 }}, {"DuckDBMaxConns=0", func(c *Config) { c.DuckDBMaxConns = 0 }}, {"AlertEvaluationInterval=0", func(c *Config) { c.AlertEvaluationInterval = 0 }}, @@ -716,14 +708,6 @@ func TestValidate(t *testing.T) { } }) - t.Run("MergeInterval=0_valid", func(t *testing.T) { - c := valid - c.MergeInterval = 0 - if err := c.Validate(); err != nil { - t.Errorf("MergeInterval=0 should disable the merge pass: %v", err) - } - }) - t.Run("local mode allows absent SMTP and agent", func(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" diff --git a/internal/config/sizing.go b/internal/config/sizing.go index 162a95ea..22c78fc5 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -37,9 +37,7 @@ const ( // exceed available parallelism they add contention rather than concurrency. maxAutoDuckDBConns = 16 - // minDuckDBConns preserves an invariant rather than a preference: - // internal/lake rejects max_connections > 1 without a shared write gate, - // and a pool of 1 serializes reads behind writes. + // minDuckDBConns keeps one long scan from serializing all other reads. minDuckDBConns = 2 ) diff --git a/internal/config/sizing_test.go b/internal/config/sizing_test.go index c39324a2..95a6ff4f 100644 --- a/internal/config/sizing_test.go +++ b/internal/config/sizing_test.go @@ -52,10 +52,8 @@ func TestResolveDuckDBMemoryDeclinesForAbsurdlySmallMachines(t *testing.T) { } } -func TestResolveDuckDBMaxConnsKeepsTheWriteGateInvariant(t *testing.T) { - // internal/lake refuses to start when max_connections > 1 without a write - // gate; at or below 1 it serializes everything through one handle. The - // floor is an invariant, not a preference. +func TestResolveDuckDBMaxConnsKeepsReadConcurrency(t *testing.T) { + // At or below 1 every query is serialized through one handle. for _, cores := range []int{0, 1, 2} { if got := resolveDuckDBMaxConns(cores); got < 2 { t.Fatalf("resolveDuckDBMaxConns(%d) = %d, want at least 2", cores, got) diff --git a/internal/ingest/attrs_test.go b/internal/ingest/attrs_test.go index 381ce2f6..465f9022 100644 --- a/internal/ingest/attrs_test.go +++ b/internal/ingest/attrs_test.go @@ -163,8 +163,8 @@ func TestSpanDurationMs(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - if got := spanDurationMs(c.start, c.end); got != c.want { - t.Errorf("spanDurationMs(%d, %d) = %v, want %v", c.start, c.end, got, c.want) + if got := spanDurationMS(c.start, c.end); got != c.want { + t.Errorf("spanDurationMS(%d, %d) = %v, want %v", c.start, c.end, got, c.want) } }) } diff --git a/internal/ingest/http_test.go b/internal/ingest/http_test.go index e0bb6eba..3f3ebb64 100644 --- a/internal/ingest/http_test.go +++ b/internal/ingest/http_test.go @@ -22,17 +22,17 @@ import ( "google.golang.org/protobuf/proto" "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake" "github.com/labstack/fanout/internal/settings" + "github.com/labstack/fanout/internal/telemetry" ) type httpIngestFixture struct { handler http.Handler token string store *settings.Store - spans chan lake.SpanRow - logs chan lake.LogRow - metrics chan lake.MetricRow + spans chan telemetry.Span + logs chan telemetry.Log + metrics chan telemetry.Metric } func newHTTPIngestFixture(t *testing.T, configured bool) *httpIngestFixture { @@ -50,9 +50,9 @@ func newHTTPIngestFixture(t *testing.T, configured bool) *httpIngestFixture { t.Fatalf("SetIngest: %v", err) } } - spans := make(chan lake.SpanRow, 8) - logs := make(chan lake.LogRow, 8) - metrics := make(chan lake.MetricRow, 8) + spans := make(chan telemetry.Span, 8) + logs := make(chan telemetry.Log, 8) + metrics := make(chan telemetry.Metric, 8) srv := NewServer(config.Config{DefaultNamespace: "default"}, spans, logs, metrics) return &httpIngestFixture{ handler: NewHTTPHandler(srv, store), @@ -267,8 +267,8 @@ func TestReadOTLPHTTPBodyLimitsDecompressedSize(t *testing.T) { func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("traces", func(t *testing.T) { request := testTraceRequest() - grpcRows := make(chan lake.SpanRow, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, grpcRows, make(chan lake.LogRow, 1), make(chan lake.MetricRow, 1)) + grpcRows := make(chan telemetry.Span, 1) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, grpcRows, make(chan telemetry.Log, 1), make(chan telemetry.Metric, 1)) if _, err := grpcSrv.exportTraces(context.Background(), request); err != nil { t.Fatal(err) } @@ -284,8 +284,8 @@ func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("logs", func(t *testing.T) { request := testLogsRequest() - grpcRows := make(chan lake.LogRow, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan lake.SpanRow, 1), grpcRows, make(chan lake.MetricRow, 1)) + grpcRows := make(chan telemetry.Log, 1) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan telemetry.Span, 1), grpcRows, make(chan telemetry.Metric, 1)) if _, err := grpcSrv.exportLogs(context.Background(), request); err != nil { t.Fatal(err) } @@ -301,8 +301,8 @@ func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("metrics", func(t *testing.T) { request := testMetricsRequest() - grpcRows := make(chan lake.MetricRow, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan lake.SpanRow, 1), make(chan lake.LogRow, 1), grpcRows) + grpcRows := make(chan telemetry.Metric, 1) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan telemetry.Span, 1), make(chan telemetry.Log, 1), grpcRows) if _, err := grpcSrv.exportMetrics(context.Background(), request); err != nil { t.Fatal(err) } diff --git a/internal/ingest/server.go b/internal/ingest/server.go index 4379fe01..03933ef6 100644 --- a/internal/ingest/server.go +++ b/internal/ingest/server.go @@ -24,14 +24,14 @@ import ( "google.golang.org/grpc" "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake" + "github.com/labstack/fanout/internal/telemetry" ) type Server struct { cfg config.Config - outSpans chan<- lake.SpanRow - outLogs chan<- lake.LogRow - outMetrics chan<- lake.MetricRow + outSpans chan<- telemetry.Span + outLogs chan<- telemetry.Log + outMetrics chan<- telemetry.Metric } type traceService struct { @@ -49,7 +49,7 @@ type metricsService struct { srv *Server } -func NewServer(cfg config.Config, spans chan<- lake.SpanRow, logs chan<- lake.LogRow, metrics chan<- lake.MetricRow) *Server { +func NewServer(cfg config.Config, spans chan<- telemetry.Span, logs chan<- telemetry.Log, metrics chan<- telemetry.Metric) *Server { return &Server{cfg: cfg, outSpans: spans, outLogs: logs, outMetrics: metrics} } @@ -78,7 +78,7 @@ func (s *Server) exportTraces(ctx context.Context, req *collectortrace.ExportTra for _, ss := range rs.ScopeSpans { scopeName, scopeVer := scopeInfo(ss.Scope) for _, sp := range ss.Spans { - row := lake.SpanRow{ + row := telemetry.Span{ Namespace: namespace, TraceID: fmt.Sprintf("%x", sp.TraceId), SpanID: fmt.Sprintf("%x", sp.SpanId), @@ -88,7 +88,7 @@ func (s *Server) exportTraces(ctx context.Context, req *collectortrace.ExportTra Kind: sp.Kind.String(), StartUnixNanos: int64(sp.StartTimeUnixNano), EndUnixNanos: int64(sp.EndTimeUnixNano), - DurationMs: spanDurationMs(sp.StartTimeUnixNano, sp.EndTimeUnixNano), + DurationMS: spanDurationMS(sp.StartTimeUnixNano, sp.EndTimeUnixNano), StatusCode: sp.Status.Code.String(), StatusMsg: sp.Status.Message, ResourceJSON: resourceJSON, @@ -148,7 +148,7 @@ func (s *Server) exportLogs(ctx context.Context, req *collectorlogs.ExportLogsSe for _, lr := range sl.LogRecords { body := bodyString(lr.Body) tmpl := safeNormalizeTemplate(body) - row := lake.LogRow{ + row := telemetry.Log{ Namespace: namespace, TimeUnixNanos: int64(lr.TimeUnixNano), ObservedTimeNanos: int64(lr.ObservedTimeUnixNano), @@ -199,13 +199,13 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export switch d := m.Data.(type) { case *metricspb.Metric_Gauge: for _, dp := range d.Gauge.DataPoints { - row := lake.MetricRow{ + row := telemetry.Metric{ Namespace: namespace, TimeUnixNanos: int64(dp.TimeUnixNano), Name: m.Name, Description: m.Description, Unit: m.Unit, - MType: "gauge", + Type: "gauge", ServiceName: svc, Value: number(dp.Value), ExemplarsJSON: exemplarsToJSON(dp.Exemplars), @@ -227,13 +227,13 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export kind = "sum_delta" } for _, dp := range d.Sum.DataPoints { - row := lake.MetricRow{ + row := telemetry.Metric{ Namespace: namespace, TimeUnixNanos: int64(dp.TimeUnixNano), Name: m.Name, Description: m.Description, Unit: m.Unit, - MType: kind, + Type: kind, ServiceName: svc, Value: number(dp.Value), ExemplarsJSON: exemplarsToJSON(dp.Exemplars), @@ -255,13 +255,13 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export if dp.Sum != nil { histSum = *dp.Sum } - row := lake.MetricRow{ + row := telemetry.Metric{ Namespace: namespace, TimeUnixNanos: int64(dp.TimeUnixNano), Name: m.Name, Description: m.Description, Unit: m.Unit, - MType: "histogram", + Type: "histogram", ServiceName: svc, HistBoundsJSON: toJSON(dp.ExplicitBounds), HistCountsJSON: toJSON(dp.BucketCounts), @@ -286,13 +286,13 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export if dp.Sum != nil { histSum = *dp.Sum } - row := lake.MetricRow{ + row := telemetry.Metric{ Namespace: namespace, TimeUnixNanos: int64(dp.TimeUnixNano), Name: m.Name, Description: m.Description, Unit: m.Unit, - MType: "exp_histogram", + Type: "exp_histogram", ServiceName: svc, HistBoundsJSON: toJSON(expHistBuckets(dp)), HistCountsJSON: toJSON(expHistCounts(dp)), @@ -313,13 +313,13 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export } case *metricspb.Metric_Summary: for _, dp := range d.Summary.DataPoints { - row := lake.MetricRow{ + row := telemetry.Metric{ Namespace: namespace, TimeUnixNanos: int64(dp.TimeUnixNano), Name: m.Name, Description: m.Description, Unit: m.Unit, - MType: "summary", + Type: "summary", ServiceName: svc, HistBoundsJSON: toJSON(summaryQuantiles(dp)), HistCountsJSON: toJSON(summaryValues(dp)), @@ -346,11 +346,11 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export // ---- helpers ---- -// spanDurationMs computes a span's duration in milliseconds, guarding against +// spanDurationMS computes a span's duration in milliseconds, guarding against // the unsigned underflow that a malformed span (end before start) would produce: // uint64 subtraction wraps to a huge positive value, which would otherwise be // stored as a multi-century duration. Such spans are clamped to 0. -func spanDurationMs(startNano, endNano uint64) float64 { +func spanDurationMS(startNano, endNano uint64) float64 { if endNano < startNano { return 0 } diff --git a/internal/ingest/server_test.go b/internal/ingest/server_test.go index 7d52a632..a8389942 100644 --- a/internal/ingest/server_test.go +++ b/internal/ingest/server_test.go @@ -12,7 +12,7 @@ import ( tracepb "go.opentelemetry.io/proto/otlp/trace/v1" "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake" + "github.com/labstack/fanout/internal/telemetry" ) func TestToJSON(t *testing.T) { @@ -507,9 +507,9 @@ func TestExtractException(t *testing.T) { func TestTraceExportContextCancellation(t *testing.T) { // Use an unbuffered channel so the send blocks - spans := make(chan lake.SpanRow) - logs := make(chan lake.LogRow, 1) - metrics := make(chan lake.MetricRow, 1) + spans := make(chan telemetry.Span) + logs := make(chan telemetry.Log, 1) + metrics := make(chan telemetry.Metric, 1) srv := NewServer(config.Config{}, spans, logs, metrics) ts := &traceService{srv: srv} diff --git a/internal/lake/writer.go b/internal/lake/writer.go deleted file mode 100644 index e8fe3f68..00000000 --- a/internal/lake/writer.go +++ /dev/null @@ -1,615 +0,0 @@ -package lake - -import ( - "context" - "database/sql" - "database/sql/driver" - "errors" - "fmt" - "log/slog" - "time" - - "github.com/duckdb/duckdb-go/v2" - "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake/writegate" - "github.com/labstack/fanout/internal/metrics" -) - -type SpanRow struct { - Namespace string - TraceID string - SpanID string - ParentSpanID string - ServiceName string - Name string - Kind string - StartUnixNanos int64 - EndUnixNanos int64 - DurationMs float64 - StatusCode string - StatusMsg string - ResourceJSON []byte - AttributesJSON []byte - EventsJSON []byte - LinksJSON []byte - TraceState string - Flags uint32 - ScopeName string - ScopeVersion string - IngestedAt int64 - HTTPMethod string - HTTPStatusCode string - HTTPRoute string - DBSystem string - RPCMethod string - RPCService string - PeerService string - ServiceVersion string - DeploymentEnv string - ExceptionType string - ExceptionMessage string -} - -type LogRow struct { - Namespace string - TimeUnixNanos int64 - ObservedTimeNanos int64 - Severity string - SeverityNumber int32 - Body string - ServiceName string - TraceID string - SpanID string - Flags uint32 - ResourceJSON []byte - AttributesJSON []byte - ScopeName string - ScopeVersion string - IngestedAt int64 - BodyTemplate string -} - -type MetricRow struct { - Namespace string - TimeUnixNanos int64 - Name string - Description string - Unit string - MType string - ServiceName string - Value float64 - HistBoundsJSON []byte - HistCountsJSON []byte - HistCount int64 - HistSum float64 - ExemplarsJSON []byte - AttributesJSON []byte - ResourceJSON []byte - ScopeName string - ScopeVersion string - IngestedAt int64 -} - -// flushQueueDepth bounds how many filled batches can be queued for the flush -// worker before the receive loop blocks. Blocking here is the intended -// backpressure: it propagates to the ingest channels rather than letting memory -// grow unbounded when the database can't keep up. -const flushQueueDepth = 4 - -type Writer struct { - cfg config.Config - db *sql.DB - chSpans <-chan SpanRow - chLogs <-chan LogRow - chMetrics <-chan MetricRow - // Buffers are owned exclusively by the Run goroutine — no mutex needed. On - // flush their contents are copied into a detached batch and the buffers are - // truncated-and-retained, so the hot receive-loop append never reallocates. - bufSpans []SpanRow - bufLogs []LogRow - bufMetrics []MetricRow - done chan struct{} - // writeGate, when set, serializes appender flushes against the query layer's - // rollup/maintenance commits so two connections never commit to the DuckLake - // catalog at once on a multi-connection pool. Nil is fine (single-connection - // pools already serialize through one handle). - writeGate *writegate.WriteGate -} - -// UseWriteGate shares the query layer's write-serialization gate with the -// writer. Call before Run when the DuckDB pool may hold more than one connection. -func (w *Writer) UseWriteGate(gate *writegate.WriteGate) { w.writeGate = gate } - -// flushBatch is a detached set of rows handed to the flush worker. The worker -// owns the slices once sent. -type flushBatch struct { - spans []SpanRow - logs []LogRow - metrics []MetricRow -} - -func NewWriter(cfg config.Config, db *sql.DB, spans <-chan SpanRow, logs <-chan LogRow, metricsCh <-chan MetricRow) *Writer { - return &Writer{ - cfg: cfg, - db: db, - chSpans: spans, - chLogs: logs, - chMetrics: metricsCh, - done: make(chan struct{}), - } -} - -// Wait blocks until Run() has returned (final flush complete). -func (w *Writer) Wait() { - <-w.done -} - -func (w *Writer) Run(ctx context.Context) error { - defer close(w.done) - - // A multi-connection pool requires the shared write gate so appender flushes - // don't commit concurrently with rollups. Fail loudly rather than silently - // running unserialized writes (which surface only as catalog-lock errors - // under load). Single-connection pools serialize through the one handle, so a - // nil gate is fine there. - if w.cfg.DuckDBMaxConns > 1 && w.writeGate == nil { - return fmt.Errorf("lake writer: storage.duckdb.max_connections=%d requires a shared write gate; call UseWriteGate before Run", w.cfg.DuckDBMaxConns) - } - - // Flushes run on a dedicated worker so a slow database insert never stalls the - // receive loop below (which would stop draining the ingest channels and apply - // backpressure all the way to the gRPC handlers). Run detaches a filled batch - // and hands it off; the worker serializes the actual writes and retries. - flushCh := make(chan flushBatch, flushQueueDepth) - workerDone := make(chan error, 1) - go w.flushWorker(flushCh, workerDone) - - ticker := time.NewTicker(w.cfg.FlushInterval) - defer ticker.Stop() - - spansCh := w.chSpans - logsCh := w.chLogs - metricsCh := w.chMetrics - - finish := func() error { - w.drainChannels(&spansCh, &logsCh, &metricsCh) - w.flush(flushCh) - close(flushCh) - return <-workerDone - } - - for { - select { - case r, ok := <-spansCh: - if !ok { - spansCh = nil - continue - } - w.bufSpans = append(w.bufSpans, r) - metrics.RecordIngest("spans", 1) - metrics.UpdateQueueDepth("spans", len(spansCh)) - if w.shouldFlush() { - w.flush(flushCh) - } - case r, ok := <-logsCh: - if !ok { - logsCh = nil - continue - } - w.bufLogs = append(w.bufLogs, r) - metrics.RecordIngest("logs", 1) - metrics.UpdateQueueDepth("logs", len(logsCh)) - if w.shouldFlush() { - w.flush(flushCh) - } - case r, ok := <-metricsCh: - if !ok { - metricsCh = nil - continue - } - w.bufMetrics = append(w.bufMetrics, r) - metrics.RecordIngest("metrics", 1) - metrics.UpdateQueueDepth("metrics", len(metricsCh)) - if w.shouldFlush() { - w.flush(flushCh) - } - case <-ticker.C: - w.flush(flushCh) - case <-ctx.Done(): - return finish() - } - - if spansCh == nil && logsCh == nil && metricsCh == nil { - return finish() - } - } -} - -// shouldFlush reports whether any buffer has reached the configured batch size. -func (w *Writer) shouldFlush() bool { - total := len(w.bufSpans) + len(w.bufLogs) + len(w.bufMetrics) - return len(w.bufSpans) >= w.cfg.FlushBatchSize || - len(w.bufLogs) >= w.cfg.FlushBatchSize || - len(w.bufMetrics) >= w.cfg.FlushBatchSize || - total >= w.cfg.FlushBatchSize -} - -// flush detaches the current buffers and hands them to the flush worker. Sending -// on flushCh blocks if the worker is behind, which is the intended backpressure. -func (w *Writer) flush(flushCh chan<- flushBatch) { - // Copy the filled rows into a freshly-sized batch and RETAIN the receive - // buffers (truncate, keep the backing array). The per-row append in the hot - // receive loop then never reallocates after warmup — it was the top ingest - // allocator (profiled ~11.5GB / 23%), and a sync.Pool didn't help because - // the GC-heavy workload evicts pooled buffers every cycle. The cost moves to - // one exact-sized copy per flush (infrequent) instead of a per-row regrow. - batch := flushBatch{} - if len(w.bufSpans) > 0 { - batch.spans = make([]SpanRow, len(w.bufSpans)) - copy(batch.spans, w.bufSpans) - w.bufSpans = w.bufSpans[:0] - } - if len(w.bufLogs) > 0 { - batch.logs = make([]LogRow, len(w.bufLogs)) - copy(batch.logs, w.bufLogs) - w.bufLogs = w.bufLogs[:0] - } - if len(w.bufMetrics) > 0 { - batch.metrics = make([]MetricRow, len(w.bufMetrics)) - copy(batch.metrics, w.bufMetrics) - w.bufMetrics = w.bufMetrics[:0] - } - if batch.spans == nil && batch.logs == nil && batch.metrics == nil { - return - } - flushCh <- batch -} - -// flushWorker serializes all database writes. It carries rows that failed to -// insert forward and prepends them to the next batch so a transient error -// doesn't drop data (until the retry buffer cap is exceeded — see retainRows). -// When the input closes, it retries the final carry and reports any rows that -// remain unwritten to Run. -func (w *Writer) flushWorker(flushCh <-chan flushBatch, workerDone chan<- error) { - var carry flushBatch - var spanErr, logErr, metricErr error - for batch := range flushCh { - // Common path (no retry leftover): adopt the batch slice directly — no - // copy. Only when carry holds un-written rows from a failed flush do we - // append (carry's backing array is reused across flushes). - if len(carry.spans) == 0 { - carry.spans = batch.spans - } else { - carry.spans = append(carry.spans, batch.spans...) - } - if len(carry.logs) == 0 { - carry.logs = batch.logs - } else { - carry.logs = append(carry.logs, batch.logs...) - } - if len(carry.metrics) == 0 { - carry.metrics = batch.metrics - } else { - carry.metrics = append(carry.metrics, batch.metrics...) - } - carry.spans, spanErr = writeRows(carry.spans, "spans", w.insertSpans, w.retryCap()) - carry.logs, logErr = writeRows(carry.logs, "logs", w.insertLogs, w.retryCap()) - carry.metrics, metricErr = writeRows(carry.metrics, "metrics", w.insertMetrics, w.retryCap()) - } - - // A normal batch already had one attempt above. Give the final carry two - // additional attempts with a short backoff, then surface a hard error to Run. - // Successful signals are cleared independently so one failing table cannot - // cause already-committed rows from another table to be duplicated. - for attempt := 1; hasRows(carry) && attempt <= 2; attempt++ { - time.Sleep(time.Duration(attempt) * 100 * time.Millisecond) - carry.spans, spanErr = writeRows(carry.spans, "spans", w.insertSpans, w.retryCap()) - carry.logs, logErr = writeRows(carry.logs, "logs", w.insertLogs, w.retryCap()) - carry.metrics, metricErr = writeRows(carry.metrics, "metrics", w.insertMetrics, w.retryCap()) - } - - var errs []error - if len(carry.spans) > 0 { - errs = append(errs, fmt.Errorf("spans final flush (%d rows): %w", len(carry.spans), spanErr)) - } - if len(carry.logs) > 0 { - errs = append(errs, fmt.Errorf("logs final flush (%d rows): %w", len(carry.logs), logErr)) - } - if len(carry.metrics) > 0 { - errs = append(errs, fmt.Errorf("metrics final flush (%d rows): %w", len(carry.metrics), metricErr)) - } - workerDone <- errors.Join(errs...) -} - -func hasRows(batch flushBatch) bool { - return len(batch.spans) > 0 || len(batch.logs) > 0 || len(batch.metrics) > 0 -} - -// writeRows inserts a batch and returns the rows to carry forward plus the -// insertion error: empty/nil on success, or the retry-capped remainder/error -// on failure. -func writeRows[T any](rows []T, signal string, insert func([]T) error, retryCap int) ([]T, error) { - if len(rows) == 0 { - return rows[:0], nil - } - start := time.Now() - if err := insert(rows); err != nil { - slog.Error("write failed", "signal", signal, "err", err) - metrics.FlushErrors.WithLabelValues(signal).Inc() - return retainRows(rows, retryCap, signal), err - } - metrics.RecordFlush(signal, 0, time.Since(start).Seconds()) - return rows[:0], nil -} - -// drainChannels non-blockingly pulls any buffered rows into the local buffers -// during shutdown. It honors the ok flag so a closed channel is retired (set to -// nil) instead of spinning on zero values. -func (w *Writer) drainChannels(spansCh *<-chan SpanRow, logsCh *<-chan LogRow, metricsCh *<-chan MetricRow) { - for { - drained := false - - select { - case r, ok := <-*spansCh: - if ok { - w.bufSpans = append(w.bufSpans, r) - drained = true - } else { - *spansCh = nil - } - default: - } - select { - case r, ok := <-*logsCh: - if ok { - w.bufLogs = append(w.bufLogs, r) - drained = true - } else { - *logsCh = nil - } - default: - } - select { - case r, ok := <-*metricsCh: - if ok { - w.bufMetrics = append(w.bufMetrics, r) - drained = true - } else { - *metricsCh = nil - } - default: - } - - if !drained { - return - } - } -} - -func (w *Writer) retryCap() int { - maxRetry := w.cfg.FlushBatchSize * 3 - if maxRetry < 0 { - return 0 - } - return maxRetry -} - -func (w *Writer) insertSpans(rows []SpanRow) error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - return withAppender(ctx, w.db, w.writeGate, writegate.WriteIngestSpans, "spans", func(a *duckdb.Appender) error { - for _, row := range rows { - namespace := normalizeNamespace(row.Namespace) - if err := a.AppendRow( - namespace, - row.TraceID, - row.SpanID, - optionalString(row.ParentSpanID), - row.ServiceName, - row.Name, - row.Kind, - eventTime(row.StartUnixNanos, 0, row.IngestedAt), - optionalTime(row.EndUnixNanos), - row.StartUnixNanos, - row.EndUnixNanos, - row.DurationMs, - row.StatusCode, - optionalString(row.StatusMsg), - optionalJSON(row.ResourceJSON), - optionalJSON(row.AttributesJSON), - optionalJSON(row.EventsJSON), - optionalJSON(row.LinksJSON), - optionalString(row.TraceState), - int64(row.Flags), - optionalString(row.ScopeName), - optionalString(row.ScopeVersion), - optionalTime(row.IngestedAt), - row.IngestedAt, - optionalString(row.HTTPMethod), - optionalString(row.HTTPStatusCode), - optionalString(row.HTTPRoute), - optionalString(row.DBSystem), - optionalString(row.RPCMethod), - optionalString(row.RPCService), - optionalString(row.PeerService), - optionalString(row.ServiceVersion), - optionalString(row.DeploymentEnv), - optionalString(row.ExceptionType), - optionalString(row.ExceptionMessage), - ); err != nil { - // Skip the malformed row rather than aborting the batch: the - // appender flushes rows already added on Close, so returning here - // would commit the prefix and then re-append it on retry (a - // duplicate), and a permanently-bad row would poison every flush. - slog.Error("skip malformed span row", "err", err) - metrics.RowsDropped.WithLabelValues("spans").Inc() - continue - } - } - return nil - }) -} - -func (w *Writer) insertLogs(rows []LogRow) error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - return withAppender(ctx, w.db, w.writeGate, writegate.WriteIngestLogs, "logs", func(a *duckdb.Appender) error { - for _, row := range rows { - namespace := normalizeNamespace(row.Namespace) - if err := a.AppendRow( - namespace, - eventTime(row.TimeUnixNanos, row.ObservedTimeNanos, row.IngestedAt), - eventTime(row.ObservedTimeNanos, row.TimeUnixNanos, row.IngestedAt), - row.TimeUnixNanos, - optionalInt64(row.ObservedTimeNanos), - row.Severity, - int64(row.SeverityNumber), - row.Body, - optionalString(row.ServiceName), - optionalString(row.TraceID), - optionalString(row.SpanID), - int64(row.Flags), - optionalJSON(row.ResourceJSON), - optionalJSON(row.AttributesJSON), - optionalString(row.ScopeName), - optionalString(row.ScopeVersion), - optionalTime(row.IngestedAt), - row.IngestedAt, - optionalString(row.BodyTemplate), - ); err != nil { - slog.Error("skip malformed log row", "err", err) - metrics.RowsDropped.WithLabelValues("logs").Inc() - continue - } - } - return nil - }) -} - -func (w *Writer) insertMetrics(rows []MetricRow) error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - return withAppender(ctx, w.db, w.writeGate, writegate.WriteIngestMetrics, "metrics", func(a *duckdb.Appender) error { - for _, row := range rows { - namespace := normalizeNamespace(row.Namespace) - if err := a.AppendRow( - namespace, - eventTime(row.TimeUnixNanos, 0, row.IngestedAt), - row.TimeUnixNanos, - row.Name, - optionalString(row.Description), - optionalString(row.Unit), - row.MType, - optionalString(row.ServiceName), - row.Value, - optionalJSON(row.HistBoundsJSON), - optionalJSON(row.HistCountsJSON), - optionalInt64(row.HistCount), - row.HistSum, - optionalJSON(row.ExemplarsJSON), - optionalJSON(row.AttributesJSON), - optionalJSON(row.ResourceJSON), - optionalString(row.ScopeName), - optionalString(row.ScopeVersion), - optionalTime(row.IngestedAt), - row.IngestedAt, - ); err != nil { - slog.Error("skip malformed metric row", "err", err) - metrics.RowsDropped.WithLabelValues("metrics").Inc() - continue - } - } - return nil - }) -} - -func normalizeNamespace(namespace string) string { - if namespace == "" { - namespace = "default" - } - return namespace -} - -// eventTime keeps query event-time columns populated when a producer omits its -// primary OTLP timestamp. The fallback order is primary, secondary (for logs), -// then ingest time. Raw *_unix_nano fields remain unchanged and therefore -// preserve which timestamps the producer actually supplied. -func eventTime(primary, secondary, ingested int64) any { - for _, nanos := range []int64{primary, secondary, ingested} { - if nanos > 0 { - return time.Unix(0, nanos).UTC() - } - } - return nil -} - -func optionalString(v string) any { - if v == "" { - return nil - } - return v -} - -func optionalJSON(v []byte) any { - if len(v) == 0 { - return nil - } - return string(v) -} - -func optionalTime(unixNano int64) any { - if unixNano <= 0 { - return nil - } - return time.Unix(0, unixNano).UTC() -} - -func optionalInt64(v int64) any { - if v == 0 { - return nil - } - return v -} - -func withAppender(ctx context.Context, db *sql.DB, gate *writegate.WriteGate, operation writegate.WriteOperation, table string, fn func(a *duckdb.Appender) error) error { - // Acquire the write gate before the connection so lock ordering matches the - // query layer (write gate → conn); the two write paths therefore can't deadlock - // against each other. (A writer holding the gate can still wait on a connection - // behind long-running readers on a small pool — that's throughput, not a - // deadlock, since readers never take the write gate.) - if gate != nil { - unlock := gate.Lock(operation) - defer unlock() - } - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - - return conn.Raw(func(raw any) error { - driverConn, ok := raw.(driver.Conn) - if !ok { - return fmt.Errorf("unexpected driver connection type %T", raw) - } - appender, err := duckdb.NewAppender(driverConn, "lake", "", table) - if err != nil { - return err - } - if err := fn(appender); err != nil { - _ = appender.Close() - return err - } - return appender.Close() - }) -} - -func retainRows[T any](rows []T, maxRetry int, signal string) []T { - if len(rows) <= maxRetry { - return rows - } - dropped := len(rows) - maxRetry - slog.Error("data loss: retry buffer full, dropping rows", "signal", signal, "buffered", len(rows), "dropped", dropped) - metrics.RowsDropped.WithLabelValues(signal).Add(float64(dropped)) - return rows[:maxRetry] -} diff --git a/internal/lake/writer_test.go b/internal/lake/writer_test.go deleted file mode 100644 index f17e68b3..00000000 --- a/internal/lake/writer_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package lake - -import ( - "context" - "database/sql" - "testing" - "time" - - _ "github.com/duckdb/duckdb-go/v2" - - "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/query" -) - -func openWriterTestDB(t *testing.T) *sql.DB { - t.Helper() - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatalf("open duckdb: %v", err) - } - if _, err := db.Exec(`ATTACH ':memory:' AS lake`); err != nil { - t.Fatalf("attach lake catalog: %v", err) - } - if err := query.CreateTables(db); err != nil { - t.Fatalf("CreateTables failed: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - return db -} - -func TestWriterFlushBatchSize(t *testing.T) { - db := openWriterTestDB(t) - - chSpans := make(chan SpanRow, 10) - chLogs := make(chan LogRow, 10) - chMetrics := make(chan MetricRow, 10) - - w := NewWriter(config.Config{ - FlushInterval: time.Minute, - FlushBatchSize: 2, - DefaultNamespace: "default", - }, db, chSpans, chLogs, chMetrics) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { _ = w.Run(ctx) }() - - now := time.Now().UnixNano() - chSpans <- SpanRow{ - Namespace: "ns-a", - TraceID: "trace-1", - SpanID: "span-1", - ServiceName: "svc", - Name: "op", - StartUnixNanos: now, - EndUnixNanos: now + int64(time.Millisecond), - DurationMs: 1, - IngestedAt: now, - } - chSpans <- SpanRow{ - Namespace: "ns-a", - TraceID: "trace-1", - SpanID: "span-2", - ServiceName: "svc", - Name: "op", - StartUnixNanos: now, - EndUnixNanos: now + int64(time.Millisecond), - DurationMs: 1, - IngestedAt: now, - } - - requireCount(t, db, `SELECT count(*) FROM lake.spans`, 2) -} - -func TestWriterFlushesRemainderOnShutdown(t *testing.T) { - db := openWriterTestDB(t) - - chSpans := make(chan SpanRow, 10) - chLogs := make(chan LogRow, 10) - chMetrics := make(chan MetricRow, 10) - - w := NewWriter(config.Config{ - FlushInterval: time.Hour, // long, so only shutdown triggers the flush - FlushBatchSize: 1000, // large, so the single row never hits a size flush - DefaultNamespace: "default", - }, db, chSpans, chLogs, chMetrics) - - ctx, cancel := context.WithCancel(context.Background()) - go func() { _ = w.Run(ctx) }() - - now := time.Now().UnixNano() - chSpans <- SpanRow{ - Namespace: "ns-a", TraceID: "t", SpanID: "s", ServiceName: "svc", Name: "op", - StartUnixNanos: now, EndUnixNanos: now + 1, DurationMs: 1, IngestedAt: now, - } - - // Give the row time to be received into the buffer, then shut down. The - // remaining buffered row must be drained and flushed before Wait() returns. - time.Sleep(100 * time.Millisecond) - cancel() - w.Wait() - - var got int - if err := db.QueryRow(`SELECT count(*) FROM lake.spans`).Scan(&got); err != nil { - t.Fatalf("count query failed: %v", err) - } - if got != 1 { - t.Fatalf("count = %d, want 1 (row should be flushed on shutdown)", got) - } -} - -func TestEventTimeFallsBackWithoutSchemaDuplication(t *testing.T) { - ingested := time.Now().UTC().Truncate(time.Microsecond) - got, ok := eventTime(0, 0, ingested.UnixNano()).(time.Time) - if !ok { - t.Fatalf("eventTime() type = %T, want time.Time", got) - } - if !got.Equal(ingested) { - t.Fatalf("eventTime() = %s, want %s", got, ingested) - } - - observed := ingested.Add(-time.Second) - got, ok = eventTime(0, observed.UnixNano(), ingested.UnixNano()).(time.Time) - if !ok || !got.Equal(observed) { - t.Fatalf("eventTime() secondary fallback = %v, want %s", got, observed) - } - - primary := observed.Add(-time.Second) - got, ok = eventTime(primary.UnixNano(), observed.UnixNano(), ingested.UnixNano()).(time.Time) - if !ok || !got.Equal(primary) { - t.Fatalf("eventTime() primary timestamp = %v, want %s", got, primary) - } -} - -func TestFlushWorkerReportsUnwrittenFinalCarry(t *testing.T) { - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatalf("open duckdb: %v", err) - } - if err := db.Close(); err != nil { - t.Fatalf("close duckdb: %v", err) - } - - w := &Writer{db: db, cfg: config.Config{FlushBatchSize: 10}} - flushCh := make(chan flushBatch, 1) - done := make(chan error, 1) - flushCh <- flushBatch{spans: []SpanRow{{TraceID: "t", SpanID: "s"}}} - close(flushCh) - w.flushWorker(flushCh, done) - if err := <-done; err == nil { - t.Fatal("flushWorker() error = nil, want final-flush error") - } -} - -func requireCount(t *testing.T, db *sql.DB, q string, want int) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - var got int - if err := db.QueryRow(q).Scan(&got); err == nil && got == want { - return - } - time.Sleep(25 * time.Millisecond) - } - - var got int - if err := db.QueryRow(q).Scan(&got); err != nil { - t.Fatalf("count query failed: %v", err) - } - t.Fatalf("count = %d, want %d", got, want) -} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 859f7099..ded6483c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -24,20 +24,21 @@ const ( RollupDisabled RollupResult = "disabled" ) -type DuckLakeOperation string +type TelemetryOperation string const ( - DuckLakeMerge DuckLakeOperation = "merge" - DuckLakeMaintenance DuckLakeOperation = "maintenance" + TelemetryCompaction TelemetryOperation = "compaction" + TelemetryMaintenance TelemetryOperation = "maintenance" ) -type DuckLakeResult string +type TelemetryResult string const ( - DuckLakeSuccess DuckLakeResult = "success" - DuckLakeError DuckLakeResult = "error" - DuckLakeDisabled DuckLakeResult = "disabled" - DuckLakeThrottled DuckLakeResult = "throttled" + TelemetrySuccess TelemetryResult = "success" + TelemetryError TelemetryResult = "error" + TelemetryDisabled TelemetryResult = "disabled" + TelemetryThrottled TelemetryResult = "throttled" + TelemetryNoop TelemetryResult = "noop" ) var ( @@ -80,13 +81,13 @@ var ( WriteGateWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "fanout_write_gate_wait_seconds", - Help: "Time spent waiting to enter the DuckLake catalog write critical section", + Help: "Time spent waiting to enter the Telemetry catalog write critical section", Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, }, []string{"operation"}) WriteGateHold = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "fanout_write_gate_hold_seconds", - Help: "Time spent inside the DuckLake catalog write critical section", + Help: "Time spent inside the Telemetry catalog write critical section", Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, }, []string{"operation"}) @@ -170,26 +171,26 @@ var ( Help: "Estimated bounded catch-up chunks remaining for the rollup", }, []string{"rollup"}) - DuckLakeOperationTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "fanout_ducklake_operation_total", - Help: "DuckLake merge and maintenance calls by bounded outcome", + TelemetryOperationTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "fanout_telemetry_operation_total", + Help: "Telemetry compaction and maintenance calls by bounded outcome", }, []string{"operation", "result"}) - DuckLakeOperationDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "fanout_ducklake_operation_duration_seconds", - Help: "Executed DuckLake merge and maintenance duration in seconds, including write-gate wait", + TelemetryOperationDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "fanout_telemetry_operation_duration_seconds", + Help: "Executed telemetry compaction and maintenance duration in seconds", Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120, 300}, }, []string{"operation"}) // Storage metrics - LakeSize = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "fanout_lake_size_bytes", - Help: "Total size of lake data in bytes", + ParquetSize = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fanout_parquet_size_bytes", + Help: "Total size of telemetry Parquet files in bytes", }, []string{"signal"}) - LakePartitions = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "fanout_lake_partitions", - Help: "Number of partitions per signal", + ParquetFiles = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fanout_parquet_files", + Help: "Number of telemetry Parquet files per signal", }, []string{"signal"}) // HTTP metrics @@ -228,7 +229,7 @@ func RecordFlush(signal string, bytes int64, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } -// RecordWriteGate records one complete DuckLake catalog write critical section. +// RecordWriteGate records one complete Telemetry catalog write critical section. // Callers constrain operation to the fixed writegate.WriteOperation set so this // metric cannot grow with tenant or telemetry cardinality. func RecordWriteGate(operation string, waitSec, holdSec float64) { @@ -294,21 +295,21 @@ func UpdateRollupProgress(component RollupComponent, enabled bool, watermarkNano RollupBacklogChunks.WithLabelValues(string(component)).Set(float64(backlogChunks)) } -// RecordDuckLakeOperation records an outcome for merge or maintenance. Skipped +// RecordTelemetryOperation records an outcome for compaction or maintenance. Skipped // calls have no duration sample so throttle ticks cannot distort execution p95. -func RecordDuckLakeOperation(operation DuckLakeOperation, result DuckLakeResult, durationSec float64) { - validateDuckLakeOperation(operation) - validateDuckLakeResult(result) - DuckLakeOperationTotal.WithLabelValues(string(operation), string(result)).Inc() - if result == DuckLakeSuccess || result == DuckLakeError { - DuckLakeOperationDuration.WithLabelValues(string(operation)).Observe(math.Max(durationSec, 0)) +func RecordTelemetryOperation(operation TelemetryOperation, result TelemetryResult, durationSec float64) { + validateTelemetryOperation(operation) + validateTelemetryResult(result) + TelemetryOperationTotal.WithLabelValues(string(operation), string(result)).Inc() + if result == TelemetrySuccess || result == TelemetryError { + TelemetryOperationDuration.WithLabelValues(string(operation)).Observe(math.Max(durationSec, 0)) } } -// UpdateLakeStats updates lake storage metrics -func UpdateLakeStats(signal string, bytes int64, partitions int) { - LakeSize.WithLabelValues(signal).Set(float64(bytes)) - LakePartitions.WithLabelValues(signal).Set(float64(partitions)) +// UpdateParquetStats updates open Parquet storage metrics. +func UpdateParquetStats(signal string, bytes int64, partitions int) { + ParquetSize.WithLabelValues(signal).Set(float64(bytes)) + ParquetFiles.WithLabelValues(signal).Set(float64(partitions)) } // UpdateQueueDepth updates queue depth metric @@ -334,20 +335,20 @@ func validateRollupResult(result RollupResult) { } } -func validateDuckLakeOperation(operation DuckLakeOperation) { +func validateTelemetryOperation(operation TelemetryOperation) { switch operation { - case DuckLakeMerge, DuckLakeMaintenance: + case TelemetryCompaction, TelemetryMaintenance: return default: - panic("metrics: invalid DuckLake operation: " + string(operation)) + panic("metrics: invalid Telemetry operation: " + string(operation)) } } -func validateDuckLakeResult(result DuckLakeResult) { +func validateTelemetryResult(result TelemetryResult) { switch result { - case DuckLakeSuccess, DuckLakeError, DuckLakeDisabled, DuckLakeThrottled: + case TelemetrySuccess, TelemetryError, TelemetryDisabled, TelemetryThrottled, TelemetryNoop: return default: - panic("metrics: invalid DuckLake result: " + string(result)) + panic("metrics: invalid Telemetry result: " + string(result)) } } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 2acba47d..3bfc9010 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -136,32 +136,32 @@ func TestRecordRollupComponentAndProgress(t *testing.T) { } } -func TestRecordDuckLakeOperationOutcomes(t *testing.T) { - DuckLakeOperationTotal.Reset() - DuckLakeOperationDuration.Reset() +func TestRecordTelemetryOperationOutcomes(t *testing.T) { + TelemetryOperationTotal.Reset() + TelemetryOperationDuration.Reset() - RecordDuckLakeOperation(DuckLakeMerge, DuckLakeSuccess, 0.2) - RecordDuckLakeOperation(DuckLakeMerge, DuckLakeThrottled, 0) - RecordDuckLakeOperation(DuckLakeMerge, DuckLakeDisabled, 0) - RecordDuckLakeOperation(DuckLakeMaintenance, DuckLakeError, 1.5) + RecordTelemetryOperation(TelemetryCompaction, TelemetrySuccess, 0.2) + RecordTelemetryOperation(TelemetryCompaction, TelemetryThrottled, 0) + RecordTelemetryOperation(TelemetryCompaction, TelemetryDisabled, 0) + RecordTelemetryOperation(TelemetryMaintenance, TelemetryError, 1.5) for _, tc := range []struct { operation string result string }{ - {"merge", "success"}, - {"merge", "throttled"}, - {"merge", "disabled"}, + {"compaction", "success"}, + {"compaction", "throttled"}, + {"compaction", "disabled"}, {"maintenance", "error"}, } { - if got := testutil.ToFloat64(DuckLakeOperationTotal.WithLabelValues(tc.operation, tc.result)); got != 1 { - t.Errorf("DuckLakeOperationTotal[%s,%s] = %f, want 1", tc.operation, tc.result, got) + if got := testutil.ToFloat64(TelemetryOperationTotal.WithLabelValues(tc.operation, tc.result)); got != 1 { + t.Errorf("TelemetryOperationTotal[%s,%s] = %f, want 1", tc.operation, tc.result, got) } } - if got := histogramSampleCount(t, "fanout_ducklake_operation_duration_seconds", "operation", "merge"); got != 1 { + if got := histogramSampleCount(t, "fanout_telemetry_operation_duration_seconds", "operation", "compaction"); got != 1 { t.Errorf("merge duration samples = %d, want 1 (skipped outcomes must not distort duration)", got) } - if got := histogramSampleCount(t, "fanout_ducklake_operation_duration_seconds", "operation", "maintenance"); got != 1 { + if got := histogramSampleCount(t, "fanout_telemetry_operation_duration_seconds", "operation", "maintenance"); got != 1 { t.Errorf("maintenance duration samples = %d, want 1", got) } } @@ -170,8 +170,8 @@ func TestBoundedMetricLabelsRejectUnknownValues(t *testing.T) { for name, call := range map[string]func(){ "rollup component": func() { RecordRollupComponent(RollupComponent("tenant"), RollupSuccess, 0, 0) }, "rollup result": func() { RecordRollupComponent(RollupService, RollupResult("unknown"), 0, 0) }, - "lake operation": func() { RecordDuckLakeOperation(DuckLakeOperation("query"), DuckLakeSuccess, 0) }, - "lake result": func() { RecordDuckLakeOperation(DuckLakeMerge, DuckLakeResult("unknown"), 0) }, + "lake operation": func() { RecordTelemetryOperation(TelemetryOperation("query"), TelemetrySuccess, 0) }, + "lake result": func() { RecordTelemetryOperation(TelemetryCompaction, TelemetryResult("unknown"), 0) }, } { t.Run(name, func(t *testing.T) { defer func() { @@ -184,30 +184,30 @@ func TestBoundedMetricLabelsRejectUnknownValues(t *testing.T) { } } -func TestUpdateLakeStats(t *testing.T) { +func TestUpdateParquetStats(t *testing.T) { // Reset metrics - LakeSize.Reset() - LakePartitions.Reset() + ParquetSize.Reset() + ParquetFiles.Reset() - UpdateLakeStats("spans", 1024*1024, 10) - UpdateLakeStats("logs", 512*1024, 5) + UpdateParquetStats("spans", 1024*1024, 10) + UpdateParquetStats("logs", 512*1024, 5) // Check spans size - spansSize := testutil.ToFloat64(LakeSize.WithLabelValues("spans")) + spansSize := testutil.ToFloat64(ParquetSize.WithLabelValues("spans")) if spansSize != 1024*1024 { - t.Errorf("LakeSize[spans] = %f, want %d", spansSize, 1024*1024) + t.Errorf("ParquetSize[spans] = %f, want %d", spansSize, 1024*1024) } // Check spans partitions - spansPartitions := testutil.ToFloat64(LakePartitions.WithLabelValues("spans")) + spansPartitions := testutil.ToFloat64(ParquetFiles.WithLabelValues("spans")) if spansPartitions != 10 { - t.Errorf("LakePartitions[spans] = %f, want 10", spansPartitions) + t.Errorf("ParquetFiles[spans] = %f, want 10", spansPartitions) } // Check logs size - logsSize := testutil.ToFloat64(LakeSize.WithLabelValues("logs")) + logsSize := testutil.ToFloat64(ParquetSize.WithLabelValues("logs")) if logsSize != 512*1024 { - t.Errorf("LakeSize[logs] = %f, want %d", logsSize, 512*1024) + t.Errorf("ParquetSize[logs] = %f, want %d", logsSize, 512*1024) } } diff --git a/internal/observability/logs.go b/internal/observability/logs.go index e4d2300c..4e8a5e37 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -3,39 +3,12 @@ package observability import ( "context" "fmt" + "sort" "strings" -) - -// The user-supplied search filter must compare against the REDACTED body, -// not the raw column: a telemetry viewer could otherwise confirm a secret's -// presence by probing search= even though the display shows -// [REDACTED]. redactedBodySQL replays the exact -// Go-side patterns inside DuckDB (same RE2 engine; parity pinned by -// TestRedactSQLMatchesGo), and both the row query and the histogram query -// use it so their counts can never disagree and leak the same signal. -var redactedBodySQL = redactLogBodySQL("COALESCE(body, '')") - -var logsEntriesQuery = ` -SELECT time, COALESCE(severity, ''), COALESCE(service, ''), COALESCE(body, ''), - COALESCE(trace_id, ''), COALESCE(span_id, '') -FROM logs -WHERE time >= ? AND time < ? AND (? = '' OR namespace = ?) - AND (? = '' OR service = ?) - AND (? = '' OR upper(severity) = upper(?)) - AND (? = '' OR ` + redactedBodySQL + ` ILIKE ?) -ORDER BY time DESC -LIMIT ?` + "time" -var logsBucketsQuery = ` -SELECT time_bucket(INTERVAL '5 minutes', time) AS point_time, - COALESCE(NULLIF(upper(severity), ''), 'UNSPECIFIED'), CAST(COUNT(*) AS BIGINT) -FROM logs -WHERE time >= ? AND time < ? AND (? = '' OR namespace = ?) - AND (? = '' OR service = ?) - AND (? = '' OR upper(severity) = upper(?)) - AND (? = '' OR ` + redactedBodySQL + ` ILIKE ?) -GROUP BY point_time, severity -ORDER BY point_time ASC, severity ASC` + "github.com/labstack/fanout/internal/telemetry" +) func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, search string, limit int) (Result[Logs], error) { scope, err := s.normalizeScope(scope) @@ -46,56 +19,60 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear if err != nil { return Result[Logs]{}, err } - service = strings.TrimSpace(service) - severity = strings.TrimSpace(severity) - search = strings.TrimSpace(search) - pattern := search - if pattern != "" { - pattern = "%" + pattern + "%" - } - + service, severity, search = strings.TrimSpace(service), strings.TrimSpace(severity), strings.TrimSpace(search) + search = strings.ToLower(search) data := Logs{Entries: []LogEntry{}, Buckets: []LogBucket{}} - rows, err := s.db.QueryContext(ctx, logsEntriesQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, severity, severity, search, pattern, limit) - if err != nil { - return Result[Logs]{}, fmt.Errorf("query logs: %w", err) + type bucketKey struct { + time int64 + severity string } - for rows.Next() { - var entry LogEntry - if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { - rows.Close() - return Result[Logs]{}, fmt.Errorf("scan log: %w", err) + buckets := make(map[bucketKey]int64) + unlock := s.repository.ReadLock() + err = s.repository.Logs.Scan(scope.Start.UnixNano(), scope.End.UnixNano(), func(row telemetry.Log) bool { + select { + case <-ctx.Done(): + return false + default: } - entry.Body = redactLogBody(entry.Body) - data.Entries = append(data.Entries, entry) - } - if err := rows.Err(); err != nil { - rows.Close() - return Result[Logs]{}, fmt.Errorf("iterate logs: %w", err) - } - rows.Close() - - rows, err = s.db.QueryContext(ctx, logsBucketsQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, severity, severity, search, pattern) + if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { + return true + } + body := redactLogBody(row.Body) + if search != "" && !strings.Contains(strings.ToLower(body), search) { + return true + } + entryTime := time.Unix(0, row.EventUnixNanos).UTC() + data.Entries = append(data.Entries, LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) + bucketSeverity := strings.ToUpper(row.Severity) + if bucketSeverity == "" { + bucketSeverity = "UNSPECIFIED" + } + bucketNanos := entryTime.Truncate(5 * time.Minute).UnixNano() + buckets[bucketKey{time: bucketNanos, severity: bucketSeverity}]++ + return true + }) + unlock() if err != nil { - return Result[Logs]{}, fmt.Errorf("query log histogram: %w", err) + return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) } - for rows.Next() { - var bucket LogBucket - if err := rows.Scan(&bucket.Time, &bucket.Severity, &bucket.Count); err != nil { - rows.Close() - return Result[Logs]{}, fmt.Errorf("scan log histogram: %w", err) - } - data.Buckets = append(data.Buckets, bucket) + if err := ctx.Err(); err != nil { + return Result[Logs]{}, err } - if err := rows.Err(); err != nil { - rows.Close() - return Result[Logs]{}, fmt.Errorf("iterate log histogram: %w", err) + sort.Slice(data.Entries, func(i, j int) bool { return data.Entries[i].Time.After(data.Entries[j].Time) }) + if len(data.Entries) > limit { + data.Entries = data.Entries[:limit] } - rows.Close() - + for key, count := range buckets { + data.Buckets = append(data.Buckets, LogBucket{Time: time.Unix(0, key.time).UTC(), Severity: key.severity, Count: count}) + } + sort.Slice(data.Buckets, func(i, j int) bool { + if data.Buckets[i].Time.Equal(data.Buckets[j].Time) { + return data.Buckets[i].Severity < data.Buckets[j].Severity + } + return data.Buckets[i].Time.Before(data.Buckets[j].Time) + }) return Result[Logs]{ - Schema: LogsSchema, - Summary: fmt.Sprintf("%d logs matched the selected telemetry window", len(data.Entries)), - Data: data, - Provenance: s.provenanceFor(scope, "logs"), + Schema: LogsSchema, Summary: fmt.Sprintf("%d logs matched the selected telemetry window", len(data.Entries)), + Data: data, Provenance: s.provenanceFor(scope, "fanout_segments"), }, nil } diff --git a/internal/observability/namespace_test.go b/internal/observability/namespace_test.go index 5a62d33b..0b9092c7 100644 --- a/internal/observability/namespace_test.go +++ b/internal/observability/namespace_test.go @@ -38,7 +38,7 @@ CREATE TABLE service_rollup ( t.Fatalf("insert service rollups: %v", err) } - svc := New(db) + svc := New(db, newTestRepository(t)) result, err := svc.Overview(context.Background(), Scope{Start: stamp.Add(-time.Minute), End: stamp.Add(time.Minute)}, 100) if err != nil { t.Fatalf("Overview: %v", err) diff --git a/internal/observability/performance_benchmark_test.go b/internal/observability/performance_benchmark_test.go index 57fc1a56..898e31d2 100644 --- a/internal/observability/performance_benchmark_test.go +++ b/internal/observability/performance_benchmark_test.go @@ -19,9 +19,6 @@ func BenchmarkEndpointQueries24Hours(b *testing.B) { if _, err := db.Exec(`SET TimeZone='UTC'; SET threads=4`); err != nil { b.Fatal(err) } - if _, err := db.Exec(`ATTACH ':memory:' AS lake`); err != nil { - b.Fatal(err) - } if err := query.CreateTables(db); err != nil { b.Fatal(err) } diff --git a/internal/observability/performance_rollup_test.go b/internal/observability/performance_rollup_test.go index 79a61ccf..831da3fa 100644 --- a/internal/observability/performance_rollup_test.go +++ b/internal/observability/performance_rollup_test.go @@ -21,9 +21,6 @@ func TestEndpointRollupQueryMergesBucketsAndExactBoundaries(t *testing.T) { if _, err := db.Exec(`SET TimeZone='UTC'`); err != nil { t.Fatalf("set timezone: %v", err) } - if _, err := db.Exec(`ATTACH ':memory:' AS lake`); err != nil { - t.Fatalf("attach lake catalog: %v", err) - } if err := query.CreateTables(db); err != nil { t.Fatalf("CreateTables: %v", err) } @@ -93,7 +90,7 @@ FROM (VALUES ` + seed.values + `) t(ms)` t.Fatalf("seed endpoint rollup state: %v", err) } - svc := New(db) + svc := New(db, newTestRepository(t)) svc.endpointMature.Store(true) var cachedCalls, totalCachedCalls int64 var minBucket, maxBucket time.Time diff --git a/internal/observability/service.go b/internal/observability/service.go index 6ee193b8..37f3e90d 100644 --- a/internal/observability/service.go +++ b/internal/observability/service.go @@ -10,6 +10,7 @@ import ( "time" appid "github.com/labstack/fanout/internal/id" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) const ( @@ -32,12 +33,16 @@ type DB interface { type Service struct { db DB + repository *telemetrystore.Repository now func() time.Time endpointMature atomic.Bool } -func New(db DB) *Service { - return &Service{db: db, now: time.Now} +func New(db DB, repository *telemetrystore.Repository) *Service { + if db == nil || repository == nil { + panic("observability requires query engine and telemetry repository") + } + return &Service{db: db, repository: repository, now: time.Now} } func (s *Service) normalizeScope(scope Scope) (Scope, error) { diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 011b538f..e2437623 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -9,8 +9,20 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) +func newTestRepository(t *testing.T) *telemetrystore.Repository { + t.Helper() + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + return repository +} + func newMockService(t *testing.T) (*Service, sqlmock.Sqlmock) { t.Helper() db, mock, err := sqlmock.New() @@ -18,7 +30,7 @@ func newMockService(t *testing.T) (*Service, sqlmock.Sqlmock) { t.Fatalf("sqlmock.New: %v", err) } t.Cleanup(func() { _ = db.Close() }) - svc := New(db) + svc := New(db, newTestRepository(t)) svc.now = func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) } return svc, mock } @@ -204,16 +216,15 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(recentTraceQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout"). WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) - mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). - WithArgs(start, end, "prod", "prod", "trace-1", 20). - WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). - AddRow("root", "", "checkout", "POST /pay", "SERVER", start, 200.0, "ERROR", "declined"). - AddRow("child", "root", "payments", "charge", "CLIENT", start.Add(20*time.Millisecond), 80.0, "OK", "")) - mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs(start, end, "prod", "prod", "trace-1", 20). - WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(150*time.Millisecond), "ERROR", "checkout", "payment declined", "trace-1", "root"). - AddRow(start.Add(160*time.Millisecond), "ERROR", "payments", `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, "trace-1", "child")) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ + {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "ERROR", StatusMsg: "declined"}, + {Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"}, + }, Logs: []telemetry.Log{ + {Namespace: "prod", TimeUnixNanos: start.Add(150 * time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "payment declined", TraceID: "trace-1", SpanID: "root"}, + {Namespace: "prod", TimeUnixNanos: start.Add(160 * time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "payments", Body: `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, TraceID: "trace-1", SpanID: "child"}, + }}); err != nil { + t.Fatalf("commit trace fixture: %v", err) + } result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "checkout", 20) if err != nil { @@ -237,16 +248,13 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - mock.ExpectQuery(regexp.QuoteMeta(logsEntriesQuery)). - WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "%declined%", 10). - WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start, "ERROR", "checkout", "payment declined", "trace-1", "root"). - AddRow(start, "ERROR", "checkout", "card declined: token=abc123", "trace-2", "root2"). - AddRow(start, "ERROR", "checkout", `auth declined: {"password":"hunter2"}`, "trace-3", "root3")) - mock.ExpectQuery(regexp.QuoteMeta(logsBucketsQuery)). - WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "%declined%"). - WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). - AddRow(start, "ERROR", int64(3))) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "logs-fixture", Logs: []telemetry.Log{ + {Namespace: "prod", TimeUnixNanos: start.UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "payment declined", TraceID: "trace-1", SpanID: "root"}, + {Namespace: "prod", TimeUnixNanos: start.Add(time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "card declined: token=abc123", TraceID: "trace-2", SpanID: "root2"}, + {Namespace: "prod", TimeUnixNanos: start.Add(2 * time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: `auth declined: {"password":"hunter2"}`, TraceID: "trace-3", SpanID: "root3"}, + }}); err != nil { + t.Fatalf("commit logs fixture: %v", err) + } result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "checkout", "error", "declined", 10) if err != nil { @@ -258,8 +266,8 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { if want := "card declined: token=[REDACTED]"; result.Data.Entries[1].Body != want { t.Fatalf("log body = %q, want %q (redaction bypassed)", result.Data.Entries[1].Body, want) } - if want := `auth declined: {"password":"[REDACTED]"}`; result.Data.Entries[2].Body != want { - t.Fatalf("log body = %q, want %q (redaction bypassed)", result.Data.Entries[2].Body, want) + if want := `auth declined: {"password":"[REDACTED]"}`; result.Data.Entries[0].Body != want { + t.Fatalf("log body = %q, want %q (redaction bypassed)", result.Data.Entries[0].Body, want) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) diff --git a/internal/observability/trace.go b/internal/observability/trace.go index 5a22faee..fe0ed87f 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -6,6 +6,8 @@ import ( "sort" "strings" "time" + + "github.com/labstack/fanout/internal/telemetry" ) const recentTraceQuery = ` @@ -17,22 +19,6 @@ ORDER BY MAX(CASE WHEN upper(status) IN ('ERROR', 'STATUS_CODE_ERROR') THEN 1 EL MAX(end_time) - MIN(start_time) DESC LIMIT 1` -const traceSpansQuery = ` -SELECT span_id, COALESCE(parent_span_id, ''), service, operation, kind, start_time, - duration_ms, COALESCE(status, ''), COALESCE(status_message, '') -FROM spans -WHERE start_time >= ? AND start_time < ? AND (? = '' OR namespace = ?) AND trace_id = ? -ORDER BY start_time ASC, duration_ms DESC -LIMIT ?` - -const traceLogsQuery = ` -SELECT time, COALESCE(severity, ''), COALESCE(service, ''), COALESCE(body, ''), - COALESCE(trace_id, ''), COALESCE(span_id, '') -FROM logs -WHERE time >= ? AND time < ? AND (? = '' OR namespace = ?) AND trace_id = ? -ORDER BY time ASC -LIMIT ?` - func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service string, limit int) (Result[TraceDetail], error) { scope, err := s.normalizeScope(scope) if err != nil { @@ -42,8 +28,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin if err != nil { return Result[TraceDetail]{}, err } - traceID = strings.TrimSpace(traceID) - service = strings.TrimSpace(service) + traceID, service = strings.TrimSpace(traceID), strings.TrimSpace(service) if traceID == "" { rows, queryErr := s.db.QueryContext(ctx, recentTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service) if queryErr != nil { @@ -64,70 +49,71 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} if traceID != "" { - rows, queryErr := s.db.QueryContext(ctx, traceSpansQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, traceID, limit) - if queryErr != nil { - return Result[TraceDetail]{}, fmt.Errorf("query trace spans: %w", queryErr) + unlock := s.repository.ReadLock() + storedSpans, readErr := s.repository.Spans.Trace(traceID) + if readErr != nil { + unlock() + return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } - serviceSet := map[string]struct{}{} - var first time.Time - var last time.Time - for rows.Next() { - var span TraceSpan - if err := rows.Scan(&span.SpanID, &span.ParentSpanID, &span.Service, &span.Operation, &span.Kind, &span.Start, &span.DurationMS, &span.Status, &span.StatusMessage); err != nil { - rows.Close() - return Result[TraceDetail]{}, fmt.Errorf("scan trace span: %w", err) + startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() + for _, row := range storedSpans { + if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || + (scope.Namespace != "" && row.Namespace != scope.Namespace) { + continue } + data.Spans = append(data.Spans, TraceSpan{SpanID: row.SpanID, ParentSpanID: row.ParentSpanID, Service: row.ServiceName, Operation: row.Name, Kind: row.Kind, Start: time.Unix(0, row.StartUnixNanos).UTC(), DurationMS: row.DurationMS, Status: row.StatusCode, StatusMessage: row.StatusMsg}) + } + sort.Slice(data.Spans, func(i, j int) bool { + if data.Spans[i].Start.Equal(data.Spans[j].Start) { + return data.Spans[i].DurationMS > data.Spans[j].DurationMS + } + return data.Spans[i].Start.Before(data.Spans[j].Start) + }) + if len(data.Spans) > limit { + data.Spans = data.Spans[:limit] + } + + serviceSet := make(map[string]struct{}) + var first, last time.Time + for _, span := range data.Spans { if first.IsZero() || span.Start.Before(first) { first = span.Start } - end := span.Start.Add(time.Duration(span.DurationMS * float64(time.Millisecond))) - if end.After(last) { + if end := span.Start.Add(time.Duration(span.DurationMS * float64(time.Millisecond))); end.After(last) { last = end } if strings.Contains(strings.ToUpper(span.Status), "ERROR") { data.HasError = true } - serviceSet[span.Service] = struct{}{} - data.Spans = append(data.Spans, span) - } - if err := rows.Err(); err != nil { - rows.Close() - return Result[TraceDetail]{}, fmt.Errorf("iterate trace spans: %w", err) + if span.Service != "" { + serviceSet[span.Service] = struct{}{} + } } - rows.Close() if !first.IsZero() { data.DurationMS = last.Sub(first).Seconds() * 1000 } for name := range serviceSet { - if name != "" { - data.Services = append(data.Services, name) - } + data.Services = append(data.Services, name) } sort.Strings(data.Services) - rows, queryErr = s.db.QueryContext(ctx, traceLogsQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, traceID, limit) - if queryErr != nil { - return Result[TraceDetail]{}, fmt.Errorf("query trace logs: %w", queryErr) - } - for rows.Next() { - var entry LogEntry - if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { - rows.Close() - return Result[TraceDetail]{}, fmt.Errorf("scan trace log: %w", err) + readErr = s.repository.Logs.Scan(startNanos, endNanos, func(row telemetry.Log) bool { + if row.TraceID != traceID || (scope.Namespace != "" && row.Namespace != scope.Namespace) { + return true } - entry.Body = redactLogBody(entry.Body) - data.Logs = append(data.Logs, entry) + data.Logs = append(data.Logs, LogEntry{Time: time.Unix(0, row.EventUnixNanos).UTC(), Severity: row.Severity, Service: row.ServiceName, Body: redactLogBody(row.Body), TraceID: row.TraceID, SpanID: row.SpanID}) + return len(data.Logs) < limit + }) + unlock() + if readErr != nil { + return Result[TraceDetail]{}, fmt.Errorf("read trace logs: %w", readErr) } - if err := rows.Err(); err != nil { - rows.Close() - return Result[TraceDetail]{}, fmt.Errorf("iterate trace logs: %w", err) - } - rows.Close() + sort.Slice(data.Logs, func(i, j int) bool { return data.Logs[i].Time.Before(data.Logs[j].Time) }) } summary := "No traces found in this telemetry window" if traceID != "" { summary = fmt.Sprintf("Trace %s contains %d spans across %d services", traceID, len(data.Spans), len(data.Services)) } - return Result[TraceDetail]{Schema: TraceSchema, Summary: summary, Data: data, Provenance: s.provenanceFor(scope, "spans + logs")}, nil + return Result[TraceDetail]{Schema: TraceSchema, Summary: summary, Data: data, Provenance: s.provenanceFor(scope, "fanout_segments")}, nil } diff --git a/internal/query/duck.go b/internal/query/duck.go index 102ea31f..321e0304 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -14,25 +14,27 @@ import ( "time" "github.com/duckdb/duckdb-go/v2" - _ "modernc.org/sqlite" // SQLite driver used to put the DuckLake catalog in WAL mode "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake/writegate" "github.com/labstack/fanout/internal/metrics" + "github.com/labstack/fanout/internal/query/writegate" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) type Duck struct { DB *sql.DB cfg config.Config lastMaintenance time.Time - lastMerge time.Time // cadence for the frequent merge-only compaction pass + repository *telemetrystore.Repository // rollupLagNanos holds the rollup watermark back from the max ingested // timestamp so late/out-of-order commits aren't skipped. Zero disables the // lag (no trailing window). rollupLagNanos int64 - // writeGate serializes and measures all DuckLake catalog write commits so - // multiple pooled connections never commit to the SQLite catalog concurrently. + // writeGate serializes writes to the rebuildable DuckDB rollup cache. writeGate writegate.WriteGate + // parquetMu prevents retention from unlinking a file while DuckDB is opening + // the immutable files selected for a new query. + parquetMu sync.RWMutex // maintHealthMu guards the maintenance health fields below, which the // readiness probe reads while the maintenance pass writes them. maintHealthMu sync.Mutex @@ -68,9 +70,8 @@ const ( defaultDuckDBPoolSize = 1 ) -// WriteGate returns the shared catalog write gate. The ingest writer must use it -// around appender flushes so writes never overlap rollup/maintenance commits on -// a multi-connection pool. +// WriteGate returns the gate that serializes writes to DuckDB's rebuildable +// rollup cache. func (d *Duck) WriteGate() *writegate.WriteGate { return &d.writeGate } // duckDBPoolSize is the effective connection-pool size: the configured value, @@ -172,7 +173,10 @@ func parseDuckBytes(s string) (int64, bool) { } } -func NewDuck(ctx context.Context, cfg config.Config) (*Duck, error) { +func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore.Repository) (*Duck, error) { + if repository == nil { + return nil, errors.New("telemetry repository is required") + } if err := os.MkdirAll(cfg.QueryDir(), 0o755); err != nil { return nil, fmt.Errorf("create query dir: %w", err) } @@ -182,40 +186,20 @@ func NewDuck(ctx context.Context, cfg config.Config) (*Duck, error) { dbPath := cfg.QueryDuckDBPath() tempDir := cfg.QueryTempDir() - metadataPath := cfg.TelemetryDuckLakePath() - dataPath := cfg.TelemetryParquetDir() if err := os.MkdirAll(tempDir, 0o755); err != nil { return nil, fmt.Errorf("create temp dir: %w", err) } - if err := os.MkdirAll(dataPath, 0o755); err != nil { - return nil, fmt.Errorf("create telemetry parquet dir: %w", err) - } - - // Put the DuckLake SQLite catalog in WAL mode before DuckDB attaches it, so - // read queries run concurrently with the single writer instead of failing - // with "database is locked" (rollback-journal mode), and a crashed writer - // can't leave the catalog permanently locked. Must run before openDuckDB. - if err := enableCatalogWAL(metadataPath); err != nil { - return nil, fmt.Errorf("enable WAL on DuckLake catalog: %w", err) - } - dsn, err := duckDSN(dbPath, cfg.DuckDBMemory, cfg.DuckDBThreads) if err != nil { return nil, err } - db, err := openDuckDB(ctx, dsn, tempDir, metadataPath, dataPath, duckDBPoolSize(cfg)) + db, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { - return nil, fmt.Errorf( - "open duckdb catalog: %w (if the local cache catalog is corrupted, remove %s and %s; DuckLake data remains in %s)", - err, - dbPath, - dbPath+".wal", - dataPath, - ) + return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - d := &Duck{DB: db, cfg: cfg, rollupLagNanos: rollupLagFromConfig(cfg)} + d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: rollupLagFromConfig(cfg)} if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -224,7 +208,11 @@ func NewDuck(ctx context.Context, cfg config.Config) (*Duck, error) { slog.Warn("apply memory headroom failed; using DuckDB default memory_limit", "err", err) } } - if err := CreateTables(db); err != nil { + if err := CreateCacheTables(db); err != nil { + _ = db.Close() + return nil, err + } + if err := CreateParquetViews(db, repository.Parquet.Dir()); err != nil { _ = db.Close() return nil, err } @@ -293,16 +281,7 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { return tx.Commit() } -func openDuckDB(ctx context.Context, dsn, tempDir, metadataPath, dataPath string, maxConns int) (*sql.DB, error) { - // AUTOMATIC_MIGRATION upgrades an older on-disk DuckLake catalog to the format - // the loaded extension requires. Without it, a fanout build that bundles a - // newer DuckLake (e.g. the DuckDB 1.5.3 bump, which needs catalog v1.0) fails - // to attach an existing v0.4 catalog and the server can't boot. Migration is - // in place and forward-only. - attach := fmt.Sprintf("ATTACH IF NOT EXISTS %s AS lake (DATA_PATH %s, AUTOMATIC_MIGRATION true)", - sqlLiteral("ducklake:sqlite:"+metadataPath), - sqlLiteral(dataPath)) - +func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, error) { // temp_directory is an instance-global setting: re-setting it after the temp // dir has already been used fails with "Cannot switch temporary directory // after the current one has been used". The boot hook runs once per pooled @@ -312,11 +291,6 @@ func openDuckDB(ctx context.Context, dsn, tempDir, metadataPath, dataPath string var tempDirErr error connector, err := duckdb.NewConnector(dsn, func(execer driver.ExecerContext) error { - for _, stmt := range []string{"LOAD ducklake", "LOAD sqlite"} { - if _, err := execer.ExecContext(ctx, stmt, nil); err != nil { - return err - } - } tempDirOnce.Do(func() { _, tempDirErr = execer.ExecContext(ctx, "SET temp_directory="+sqlLiteral(tempDir), nil) }) @@ -326,9 +300,6 @@ func openDuckDB(ctx context.Context, dsn, tempDir, metadataPath, dataPath string if tempDirErr != nil { return fmt.Errorf("set temp_directory: %w", tempDirErr) } - if _, err := execer.ExecContext(ctx, attach, nil); err != nil { - return err - } return nil }) if err != nil { @@ -336,12 +307,8 @@ func openDuckDB(ctx context.Context, dsn, tempDir, metadataPath, dataPath string } db := sql.OpenDB(connector) - // DuckLake metadata lives in a SQLite catalog that locks under *concurrent* - // commits from multiple connections. The default pool of 1 serializes - // everything through one handle. Larger pools are allowed (read queries then - // run concurrently), but write commits must still be serialized by the - // caller's write gate (Duck.WriteGate) so two connections never commit at - // once. + // The on-disk DuckDB file contains only rebuildable rollups and views; Parquet + // scans may run concurrently across the machine-sized pool. if maxConns < 1 { maxConns = 1 } @@ -350,49 +317,13 @@ func openDuckDB(ctx context.Context, dsn, tempDir, metadataPath, dataPath string return db, nil } -// enableCatalogWAL switches the DuckLake SQLite catalog at metadataPath to WAL -// journal mode before DuckDB attaches it. The default rollback-journal mode -// makes a committing writer take an exclusive lock that fails concurrent readers -// with "database is locked", and a crashed or stuck writer can leave a hot -// journal that locks the catalog until every connection is dropped (observed in -// prod as a multi-day write+query outage). In WAL mode readers run concurrently -// with the single writer, and the next connection to open the catalog recovers -// the WAL automatically after a crash instead of leaving a lock-holding hot -// journal. journal_mode is persisted in the database header, so this is -// effectively a one-time migration, but it is cheap and idempotent to assert on -// every boot. WAL is the only lever available: the DuckDB sqlite extension -// exposes no busy_timeout knob, so DuckDB's own catalog connections have no -// lock-retry timeout — WAL is what prevents the reader/writer collisions. -func enableCatalogWAL(metadataPath string) (err error) { - db, err := sql.Open("sqlite", metadataPath+"?_pragma=journal_mode(wal)") - if err != nil { - return fmt.Errorf("open catalog: %w", err) - } - defer func() { - // Closing the last connection checkpoints the WAL; surface a failure here - // (e.g. the filesystem can't maintain the -wal/-shm sidecars) instead of - // discovering it later as a mysterious lock. - if cerr := db.Close(); cerr != nil && err == nil { - err = fmt.Errorf("close catalog bootstrap conn: %w", cerr) - } - }() - db.SetMaxOpenConns(1) - - var mode string - if scanErr := db.QueryRow("PRAGMA journal_mode").Scan(&mode); scanErr != nil { - return fmt.Errorf("read journal_mode: %w", scanErr) - } - if !strings.EqualFold(mode, "wal") { - return fmt.Errorf("catalog journal_mode = %q, want wal", mode) - } - return nil -} - func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" } -func (d *Duck) Close() error { return d.DB.Close() } +func (d *Duck) Close() error { + return d.DB.Close() +} // DefaultNamespace returns empty string so queries search all namespaces. func (d *Duck) DefaultNamespace() string { @@ -412,7 +343,7 @@ func (d *Duck) RunRollups(ctx context.Context) { } else if rows > 0 { slog.Info("startup rollup complete", "rows", rows, "duration", time.Since(start)) } - d.updateLakeStats(ctx) + d.updateParquetStats() ticker := time.NewTicker(d.cfg.RollupInterval) defer ticker.Stop() @@ -425,41 +356,22 @@ func (d *Duck) RunRollups(ctx context.Context) { if err != nil { slog.Error("rollup failed", "component", "rollup", "rows", rows, "err", err) } - d.updateLakeStats(ctx) + d.updateParquetStats() case <-ctx.Done(): return } } } -// updateLakeStats refreshes the per-signal file-count and byte-size gauges -// (fanout_lake_partitions / fanout_lake_size_bytes) from the DuckLake catalog. -// This is the signal that surfaces unbounded file/snapshot growth — the failure -// mode that previously OOM'd the rollup engine — so an operator or soak test can -// watch it climb. It's a cheap read; a failure here must not disturb rollups. -func (d *Duck) updateLakeStats(ctx context.Context) { - // Bound the catalog read so a degraded/bloated DuckLake can't stall the - // rollup loop (this runs inline after rollupOnce in RunRollups). - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - rows, err := d.DB.QueryContext(ctx, `SELECT table_name, file_count, file_size_bytes FROM ducklake_table_info('lake')`) +// updateParquetStats refreshes the per-signal file-count and byte-size gauges. +func (d *Duck) updateParquetStats() { + stats, err := d.repository.Parquet.Stats() if err != nil { - slog.Warn("lake stats query failed", "err", err) + slog.Warn("parquet stats failed", "err", err) return } - defer rows.Close() - for rows.Next() { - var table string - var fileCount, sizeBytes int64 - if err := rows.Scan(&table, &fileCount, &sizeBytes); err != nil { - slog.Warn("lake stats scan failed", "err", err) - return - } - // DuckLake table names (spans/logs/metrics) are the metric signal labels. - metrics.UpdateLakeStats(table, sizeBytes, int(fileCount)) - } - if err := rows.Err(); err != nil { - slog.Warn("lake stats iteration failed", "err", err) + for signal, stat := range stats { + metrics.UpdateParquetStats(signal, stat.Bytes, stat.Files) } } @@ -485,20 +397,73 @@ func (d *Duck) rollupOnce(ctx context.Context) (int, error) { affected += n } - // Compaction must run even when a rollup fails: a failing rollup is exactly - // when compaction matters most, since file/snapshot growth makes every - // retried pass heavier. The frequent merge pass keeps the file count low; the - // hourly maintenance pass handles retention + snapshot expiry + cleanup. - if err := d.runMerge(ctx); err != nil { - slog.Warn("ducklake merge failed", "err", err) - } - if err := d.runMaintenance(ctx); err != nil { - slog.Warn("ducklake maintenance failed", "err", err) + if err := d.runRepositoryMaintenance(ctx); err != nil { + slog.Warn("telemetry maintenance failed", "err", err) } return int(affected), errors.Join(errs...) } +func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { + every := d.cfg.MaintenanceInterval + if every <= 0 { + every = time.Hour + } + if !d.lastMaintenance.IsZero() && time.Since(d.lastMaintenance) < every { + metrics.RecordTelemetryOperation(metrics.TelemetryMaintenance, metrics.TelemetryThrottled, 0) + return nil + } + start := time.Now() + cutoff := time.Now().Add(-d.cfg.HotRetention).UnixNano() + var pruneErr error + if d.repository != nil { + _, pruneErr = d.repository.PruneHot(cutoff) + d.parquetMu.Lock() + var parquetErr error + if d.cfg.RetentionDays > 0 { + _, parquetErr = d.repository.PruneParquet(time.Now().Add(-time.Duration(d.cfg.RetentionDays) * 24 * time.Hour).UnixNano()) + } + compactStart := time.Now() + compacted, compactErr := d.repository.CompactParquet(ctx, d.DB, 64) + d.parquetMu.Unlock() + compactResult := metrics.TelemetryNoop + if compactErr != nil { + compactResult = metrics.TelemetryError + } else if compacted > 0 { + compactResult = metrics.TelemetrySuccess + } + metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) + pruneErr = errors.Join(pruneErr, parquetErr, compactErr) + } + var cacheErr error + if d.cfg.RetentionDays > 0 { + for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { + if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { + cacheErr = errors.Join(cacheErr, fmt.Errorf("prune %s: %w", table, err)) + } + } + } + _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") + err := errors.Join(pruneErr, cacheErr, checkpointErr) + maintenanceResult := metrics.TelemetrySuccess + if err != nil { + maintenanceResult = metrics.TelemetryError + } + metrics.RecordTelemetryOperation(metrics.TelemetryMaintenance, maintenanceResult, time.Since(start).Seconds()) + d.lastMaintenance = time.Now() + d.maintHealthMu.Lock() + d.lastMaintenanceAt = d.lastMaintenance + if err == nil { + d.lastMaintenanceOK = d.lastMaintenance + } + d.lastMaintenanceErr = err + d.maintHealthMu.Unlock() + if err == nil { + slog.Info("telemetry maintenance complete", "duration", time.Since(start)) + } + return err +} + func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { start := time.Now() result := metrics.RollupError @@ -1070,7 +1035,7 @@ span_agg AS ( ON a.namespace = s.namespace AND a.bucket = date_trunc('minute', s.start_time) AND a.service = s.service - -- Bound the scan to the affected bucket range so DuckLake prunes parquet by + -- Bound the scan to the affected bucket range so Telemetry prunes parquet by -- start_time stats instead of scanning all history (the join on a computed -- date_trunc bucket alone does not prune). The range is a provable superset -- of the affected buckets — every row in an affected bucket has start_time in @@ -1370,225 +1335,23 @@ UNION ALL SELECT namespace, bucket, caller, callee, calls, avg_ms, error_rate, edge_type FROM messaging_edges;` -// snapshotGraceMinutes is how long a superseded DuckLake snapshot (and the -// parquet files it references) is kept past compaction before expiry/cleanup -// may reclaim it. It must exceed the longest read query — reads don't hold -// the write gate, so a shorter window lets cleanup delete a file mid-scan — while -// staying well under the maintenance interval so file/snapshot growth stays -// bounded. Queries are sub-second to a few seconds; 10 minutes is ample margin. -const snapshotGraceMinutes = 10 - -// runMerge runs ONLY ducklake_merge_adjacent_files on a short cadence -// (storage.merge_interval, default 1m). Merge consolidates the newest -// small parquet files and deletes nothing, so it's cheap and safe to run often — -// keeping the queryable file count continuously low is what bounds rollup/query -// scan latency. The deletions (expire_snapshots + cleanup_old_files), which -// carry the read race and catalog cost, stay on the hourly runMaintenance -// cadence. Decoupling the two resolves the churn-vs-pileup tension: frequent -// cheap merge keeps scans fast; rare deletes keep the race and overhead away. -func (d *Duck) runMerge(ctx context.Context) error { - start := time.Now() - every := d.cfg.MergeInterval - if every <= 0 { - metrics.RecordDuckLakeOperation(metrics.DuckLakeMerge, metrics.DuckLakeDisabled, 0) - return nil // merge pass disabled - } - if !d.lastMerge.IsZero() && time.Since(d.lastMerge) < every { - metrics.RecordDuckLakeOperation(metrics.DuckLakeMerge, metrics.DuckLakeThrottled, 0) - return nil - } - // merge commits new files — serialize against other writers like maintenance. - err := func() error { - defer d.writeGate.Lock(writegate.WriteMerge)() - _, err := d.DB.ExecContext(ctx, "CALL ducklake_merge_adjacent_files('lake')") - return err - }() - d.lastMerge = time.Now() - if err != nil { - metrics.RecordDuckLakeOperation(metrics.DuckLakeMerge, metrics.DuckLakeError, time.Since(start).Seconds()) - slog.Error("merge_adjacent_files failed", "err", err) - return fmt.Errorf("merge_adjacent_files: %w", err) - } - metrics.RecordDuckLakeOperation(metrics.DuckLakeMerge, metrics.DuckLakeSuccess, time.Since(start).Seconds()) - return nil -} - -func (d *Duck) runMaintenance(ctx context.Context) error { - start := time.Now() - every := d.cfg.MaintenanceInterval - if every <= 0 { - every = time.Hour - } - if !d.lastMaintenance.IsZero() && time.Since(d.lastMaintenance) < every { - metrics.RecordDuckLakeOperation(metrics.DuckLakeMaintenance, metrics.DuckLakeThrottled, 0) - return nil - } - - // Retention deletes and the checkpoint are writes — serialize them too. - unlock := d.writeGate.Lock(writegate.WriteMaintenance) - defer unlock() - - var errs []error - if d.cfg.RetentionDays > 0 { - stmts := []struct { - name string - sql string - }{ - {name: "lake.spans", sql: fmt.Sprintf("DELETE FROM lake.spans WHERE COALESCE(start_time, ingested_at) < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - {name: "lake.logs", sql: fmt.Sprintf("DELETE FROM lake.logs WHERE COALESCE(log_time, observed_time, ingested_at) < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - {name: "lake.metrics", sql: fmt.Sprintf("DELETE FROM lake.metrics WHERE COALESCE(metric_time, ingested_at) < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - {name: "service_rollup", sql: fmt.Sprintf("DELETE FROM service_rollup WHERE bucket < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - {name: "endpoint_rollup", sql: fmt.Sprintf("DELETE FROM endpoint_rollup WHERE bucket < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - {name: "edge_rollup", sql: fmt.Sprintf("DELETE FROM edge_rollup WHERE bucket < now() - INTERVAL %d DAY", d.cfg.RetentionDays)}, - } - for _, stmt := range stmts { - res, err := d.DB.ExecContext(ctx, stmt.sql) - if err != nil { - slog.Error("maintenance delete failed", "table", stmt.name, "err", err) - errs = append(errs, fmt.Errorf("%s retention delete: %w", stmt.name, err)) - continue - } - rows, rowsErr := res.RowsAffected() - if rowsErr != nil { - slog.Info("maintenance delete complete", "table", stmt.name) - continue - } - slog.Info("maintenance delete complete", "table", stmt.name, "rows", rows) - } - } - - // DuckLake compaction. Every flush commits a snapshot and writes new parquet - // files; without merge + expiry both grow without bound until per-file - // metadata pins OOM every wide query (prod 2026-06-13: 60k snapshots, 50k - // files averaging 21KB, rollups dead at any memory_limit). Holding the write gate - // here quiesces the dataset against WRITES, the precondition for these calls. - // Order: merge rewrites small files into large ones, expiry releases the - // snapshots that referenced the small ones, cleanup deletes the files no live - // snapshot references. DuckLake never expires the newest snapshot, so a - // low-traffic instance (no new commits within the grace) still keeps a - // readable one. - // - // GRACE WINDOW (now() - snapshotGraceMinutes) on EXPIRY: the write gate does NOT - // serialize reads, so an Overview/diagnose query can be mid-scan against a - // snapshot merge just superseded; expiring + deleting its parquet immediately - // yanks the file out from under the reader ("IO Error: Cannot open file …: No - // such file or directory"). Sparing recently-superseded snapshots keeps their - // files referenced (so cleanup won't delete them) until any in-flight reader - // has finished; they're reclaimed a cycle later. At ingest.flush_interval=15s the grace - // retains ~40 snapshots (4/min × 10min) — bounded, nowhere near the 60k OOM, - // and far below the (default 1h) maintenance cycle. - // - // cleanup stays cleanup_all => true: the bundled DuckLake's - // ducklake_cleanup_old_files does NOT accept an older_than grace (verified by - // benchmark — the call errored every cycle), and it isn't needed: expiry's - // grace already prevents within-grace files from being scheduled for deletion, - // so cleanup_all only unlinks files no live snapshot references. - expireSQL := fmt.Sprintf( - "CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL %d MINUTE)", - snapshotGraceMinutes) - for _, stmt := range []struct { - name string - sql string - }{ - {name: "merge_adjacent_files", sql: "CALL ducklake_merge_adjacent_files('lake')"}, - // DuckLake deletes are merge-on-read. Rewriting materializes the retention - // deletes into Parquet so expired rows stop consuming scan and disk budget. - {name: "rewrite_data_files", sql: "CALL ducklake_rewrite_data_files('lake')"}, - {name: "expire_snapshots", sql: expireSQL}, - {name: "cleanup_old_files", sql: "CALL ducklake_cleanup_old_files('lake', cleanup_all => true)"}, - } { - if _, err := d.DB.ExecContext(ctx, stmt.sql); err != nil { - slog.Error("maintenance compaction failed", "step", stmt.name, "err", err) - errs = append(errs, fmt.Errorf("compaction %s: %w", stmt.name, err)) - } - } - - // DuckLake's catalog CHECKPOINT (as opposed to the compaction calls above) - // currently trips an internal error on live datasets with nullable string - // fields, which invalidates the whole database connection. Checkpoint only - // the main local cache until the upstream checkpoint path is stable. - if _, err := d.DB.ExecContext(ctx, "CHECKPOINT"); err != nil { - slog.Error("maintenance checkpoint failed", "target", "main", "err", err) - errs = append(errs, fmt.Errorf("checkpoint main: %w", err)) - } - d.lastMaintenance = time.Now() - - err := errors.Join(errs...) - result := metrics.DuckLakeSuccess - if err != nil { - result = metrics.DuckLakeError - } - metrics.RecordDuckLakeOperation(metrics.DuckLakeMaintenance, result, time.Since(start).Seconds()) - d.maintHealthMu.Lock() - d.lastMaintenanceAt = time.Now() - if err == nil { - d.lastMaintenanceOK = d.lastMaintenanceAt - } - d.lastMaintenanceErr = err - d.maintHealthMu.Unlock() - return err -} - // ---- Read query helpers ---- -// isTransientLakeIOError reports whether err is a DuckLake compaction race: a -// concurrent maintenance pass (merge + cleanup, which runs without blocking -// reads) unlinked a parquet file this read had planned against. Re-running the -// query re-plans against the current snapshot (the merged file), which succeeds. -func isTransientLakeIOError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - return strings.Contains(msg, "IO Error") && strings.Contains(msg, "No such file or directory") -} - -// QueryContext runs a read query, retrying briefly on the transient DuckLake -// "file deleted mid-scan" race (reads don't take the write gate, so maintenance can -// unlink a just-merged file underneath them). Cleanup is instantaneous, so a -// short backoff lets the retry re-plan against fresh files. Use this for every -// read that scans the lake instead of DB.QueryContext directly. +// QueryContext executes a read against immutable Parquet files and DuckDB's +// local rollup cache. func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { - const maxAttempts = 3 - var rows *sql.Rows - var err error - for attempt := 1; attempt <= maxAttempts; attempt++ { - rows, err = d.DB.QueryContext(ctx, query, args...) - if err == nil || attempt == maxAttempts || !isTransientLakeIOError(err) { - return rows, err - } - if rows != nil { - _ = rows.Close() - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(time.Duration(attempt) * 25 * time.Millisecond): - } - } + d.parquetMu.RLock() + rows, err := d.DB.QueryContext(ctx, query, args...) + d.parquetMu.RUnlock() return rows, err } -// QueryRowScan is the single-row analogue of QueryContext: it retries the same -// transient DuckLake "file deleted mid-scan" race. For a single-row read the IO -// error surfaces at Scan (not at QueryRowContext), so the retry wraps -// QueryRowContext+Scan together. Use it for lake-scanning single-row reads -// (`*Duck.QueryContext` covers the multi-row ones). +// QueryRowScan executes a single-row query against immutable Parquet files and +// DuckDB's local rollup cache. func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args ...any) error { - const maxAttempts = 3 - var err error - for attempt := 1; attempt <= maxAttempts; attempt++ { - err = d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) - if err == nil || attempt == maxAttempts || !isTransientLakeIOError(err) { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Duration(attempt) * 25 * time.Millisecond): - } - } - return err + d.parquetMu.RLock() + defer d.parquetMu.RUnlock() + return d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) } // ---- Queries for API ---- diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index f3f37ea8..4beea3f0 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -3,14 +3,13 @@ package query import ( "context" "errors" - "regexp" - "strings" "testing" "time" "github.com/DATA-DOG/go-sqlmock" "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/metrics" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -122,222 +121,6 @@ func TestErrorRouteRowStruct(t *testing.T) { } } -func TestRunMaintenanceContinuesAfterDeleteFailure(t *testing.T) { - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 7}, - } - - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM lake.spans WHERE COALESCE(start_time, ingested_at) < now() - INTERVAL 7 DAY")). - WillReturnResult(sqlmock.NewResult(0, 11)) - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM lake.logs WHERE COALESCE(log_time, observed_time, ingested_at) < now() - INTERVAL 7 DAY")). - WillReturnError(errors.New("log delete failed")) - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM lake.metrics WHERE COALESCE(metric_time, ingested_at) < now() - INTERVAL 7 DAY")). - WillReturnResult(sqlmock.NewResult(0, 7)) - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM service_rollup WHERE bucket < now() - INTERVAL 7 DAY")). - WillReturnResult(sqlmock.NewResult(0, 5)) - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM endpoint_rollup WHERE bucket < now() - INTERVAL 7 DAY")). - WillReturnResult(sqlmock.NewResult(0, 4)) - mock.ExpectExec(regexp.QuoteMeta("DELETE FROM edge_rollup WHERE bucket < now() - INTERVAL 7 DAY")). - WillReturnResult(sqlmock.NewResult(0, 3)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_merge_adjacent_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_rewrite_data_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL 10 MINUTE)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_cleanup_old_files('lake', cleanup_all => true)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CHECKPOINT")). - WillReturnResult(sqlmock.NewResult(0, 0)) - - err = d.runMaintenance(context.Background()) - if err == nil { - t.Fatal("runMaintenance() error = nil, want joined error") - } - if d.lastMaintenance.IsZero() { - t.Fatal("runMaintenance() did not update lastMaintenance") - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("maintenance", "error")); got != 1 { - t.Errorf("maintenance error outcomes = %f, want 1", got) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet sql expectations: %v", err) - } -} - -// A merge failure must not stop snapshot expiry, file cleanup, or the -// checkpoint — each compaction step degrades independently. -// Within storage.maintenance_interval of the last pass, runMaintenance -// must short-circuit before issuing ANY SQL — the throttle that keeps the -// retention+compaction cycle off every rollup tick. -// runMerge issues exactly one merge_adjacent_files call when due, and nothing -// on the next call within the storage.merge_interval cadence. -func TestRunMergeExecutesThenThrottles(t *testing.T) { - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db, cfg: config.Config{MergeInterval: time.Minute}} - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_merge_adjacent_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - - if err := d.runMerge(context.Background()); err != nil { - t.Fatalf("runMerge() = %v, want nil", err) - } - // Second call is within the cadence → must issue no SQL. - if err := d.runMerge(context.Background()); err != nil { - t.Fatalf("throttled runMerge() = %v, want nil", err) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet sql expectations: %v", err) - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("merge", "success")); got != 1 { - t.Errorf("merge success outcomes = %f, want 1", got) - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("merge", "throttled")); got != 1 { - t.Errorf("merge throttled outcomes = %f, want 1", got) - } -} - -// MergeInterval=0 disables the frequent merge pass entirely. -func TestRunMergeDisabled(t *testing.T) { - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db, cfg: config.Config{MergeInterval: 0}} - if err := d.runMerge(context.Background()); err != nil { - t.Fatalf("disabled runMerge() = %v, want nil", err) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("disabled runMerge should issue no SQL: %v", err) - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("merge", "disabled")); got != 1 { - t.Errorf("merge disabled outcomes = %f, want 1", got) - } -} - -func TestRunMaintenanceThrottle(t *testing.T) { - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db, cfg: config.Config{MaintenanceInterval: time.Hour}, lastMaintenance: time.Now()} - if err := d.runMaintenance(context.Background()); err != nil { - t.Fatalf("throttled runMaintenance() = %v, want nil", err) - } - // No expectations were registered: if it ran any SQL, this fails. - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("throttled runMaintenance should issue no SQL: %v", err) - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("maintenance", "throttled")); got != 1 { - t.Errorf("maintenance throttled outcomes = %f, want 1", got) - } -} - -func TestRunMaintenanceContinuesAfterCompactionFailure(t *testing.T) { - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db, cfg: config.Config{}} - - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_merge_adjacent_files('lake')")). - WillReturnError(errors.New("merge failed")) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_rewrite_data_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL 10 MINUTE)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_cleanup_old_files('lake', cleanup_all => true)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CHECKPOINT")). - WillReturnResult(sqlmock.NewResult(0, 0)) - - err = d.runMaintenance(context.Background()) - if err == nil { - t.Fatal("runMaintenance() error = nil, want merge error surfaced") - } - if lastOK, lastAt, lastErr := d.MaintenanceHealth(); lastErr == nil || !lastOK.IsZero() || lastAt.IsZero() { - t.Fatalf("MaintenanceHealth() = (%v, %v, %v), want zero lastOK, non-zero lastAt, non-nil lastErr", lastOK, lastAt, lastErr) - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("maintenance", "error")); got != 1 { - t.Errorf("maintenance error outcomes = %f, want 1", got) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet sql expectations: %v", err) - } -} - -// A failing rollup must not skip maintenance: file/snapshot growth is what -// makes the failing rollup heavier each retry, so compaction has to run anyway. -func TestRollupOnceRunsMaintenanceDespiteRollupFailure(t *testing.T) { - metrics.RollupComponentTotal.Reset() - metrics.DuckLakeOperationTotal.Reset() - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db, cfg: config.Config{}} - - // All rollup transactions fail at BeginTx. - mock.ExpectBegin().WillReturnError(errors.New("service tx failed")) - mock.ExpectBegin().WillReturnError(errors.New("endpoint tx failed")) - mock.ExpectBegin().WillReturnError(errors.New("edge tx failed")) - // Maintenance still runs: compaction calls + checkpoint (RetentionDays=0 - // skips the TTL deletes). - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_merge_adjacent_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_rewrite_data_files('lake')")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL 10 MINUTE)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CALL ducklake_cleanup_old_files('lake', cleanup_all => true)")). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectExec(regexp.QuoteMeta("CHECKPOINT")). - WillReturnResult(sqlmock.NewResult(0, 0)) - - _, err = d.rollupOnce(context.Background()) - if err == nil { - t.Fatal("rollupOnce() error = nil, want rollup errors surfaced") - } - if lastOK, lastAt, lastErr := d.MaintenanceHealth(); lastErr != nil || lastOK.IsZero() || lastAt.IsZero() { - t.Fatalf("MaintenanceHealth() = (%v, %v, %v), want non-zero lastOK and lastAt, nil lastErr", lastOK, lastAt, lastErr) - } - for _, component := range []string{"service", "endpoint", "edge"} { - if got := testutil.ToFloat64(metrics.RollupComponentTotal.WithLabelValues(component, "error")); got != 1 { - t.Errorf("%s rollup error outcomes = %f, want 1", component, got) - } - } - if got := testutil.ToFloat64(metrics.DuckLakeOperationTotal.WithLabelValues("maintenance", "success")); got != 1 { - t.Errorf("maintenance success outcomes = %f, want 1", got) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet sql expectations: %v", err) - } -} - func TestNewDuckUsesSingleConnectionPool(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -348,20 +131,20 @@ func TestNewDuckUsesSingleConnectionPool(t *testing.T) { DuckDBMemory: "128MB", } - d, err := NewDuck(ctx, cfg) + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) if err != nil { - if strings.Contains(err.Error(), "LOAD ducklake") || - strings.Contains(err.Error(), "LOAD sqlite") || - strings.Contains(err.Error(), "ATTACH") { - t.Skipf("DuckLake extensions unavailable: %v", err) - } t.Fatalf("NewDuck() error = %v", err) } defer d.Close() stats := d.DB.Stats() if stats.MaxOpenConnections != 1 { - t.Fatalf("MaxOpenConnections = %d, want 1 when DuckDBMaxConns is unset (floored for DuckLake serialization)", stats.MaxOpenConnections) + t.Fatalf("MaxOpenConnections = %d, want 1 when DuckDBMaxConns is unset", stats.MaxOpenConnections) } } diff --git a/internal/query/duck_wal_test.go b/internal/query/duck_wal_test.go deleted file mode 100644 index bfb76801..00000000 --- a/internal/query/duck_wal_test.go +++ /dev/null @@ -1,232 +0,0 @@ -package query - -import ( - "context" - "database/sql" - "path/filepath" - "strings" - "sync" - "testing" - "time" - - "github.com/labstack/fanout/internal/config" - "github.com/labstack/fanout/internal/lake/writegate" -) - -// catalogJournalMode opens the SQLite catalog at path with the modernc driver -// (registered via duck.go's blank import) and returns its journal_mode. -func catalogJournalMode(t *testing.T, path string) string { - t.Helper() - db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)") - if err != nil { - t.Fatalf("open catalog: %v", err) - } - defer db.Close() - var mode string - if err := db.QueryRow("PRAGMA journal_mode").Scan(&mode); err != nil { - t.Fatalf("read journal_mode: %v", err) - } - return mode -} - -// TestEnableCatalogWAL covers the catalog WAL bootstrap in isolation: it must -// set WAL, be idempotent, and recover (not permanently lock) when a populated -// -wal file from an unclean writer is still present. -func TestEnableCatalogWAL(t *testing.T) { - path := filepath.Join(t.TempDir(), "catalog.sqlite") - - if err := enableCatalogWAL(path); err != nil { - t.Fatalf("enableCatalogWAL: %v", err) - } - if mode := catalogJournalMode(t, path); !strings.EqualFold(mode, "wal") { - t.Fatalf("journal_mode = %q, want wal", mode) - } - - // Write through a connection and abandon it without checkpointing, leaving a - // populated -wal file — the rough shape a crashed/stuck writer leaves behind. - // Re-running the bootstrap must still succeed and report WAL. - db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)") - if err != nil { - t.Fatalf("open: %v", err) - } - defer db.Close() - if _, err := db.Exec("CREATE TABLE t(id INTEGER)"); err != nil { - t.Fatalf("create: %v", err) - } - if _, err := db.Exec("INSERT INTO t VALUES (1),(2),(3)"); err != nil { - t.Fatalf("insert: %v", err) - } - // Intentionally do NOT close db before re-running the bootstrap. - - if err := enableCatalogWAL(path); err != nil { - t.Fatalf("enableCatalogWAL (idempotent/recovery): %v", err) - } - if mode := catalogJournalMode(t, path); !strings.EqualFold(mode, "wal") { - t.Fatalf("journal_mode after reopen = %q, want wal", mode) - } -} - -// newTestDuck spins up a real DuckDB+DuckLake against a temp dir, skipping when -// the DuckLake/SQLite extensions aren't available in the environment. -func newTestDuck(t *testing.T, maxConns int) *Duck { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - cfg := config.Config{ - DataDir: t.TempDir(), - RollupInterval: time.Minute, - DuckDBMemory: "128MB", - DuckDBMaxConns: maxConns, - } - d, err := NewDuck(ctx, cfg) - if err != nil { - if strings.Contains(err.Error(), "LOAD ducklake") || - strings.Contains(err.Error(), "LOAD sqlite") || - strings.Contains(err.Error(), "ATTACH") { - t.Skipf("DuckLake extensions unavailable: %v", err) - } - t.Fatalf("NewDuck() error = %v", err) - } - t.Cleanup(func() { d.Close() }) - return d -} - -// TestNewDuckCatalogUsesWAL verifies NewDuck leaves the DuckLake catalog in WAL -// mode end-to-end. -func TestNewDuckCatalogUsesWAL(t *testing.T) { - d := newTestDuck(t, 1) - if mode := catalogJournalMode(t, d.cfg.TelemetryDuckLakePath()); !strings.EqualFold(mode, "wal") { - t.Fatalf("catalog journal_mode = %q, want wal", mode) - } -} - -// TestPoolConcurrentReadsNoLock exercises the WAL fix: with a multi-connection -// pool, concurrent read queries running alongside serialized writes must not -// fail with "database is locked". A start barrier releases all goroutines at -// once so the pool is forced to open multiple physical connections (otherwise a -// low-contention run could funnel through a single reused handle and never -// exercise concurrency). The temp_directory fix is covered separately by -// TestTempDirectorySetOnce. -func TestPoolConcurrentReadsNoLock(t *testing.T) { - d := newTestDuck(t, 4) - ctx := context.Background() - - if _, err := d.DB.ExecContext(ctx, "CREATE TABLE lake.concurrency_probe(id INTEGER, val VARCHAR)"); err != nil { - t.Fatalf("create table: %v", err) - } - - const workers = 8 - const iters = 25 - var wg sync.WaitGroup - errCh := make(chan error, workers*iters) - start := make(chan struct{}) // released once all goroutines are parked - - // Writers: serialized via the shared write gate, exactly as the ingest path - // holds it around appender flushes. - for w := 0; w < workers/2; w++ { - wg.Add(1) - go func(w int) { - defer wg.Done() - <-start - for i := 0; i < iters; i++ { - err := func() error { - defer d.WriteGate().Lock(writegate.WriteIngestSpans)() - _, err := d.DB.ExecContext(ctx, - "INSERT INTO lake.concurrency_probe VALUES (?, ?)", w*iters+i, "x") - return err - }() - if err != nil { - errCh <- err - return - } - } - }(w) - } - // Readers: concurrent, no write gate — must not collide with the writers. - for r := 0; r < workers/2; r++ { - wg.Add(1) - go func() { - defer wg.Done() - <-start - for i := 0; i < iters; i++ { - var n int - if err := d.DB.QueryRowContext(ctx, - "SELECT count(*) FROM lake.concurrency_probe").Scan(&n); err != nil { - errCh <- err - return - } - } - }() - } - close(start) - wg.Wait() - close(errCh) - - for err := range errCh { - msg := strings.ToLower(err.Error()) - if strings.Contains(msg, "database is locked") || - strings.Contains(msg, "switch temporary directory") { - t.Fatalf("regression — pool>1 catalog/temp error: %v", err) - } - t.Fatalf("unexpected error: %v", err) - } -} - -// TestTempDirectorySetOnce deterministically reproduces the "Cannot switch -// temporary directory" regression. temp_directory is an instance-global setting: -// once any connection has spilled to it, re-running SET temp_directory on a later -// connection's boot hook fails. The test forces a spill on a pinned connection, -// then opens a second pooled connection (which runs the boot hook) and asserts it -// doesn't fail. Reverting the sync.Once in openDuckDB makes this test fail. -func TestTempDirectorySetOnce(t *testing.T) { - ctx := context.Background() - // A small memory limit makes a modest sort spill to the temp directory. - // Threads pinned low to keep the per-thread pinned working set inside the - // tiny cap — with one worker per core (the default) the sort OOMs on - // many-core machines before it can spill. - cfg := config.Config{ - DataDir: t.TempDir(), - RollupInterval: time.Minute, - DuckDBMemory: "64MB", - DuckDBThreads: 2, - DuckDBMaxConns: 4, - } - d, err := NewDuck(ctx, cfg) - if err != nil { - if strings.Contains(err.Error(), "LOAD ducklake") || - strings.Contains(err.Error(), "LOAD sqlite") || - strings.Contains(err.Error(), "ATTACH") { - t.Skipf("DuckLake extensions unavailable: %v", err) - } - t.Fatalf("NewDuck() error = %v", err) - } - defer d.Close() - - // Pin connection 1 and force it to spill to the instance temp directory. - conn1, err := d.DB.Conn(ctx) - if err != nil { - t.Fatalf("conn1: %v", err) - } - defer conn1.Close() - var sink int64 - if err := conn1.QueryRowContext(ctx, - "SELECT count(*) FROM (SELECT i FROM range(20000000) t(i) ORDER BY i DESC)").Scan(&sink); err != nil { - t.Fatalf("forced-spill query: %v", err) - } - - // With conn1 still held, this must open a second physical connection and run - // its boot hook. Pre-fix, that boot re-ran SET temp_directory after the temp - // dir was in use and failed with "Cannot switch temporary directory". - conn2, err := d.DB.Conn(ctx) - if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "switch temporary directory") { - t.Fatalf("regression — temp_directory re-set on 2nd connection: %v", err) - } - t.Fatalf("conn2 (boots a fresh pooled connection): %v", err) - } - defer conn2.Close() - if err := conn2.QueryRowContext(ctx, "SELECT 1").Scan(&sink); err != nil { - t.Fatalf("conn2 query: %v", err) - } -} diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index a4834cba..feca7f1b 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -2,12 +2,13 @@ package query import ( "context" - "strings" "testing" "time" "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/metrics" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -144,22 +145,25 @@ FROM lake.spans`).Scan(&spanBuckets); err != nil { func TestSkipRollupToLatest(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - d, err := NewDuck(ctx, config.Config{DataDir: t.TempDir(), DuckDBMemory: "2GB", RetentionDays: 30}) + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "2GB", RetentionDays: 30, HotRetention: 24 * time.Hour} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + now := time.Now().UnixNano() + spans := make([]telemetry.Span, 100) + for i := range spans { + spans[i] = telemetry.Span{Namespace: "default", TraceID: "backlog", SpanID: string(rune(i + 1)), ServiceName: "svc", Kind: "SPAN_KIND_CLIENT", StartUnixNanos: now - int64(i)*int64(time.Minute), DurationMS: 10, StatusCode: "STATUS_CODE_OK", IngestedAt: now} + } + if err := repository.Commit(telemetrystore.Batch{ID: "skip-backlog", Spans: spans}); err != nil { + t.Fatalf("commit backlog: %v", err) + } + d, err := NewDuck(ctx, cfg, repository) if err != nil { - if strings.Contains(err.Error(), "ducklake") || strings.Contains(err.Error(), "ATTACH") { - t.Skipf("DuckLake unavailable: %v", err) - } t.Fatalf("NewDuck: %v", err) } defer d.Close() - - if _, err := d.DB.ExecContext(ctx, ` -INSERT INTO lake.spans (namespace, trace_id, span_id, parent_span_id, service, kind, start_time, duration_ms, status, ingested_unix_nano) -SELECT 'default', 'tr-'||i, 'c-'||i, 'p-'||i, 'svc-'||(i%5), 'SPAN_KIND_CLIENT', - now() - ((i % 120) * INTERVAL 1 MINUTE), 10.0, 'STATUS_CODE_OK', epoch_ns(now()) -FROM range(5000) t(i)`); err != nil { - t.Fatalf("insert: %v", err) - } if _, err := d.DB.ExecContext(ctx, ` INSERT INTO endpoint_rollup ( namespace, bucket, service, method, path, calls, error_count, duration_count, duration_buckets @@ -227,12 +231,9 @@ ON CONFLICT (cache_key) DO UPDATE SET last_ingested_unix_nano = 1, updated_at = // Insert one fresh live span (ingested_unix_nano = now) and verify that // rollupOnce picks it up — proves the watermark didn't over-advance and // swallow data that arrived after the skip. - if _, err := d.DB.ExecContext(ctx, ` -INSERT INTO lake.spans (namespace, trace_id, span_id, parent_span_id, service, kind, - start_time, duration_ms, status, ingested_unix_nano) -VALUES ('default', 'tr-live-1', 'sp-live-1', '', 'svc-live', 'SPAN_KIND_SERVER', - now(), 5.0, 'STATUS_CODE_OK', epoch_ns(now()))`); err != nil { - t.Fatalf("insert live span: %v", err) + liveTime := time.Now().Add(time.Millisecond).UnixNano() + if err := repository.Commit(telemetrystore.Batch{ID: "skip-live", Spans: []telemetry.Span{{Namespace: "default", TraceID: "tr-live-1", SpanID: "sp-live-1", ServiceName: "svc-live", Kind: "SPAN_KIND_SERVER", StartUnixNanos: liveTime, DurationMS: 5, StatusCode: "STATUS_CODE_OK", IngestedAt: liveTime}}}); err != nil { + t.Fatalf("commit live span: %v", err) } n2, err := d.rollupOnce(ctx) diff --git a/internal/query/hourprune_experiment_test.go b/internal/query/hourprune_experiment_test.go deleted file mode 100644 index be93e7e9..00000000 --- a/internal/query/hourprune_experiment_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package query - -import ( - "context" - "regexp" - "strconv" - "strings" - "testing" - - "github.com/labstack/fanout/internal/config" -) - -// TestHourPartitionPrunesRecentWindow is the within-day-pruning gate experiment -// (design 2026-06-19). It writes a multi-hour span dataset into the real -// hour-partitioned lake, then proves a recent-window query -// (start_time >= now() - 15min) scans only ~the current hour's rows instead of -// the whole dataset — the failure that made the rollup hit 35s as a UTC day -// filled. Run with: go test ./internal/query/ -run HourPartitionPrunes -v -func TestHourPartitionPrunesRecentWindow(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - d, err := NewDuck(ctx, config.Config{DataDir: t.TempDir(), DuckDBMemory: "512MB"}) - if err != nil { - if strings.Contains(err.Error(), "ducklake") || strings.Contains(err.Error(), "ATTACH") { - t.Skipf("DuckLake unavailable: %v", err) - } - t.Fatalf("NewDuck: %v", err) - } - defer d.Close() - - // Three hours of data, 200k rows each (600k total), each hour in its own - // hour-partition by start_time. ingested_unix_nano is "now" for all (as if - // just ingested) so it can't help pruning — only start_time partitioning can. - const perHour = 200_000 - for _, agoMin := range []int{150, 90, 5} { // 2.5h ago, 1.5h ago, 5min ago - _, err := d.DB.ExecContext(ctx, ` -INSERT INTO lake.spans (namespace, service, trace_id, span_id, start_time, status, status_message, ingested_unix_nano) -SELECT 'default', 'svc-' || (i % 50), md5(i::VARCHAR), md5((i+1)::VARCHAR), - now() - INTERVAL `+strconv.Itoa(agoMin)+` MINUTE + (i % 60) * INTERVAL 1 SECOND, - CASE WHEN i % 20 = 0 THEN 'STATUS_CODE_ERROR' ELSE 'STATUS_CODE_OK' END, - 'err', epoch_ns(now()) -FROM range(`+strconv.Itoa(perHour)+`) t(i)`) - if err != nil { - t.Fatalf("insert hour -%dmin: %v", agoMin, err) - } - } - if _, err := d.DB.ExecContext(ctx, "CALL ducklake_merge_adjacent_files('lake')"); err != nil { - t.Fatalf("merge: %v", err) - } - - var total int64 - if err := d.DB.QueryRowContext(ctx, "SELECT count(*) FROM spans").Scan(&total); err != nil { - t.Fatalf("count total: %v", err) - } - - var fileCount int64 - _ = d.DB.QueryRowContext(ctx, - `SELECT file_count FROM ducklake_table_info('lake') WHERE table_name = 'spans'`).Scan(&fileCount) - - // EXPLAIN ANALYZE the recent-window query (same shape as overviewRecentErrors). - plan := explainAnalyze(t, ctx, d, ` -SELECT service, count(*) FROM spans -WHERE start_time >= now() - INTERVAL 15 MINUTE - AND status = 'STATUS_CODE_ERROR' -GROUP BY service`) - scanned := maxScanCardinality(plan) - - t.Logf("── within-day pruning experiment (hour-partitioned) ──") - t.Logf("total rows : %d (across 3 hour-partitions)", total) - t.Logf("parquet files (merged): %d", fileCount) - t.Logf("recent-15min query scanned: %d rows", scanned) - t.Logf("→ pruned to %.1f%% of the dataset (lower is better; ~1/3 = one hour)", 100*float64(scanned)/float64(total)) - - // The recent-window query must scan well under half the dataset — i.e. it - // pruned to ~the current hour, not all three. (No pruning would scan ~total.) - if scanned >= total/2 { - t.Fatalf("recent-window query scanned %d of %d rows — within-day pruning is NOT working", scanned, total) - } -} - -func explainAnalyze(t *testing.T, ctx context.Context, d *Duck, q string) string { - t.Helper() - rows, err := d.DB.QueryContext(ctx, "EXPLAIN ANALYZE "+q) - if err != nil { - t.Fatalf("explain analyze: %v", err) - } - defer rows.Close() - var sb strings.Builder - for rows.Next() { - var k, v string - if err := rows.Scan(&k, &v); err != nil { - t.Fatalf("scan plan: %v", err) - } - sb.WriteString(v) - sb.WriteString("\n") - } - return sb.String() -} - -// maxScanCardinality pulls the largest actual-rows count from an EXPLAIN ANALYZE -// plan — for a single-table scan+aggregate that is the rows the scan produced. -var cardRe = regexp.MustCompile(`(?i)(\d[\d,]*)\s*Rows`) - -func maxScanCardinality(plan string) int64 { - var max int64 - for _, m := range cardRe.FindAllStringSubmatch(plan, -1) { - n, err := strconv.ParseInt(strings.ReplaceAll(m[1], ",", ""), 10, 64) - if err == nil && n > max { - max = n - } - } - return max -} diff --git a/internal/query/retry_test.go b/internal/query/retry_test.go deleted file mode 100644 index 0247d0bc..00000000 --- a/internal/query/retry_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package query - -import ( - "context" - "errors" - "regexp" - "testing" - - "github.com/DATA-DOG/go-sqlmock" -) - -// transient is the canonical transient DuckLake IO error string that -// isTransientLakeIOError must classify as retryable. -const transientMsg = "IO Error: Cannot open file: No such file or directory" - -// ---- isTransientLakeIOError ---- - -func TestIsTransientLakeIOError(t *testing.T) { - cases := []struct { - name string - err error - want bool - }{ - { - name: "nil error", - err: nil, - want: false, - }, - { - name: "context.Canceled", - err: context.Canceled, - want: false, - }, - { - name: "generic error", - err: errors.New("syntax error"), - want: false, - }, - { - name: "IO Error only (missing No such file)", - err: errors.New("IO Error: something else"), - want: false, - }, - { - name: "No such file only (missing IO Error)", - err: errors.New("something: No such file or directory"), - want: false, - }, - { - name: "both substrings present", - err: errors.New(transientMsg), - want: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := isTransientLakeIOError(tc.err) - if got != tc.want { - t.Errorf("isTransientLakeIOError(%v) = %v, want %v", tc.err, got, tc.want) - } - }) - } -} - -// ---- QueryContext retry tests ---- - -func TestQueryContextRetriesThenSucceeds(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db} - ctx := context.Background() - - // First call: transient error → should retry. - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnError(errors.New(transientMsg)) - // Second call: success. - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnRows(sqlmock.NewRows([]string{"x"}).AddRow(1)) - - rows, err := d.QueryContext(ctx, "SELECT 1") - if err != nil { - t.Fatalf("QueryContext() error = %v, want nil", err) - } - if rows == nil { - t.Fatal("QueryContext() rows = nil, want non-nil") - } - rows.Close() - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations (retry did not happen): %v", err) - } -} - -func TestQueryContextNonTransientReturnsImmediately(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db} - ctx := context.Background() - - // Non-transient error — exactly one call, no retry. - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnError(errors.New("syntax error")) - - _, queryErr := d.QueryContext(ctx, "SELECT 1") - if queryErr == nil { - t.Fatal("QueryContext() error = nil, want syntax error") - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unexpected extra call (retry happened on non-transient error): %v", err) - } -} - -func TestQueryContextExhaustsRetries(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db} - ctx := context.Background() - - // All 3 attempts fail with transient error. - for i := 0; i < 3; i++ { - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnError(errors.New(transientMsg)) - } - - _, queryErr := d.QueryContext(ctx, "SELECT 1") - if queryErr == nil { - t.Fatal("QueryContext() error = nil after exhausted retries, want transient error") - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unexpected call count (expected exactly 3 attempts): %v", err) - } -} - -// ---- QueryRowScan retry test ---- - -func TestQueryRowScanRetriesThenSucceeds(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - d := &Duck{DB: db} - ctx := context.Background() - - // First call: transient error (sqlmock surfaces it at Query time, which the - // wrapper classifies and retries — same observable outcome as an error at Scan). - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnError(errors.New(transientMsg)) - // Second call: success with one row. - mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")). - WillReturnRows(sqlmock.NewRows([]string{"v"}).AddRow(42)) - - var dst int - if err := d.QueryRowScan(ctx, []any{&dst}, "SELECT 1"); err != nil { - t.Fatalf("QueryRowScan() error = %v, want nil", err) - } - if dst != 42 { - t.Errorf("QueryRowScan() scanned %d, want 42", dst) - } - - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("unmet expectations (retry did not happen): %v", err) - } -} - -// ---- applyMemoryHeadroom tests ---- - -func TestApplyMemoryHeadroomCapsSmallBox(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - ctx := context.Background() - - // 6.4 GiB ≈ 80% of 8 GiB → total≈8 GiB, capped = 8 GiB - 2 GiB = 6 GiB < 6.4 GiB → SET fires. - mock.ExpectQuery(regexp.QuoteMeta("SELECT current_setting('memory_limit')")). - WillReturnRows(sqlmock.NewRows([]string{"v"}).AddRow("6.4 GiB")) - mock.ExpectExec("SET memory_limit="). - WillReturnResult(sqlmock.NewResult(0, 0)) - - if err := applyMemoryHeadroom(ctx, db); err != nil { - t.Fatalf("applyMemoryHeadroom() error = %v, want nil", err) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("SET memory_limit was not issued for small box: %v", err) - } -} - -func TestApplyMemoryHeadroomSkipsBigBox(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("sqlmock.New: %v", err) - } - defer db.Close() - - ctx := context.Background() - - // 100 GiB → total≈125 GiB, capped=123 GiB >= 100 GiB limit → no-op (capped >= limit). - mock.ExpectQuery(regexp.QuoteMeta("SELECT current_setting('memory_limit')")). - WillReturnRows(sqlmock.NewRows([]string{"v"}).AddRow("100 GiB")) - // No ExpectExec: if a SET fires, sqlmock reports it as an unexpected call - // and ExpectationsWereMet fails. - - if err := applyMemoryHeadroom(ctx, db); err != nil { - t.Fatalf("applyMemoryHeadroom() error = %v, want nil", err) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatalf("SET memory_limit was unexpectedly issued for big box: %v", err) - } -} diff --git a/internal/query/schema.go b/internal/query/schema.go index 51051e7f..c28ec49f 100644 --- a/internal/query/schema.go +++ b/internal/query/schema.go @@ -10,8 +10,9 @@ func GetSchema(dataDir string) string { const schemaTemplate = ` ## Fanout Data Schema -Fanout stores telemetry in DuckLake tables attached under the lake catalog. -The local metadata catalog, query cache, and product state live under {DATA_DIR}. +Fanout stores telemetry in indexed hot segments and open Parquet files. DuckDB +exposes the Parquet files through the read-only lake schema. The rebuildable +query cache and product state live under {DATA_DIR}. Primary query surfaces: - spans view: clean span columns for most queries @@ -22,7 +23,7 @@ Primary query surfaces: - endpoint_rollup table: minute endpoint counts, errors, and mergeable latency histograms ### 1. Spans -Base table: lake.spans +Parquet relation: lake.spans Preferred query surface: spans Important columns: diff --git a/internal/query/sql.go b/internal/query/sql.go index 127a0969..8ad94d20 100644 --- a/internal/query/sql.go +++ b/internal/query/sql.go @@ -51,12 +51,9 @@ func (d *Duck) ExecuteSQL(ctx context.Context, req SQLRequest) (resp SQLResponse req.MaxRows = 1000 } - // Set default timeout, then clamp to a hard ceiling. A query that outlives - // the DuckLake snapshot grace window can have its parquet files deleted - // mid-scan by maintenance (see snapshotGraceMinutes in duck.go), so a - // caller-supplied TimeoutMs must never exceed it. Half the grace leaves - // ample margin for the longest legitimate scan. - const maxQueryTimeoutMs = snapshotGraceMinutes * 60 * 1000 / 2 // half the snapshot grace + // Set a default timeout and clamp arbitrary SQL so one request cannot occupy + // a DuckDB worker indefinitely. + const maxQueryTimeoutMs = 5 * 60 * 1000 timeoutMs := req.TimeoutMs if timeoutMs <= 0 { timeoutMs = 30000 @@ -86,12 +83,8 @@ func (d *Duck) ExecuteSQL(ctx context.Context, req SQLRequest) (resp SQLResponse queryCtx, cancel := context.WithTimeout(ctx, time.Duration(timeoutMs)*time.Millisecond) defer cancel() - // Execute query. The raw `query` tool deliberately uses DB.QueryContext, not - // the d.QueryContext retry wrapper: arbitrary user SQL may stream large/long - // results, and a blind re-plan on a transient lake IO error is less obviously - // safe than for the fixed internal aggregate reads. This path is instead - // bounded by the timeout clamp above (well under the snapshot grace). - rows, err := d.DB.QueryContext(queryCtx, execQuery) + // Execute the query against immutable Parquet files and the local cache. + rows, err := d.QueryContext(queryCtx, execQuery) if err != nil { return SQLResponse{ Error: fmt.Sprintf("Query execution failed: %v", err), diff --git a/internal/query/views.go b/internal/query/views.go index 6ea2e9de..ff3badcb 100644 --- a/internal/query/views.go +++ b/internal/query/views.go @@ -3,6 +3,7 @@ package query import ( "database/sql" "fmt" + "path/filepath" ) const createSpansTable = ` @@ -250,15 +251,24 @@ const macroAttr = ` CREATE OR REPLACE MACRO attr(json_col, key) AS json_extract_string(json_col, '$."' || key || '"');` +// CreateTables creates mutable telemetry tables for query-kernel tests and +// benchmarks. Production startup never calls this function: telemetry is +// exposed exclusively through CreateParquetViews. func CreateTables(db *sql.DB) error { + if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS lake`); err != nil { + return fmt.Errorf("create lake schema: %w", err) + } for _, stmt := range []string{createSpansTable, createLogsTable, createMetricsTable} { if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("create table: %w", err) } } - if err := configureDuckLake(db); err != nil { - return err - } + return CreateCacheTables(db) +} + +// CreateCacheTables creates only DuckDB's rebuildable query accelerators. The +// production telemetry rows themselves live in immutable segments and Parquet. +func CreateCacheTables(db *sql.DB) error { if err := ensureCacheTable(db, "service_rollup", createServiceRollupTable, "namespace", "bucket", "service", "spans", "p50_ms", "p95_ms", "error_rate", "log_count", "metric_count"); err != nil { return err @@ -278,45 +288,27 @@ func CreateTables(db *sql.DB) error { return nil } -// CreateViews creates the clean-name views over DuckLake tables plus the attr() macro. -func CreateViews(db *sql.DB) error { - for _, stmt := range []string{macroAttr, viewSpans, viewLogs, viewMetrics} { +// CreateParquetViews exposes the repository's open Parquet files under the +// canonical lake schema used by Fanout's SQL kernel. +func CreateParquetViews(db *sql.DB, parquetDir string) error { + if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS lake`); err != nil { + return err + } + for _, signal := range []string{"spans", "logs", "metrics"} { + pattern := filepath.ToSlash(filepath.Join(parquetDir, signal, "*.parquet")) + stmt := fmt.Sprintf(`CREATE OR REPLACE VIEW lake.%s AS SELECT * FROM read_parquet(%s, union_by_name=true)`, signal, sqlLiteral(pattern)) if _, err := db.Exec(stmt); err != nil { - return fmt.Errorf("create view/macro: %w", err) + return fmt.Errorf("create parquet view lake.%s: %w", signal, err) } } return nil } -func configureDuckLake(db *sql.DB) error { - var loaded int - if err := db.QueryRow(` -SELECT count(*) -FROM duckdb_extensions() -WHERE extension_name = 'ducklake' AND loaded`).Scan(&loaded); err != nil { - return fmt.Errorf("check ducklake extension: %w", err) - } - if loaded == 0 { - return nil - } - - stmts := []string{ - `CALL lake.set_option('parquet_compression', 'zstd')`, - `CALL lake.set_option('target_file_size', '256MB')`, - // Partition by HOUR (not day) of the event time so recent-window scans - // (Overview error queries, rollup aggregations) prune by per-file - // start_time zonemaps to ~the current 1-2 hours' files instead of the whole - // day. day-partitioning let merge produce day-spanning files whose zonemaps - // couldn't prune within a day — the rollup hit 35s and query p95 5s as a UTC - // day filled. hour() is DuckLake's date-inclusive, order-preserving hour - // transform (hours since epoch), so it uniquely identifies a calendar hour. - `ALTER TABLE lake.spans SET PARTITIONED BY (namespace, hour(start_time))`, - `ALTER TABLE lake.logs SET PARTITIONED BY (namespace, hour(log_time))`, - `ALTER TABLE lake.metrics SET PARTITIONED BY (namespace, hour(metric_time))`, - } - for _, stmt := range stmts { +// CreateViews creates stable clean-name views plus the attr() macro. +func CreateViews(db *sql.DB) error { + for _, stmt := range []string{macroAttr, viewSpans, viewLogs, viewMetrics} { if _, err := db.Exec(stmt); err != nil { - return fmt.Errorf("configure ducklake: %w", err) + return fmt.Errorf("create view/macro: %w", err) } } return nil diff --git a/internal/query/views_test.go b/internal/query/views_test.go index dbf64e02..454ad55e 100644 --- a/internal/query/views_test.go +++ b/internal/query/views_test.go @@ -14,9 +14,6 @@ func openTestDuck(t *testing.T) *sql.DB { if err != nil { t.Fatalf("open duckdb: %v", err) } - if _, err := db.Exec(`ATTACH ':memory:' AS lake`); err != nil { - t.Fatalf("attach lake catalog: %v", err) - } t.Cleanup(func() { _ = db.Close() }) return db } diff --git a/internal/lake/writegate/write_gate.go b/internal/query/writegate/write_gate.go similarity index 78% rename from internal/lake/writegate/write_gate.go rename to internal/query/writegate/write_gate.go index a7a5628d..94f797b5 100644 --- a/internal/lake/writegate/write_gate.go +++ b/internal/query/writegate/write_gate.go @@ -1,5 +1,5 @@ -// Package writegate serializes and measures DuckLake catalog writes shared by -// the query kernel and telemetry writer. +// Package writegate serializes and measures writes to DuckDB's rebuildable +// rollup cache. package writegate import ( @@ -9,20 +9,16 @@ import ( "github.com/labstack/fanout/internal/metrics" ) -// WriteOperation is a bounded metric label for a DuckLake catalog write. +// WriteOperation is a bounded metric label for a rollup-cache write. // Keep this list exhaustive: arbitrary strings would create an unbounded // Prometheus label surface. type WriteOperation string const ( - WriteIngestSpans WriteOperation = "ingest_spans" - WriteIngestLogs WriteOperation = "ingest_logs" - WriteIngestMetrics WriteOperation = "ingest_metrics" WriteRollupSkip WriteOperation = "rollup_skip_to_latest" WriteRollupService WriteOperation = "rollup_service" WriteRollupEndpoint WriteOperation = "rollup_endpoint" WriteRollupEdge WriteOperation = "rollup_edge" - WriteMerge WriteOperation = "merge" WriteMaintenance WriteOperation = "maintenance" ) @@ -39,7 +35,7 @@ type WriteGate struct { // cannot tell the two orderings apart. var observe = metrics.RecordWriteGate -// Lock acquires the catalog write gate and returns its release function. +// Lock acquires the cache write gate and returns its release function. // Callers must defer the returned function before acquiring a database // connection, transaction, or appender, and must call it exactly once. func (g *WriteGate) Lock(operation WriteOperation) func() { diff --git a/internal/lake/writegate/write_gate_test.go b/internal/query/writegate/write_gate_test.go similarity index 90% rename from internal/lake/writegate/write_gate_test.go rename to internal/query/writegate/write_gate_test.go index 88af1957..a19f5c03 100644 --- a/internal/lake/writegate/write_gate_test.go +++ b/internal/query/writegate/write_gate_test.go @@ -1,5 +1,8 @@ package writegate +// These tests cover the query-cache gate; telemetry commits use their own +// repository lock and never pass through DuckDB. + import ( "testing" "time" @@ -20,7 +23,7 @@ func TestWriteGateSerializesHoldersInAcquisitionOrder(t *testing.T) { go func() { defer func() { done <- struct{}{} }() - unlock := gate.Lock(WriteMerge) + unlock := gate.Lock(WriteMaintenance) close(firstEntered) <-releaseFirst unlock() @@ -63,13 +66,13 @@ func TestWriteGateReleasesAfterPanic(t *testing.T) { t.Error("expected panic to propagate") } }() - defer gate.Lock(WriteMerge)() + defer gate.Lock(WriteMaintenance)() panic("boom") }() acquired := make(chan struct{}) go func() { - defer gate.Lock(WriteMerge)() + defer gate.Lock(WriteMaintenance)() close(acquired) }() select { @@ -96,7 +99,7 @@ func TestWriteGateObservesOutsideTheCriticalSection(t *testing.T) { }) defer restore() - gate.Lock(WriteMerge)() + gate.Lock(WriteMaintenance)() if !freeDuringObserve { t.Fatal("gate was still held while the observation ran — move the Unlock above observe()") @@ -107,16 +110,16 @@ func TestWriteGateObservesOutsideTheCriticalSection(t *testing.T) { // whether ingest is stalling behind rollups, which is the question this // instrumentation exists to answer. func TestWriteGateRecordsBothWaitAndHoldHistograms(t *testing.T) { - beforeWait := histogramCount(t, "fanout_write_gate_wait_seconds", WriteMerge) - beforeHold := histogramCount(t, "fanout_write_gate_hold_seconds", WriteMerge) + beforeWait := histogramCount(t, "fanout_write_gate_wait_seconds", WriteMaintenance) + beforeHold := histogramCount(t, "fanout_write_gate_hold_seconds", WriteMaintenance) var gate WriteGate - gate.Lock(WriteMerge)() + gate.Lock(WriteMaintenance)() - if after := histogramCount(t, "fanout_write_gate_wait_seconds", WriteMerge); after != beforeWait+1 { + if after := histogramCount(t, "fanout_write_gate_wait_seconds", WriteMaintenance); after != beforeWait+1 { t.Errorf("wait sample count = %d, want %d", after, beforeWait+1) } - if after := histogramCount(t, "fanout_write_gate_hold_seconds", WriteMerge); after != beforeHold+1 { + if after := histogramCount(t, "fanout_write_gate_hold_seconds", WriteMaintenance); after != beforeHold+1 { t.Errorf("hold sample count = %d, want %d", after, beforeHold+1) } } diff --git a/internal/storagebench/data.go b/internal/storagebench/data.go new file mode 100644 index 00000000..b33def1e --- /dev/null +++ b/internal/storagebench/data.go @@ -0,0 +1,45 @@ +package storagebench + +import ( + "fmt" + "time" + + "github.com/labstack/fanout/internal/telemetry/segment" +) + +const DayNanos = int64(24 * time.Hour) + +func Rows(offset, count, total int, base int64) []segment.Span { + rows := make([]segment.Span, count) + methods := [...]string{"GET", "POST", "PUT", "DELETE"} + for j := range rows { + i := offset + j + service := fmt.Sprintf("service-%02d", i%50) + route := fmt.Sprintf("/api/v1/resource/%02d", i%20) + statusCode, statusMessage := "OK", "" + exceptionType, exceptionMessage := "", "" + if i%20 == 0 { + statusCode, statusMessage = "ERROR", "upstream request failed" + exceptionType, exceptionMessage = "TimeoutError", "deadline exceeded while calling dependency" + } + start := base + int64(i)*DayNanos/int64(total) + duration := float64(1+(i%5000)) / 10 + rows[j] = segment.Span{ + Namespace: "default", TraceID: TraceID(uint64(i / 5)), SpanID: fmt.Sprintf("%016x", i), + ParentSpanID: fmt.Sprintf("%016x", max(i-1, 0)), ServiceName: service, + Name: methods[i%len(methods)] + " " + route, Kind: "SERVER", + StartUnixNanos: start, EndUnixNanos: start + int64(duration*float64(time.Millisecond)), DurationMS: duration, + StatusCode: statusCode, StatusMsg: statusMessage, + ResourceJSON: []byte(fmt.Sprintf(`{"service.name":"%s","host.name":"node-%02d"}`, service, i%16)), + AttributesJSON: []byte(fmt.Sprintf(`{"tenant":"tenant-%03d","region":"us-west-2","http.request.method":"%s"}`, i%200, methods[i%len(methods)])), + EventsJSON: []byte(`[]`), LinksJSON: []byte(`[]`), TraceState: "vendor=opaque", Flags: 1, + ScopeName: "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp", ScopeVersion: "0.63.0", IngestedAt: start + int64(time.Second), + HTTPMethod: methods[i%len(methods)], HTTPStatusCode: fmt.Sprintf("%d", 200+(i%5)), HTTPRoute: route, + PeerService: fmt.Sprintf("dependency-%02d", i%10), ServiceVersion: "2026.8.0", DeploymentEnv: "production", + ExceptionType: exceptionType, ExceptionMessage: exceptionMessage, + } + } + return rows +} + +func TraceID(value uint64) string { return fmt.Sprintf("%016x%016x", value*0x9e3779b97f4a7c15, value) } diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go new file mode 100644 index 00000000..81d368ad --- /dev/null +++ b/internal/telemetry/parquet.go @@ -0,0 +1,330 @@ +package telemetry + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/compress" + "github.com/apache/arrow-go/v18/parquet/pqarrow" +) + +type parquetColumn[T any] struct { + name string + typeInfo arrow.DataType + nullable bool + value func(T) any +} + +type ParquetStore struct { + dir string +} + +type ParquetStats struct { + Files int + Bytes int64 +} + +func OpenParquetStore(dir string) (*ParquetStore, error) { + for _, signal := range []string{"spans", "logs", "metrics"} { + if err := os.MkdirAll(filepath.Join(dir, signal), 0o755); err != nil { + return nil, fmt.Errorf("create parquet %s directory: %w", signal, err) + } + } + store := &ParquetStore{dir: dir} + if err := writeParquet(filepath.Join(dir, "spans", "_schema.parquet"), spanParquetColumns(), []Span{}); err != nil { + return nil, fmt.Errorf("create span parquet schema: %w", err) + } + if err := writeParquet(filepath.Join(dir, "logs", "_schema.parquet"), logParquetColumns(), []Log{}); err != nil { + return nil, fmt.Errorf("create log parquet schema: %w", err) + } + if err := writeParquet(filepath.Join(dir, "metrics", "_schema.parquet"), metricParquetColumns(), []Metric{}); err != nil { + return nil, fmt.Errorf("create metric parquet schema: %w", err) + } + return store, nil +} + +func (p *ParquetStore) Dir() string { return p.dir } + +// Stats reports the current immutable-file footprint for each signal. +func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { + stats := make(map[string]ParquetStats, 3) + for _, signal := range []string{"spans", "logs", "metrics"} { + entries, err := os.ReadDir(filepath.Join(p.dir, signal)) + if err != nil { + return nil, err + } + var signalStats ParquetStats + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".parquet" || entry.Name() == "_schema.parquet" { + continue + } + info, err := entry.Info() + if err != nil { + return nil, err + } + signalStats.Files++ + signalStats.Bytes += info.Size() + } + stats[signal] = signalStats + } + return stats, nil +} + +func (p *ParquetStore) WriteSpans(id string, rows []Span) error { + if len(rows) == 0 { + return nil + } + return writeParquet(filepath.Join(p.dir, "spans", id+".parquet"), spanParquetColumns(), rows) +} + +func (p *ParquetStore) WriteLogs(id string, rows []Log) error { + if len(rows) == 0 { + return nil + } + return writeParquet(filepath.Join(p.dir, "logs", id+".parquet"), logParquetColumns(), rows) +} + +func (p *ParquetStore) WriteMetrics(id string, rows []Metric) error { + if len(rows) == 0 { + return nil + } + return writeParquet(filepath.Join(p.dir, "metrics", id+".parquet"), metricParquetColumns(), rows) +} + +func writeParquet[T any](path string, columns []parquetColumn[T], rows []T) error { + if _, err := os.Stat(path); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + fields := make([]arrow.Field, len(columns)) + for i, column := range columns { + fields[i] = arrow.Field{Name: column.name, Type: column.typeInfo, Nullable: column.nullable} + } + schema := arrow.NewSchema(fields, nil) + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + for _, row := range rows { + for i, column := range columns { + if err := appendArrowValue(builder.Field(i), column.value(row)); err != nil { + return fmt.Errorf("append parquet column %s: %w", column.name, err) + } + } + } + record := builder.NewRecordBatch() + defer record.Release() + + tmp := path + ".tmp" + _ = os.Remove(tmp) + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return err + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(tmp) + } + }() + writer, err := pqarrow.NewFileWriter( + schema, + f, + parquet.NewWriterProperties(parquet.WithCompression(compress.Codecs.Zstd)), + pqarrow.NewArrowWriterProperties(pqarrow.WithStoreSchema()), + ) + if err != nil { + return err + } + if err := writer.Write(record); err != nil { + _ = writer.Close() + return err + } + if err := writer.Close(); err != nil { + return err + } + _ = f.Close() // pqarrow may already have closed the sink. + syncFile, err := os.OpenFile(tmp, os.O_RDWR, 0) + if err != nil { + return err + } + if err := syncFile.Sync(); err != nil { + _ = syncFile.Close() + return err + } + if err := syncFile.Close(); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + if err := syncDirectory(filepath.Dir(path)); err != nil { + return err + } + ok = true + return nil +} + +func appendArrowValue(builder array.Builder, value any) error { + if value == nil { + builder.AppendNull() + return nil + } + switch b := builder.(type) { + case *array.StringBuilder: + b.Append(value.(string)) + case *array.Int64Builder: + b.Append(value.(int64)) + case *array.Float64Builder: + b.Append(value.(float64)) + case *array.TimestampBuilder: + b.Append(arrow.Timestamp(value.(int64))) + default: + return fmt.Errorf("unsupported Arrow builder %T", builder) + } + return nil +} + +func syncDirectory(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +func text(v string) any { + if v == "" { + return nil + } + return v +} + +func jsonText(v []byte) any { + if len(v) == 0 { + return nil + } + return string(v) +} + +func nanos(primary, secondary, ingested int64) any { + for _, value := range []int64{primary, secondary, ingested} { + if value > 0 { + return value + } + } + return nil +} + +func optionalNanos(value int64) any { + if value <= 0 { + return nil + } + return value +} + +func optionalInt(value int64) any { + if value == 0 { + return nil + } + return value +} + +func spanParquetColumns() []parquetColumn[Span] { + s, i, f, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Float64, arrow.FixedWidthTypes.Timestamp_ns + return []parquetColumn[Span]{ + {"namespace", s, false, func(r Span) any { return r.Namespace }}, + {"trace_id", s, false, func(r Span) any { return r.TraceID }}, + {"span_id", s, false, func(r Span) any { return r.SpanID }}, + {"parent_span_id", s, true, func(r Span) any { return text(r.ParentSpanID) }}, + {"service", s, false, func(r Span) any { return r.ServiceName }}, + {"operation", s, false, func(r Span) any { return r.Name }}, + {"kind", s, false, func(r Span) any { return r.Kind }}, + {"start_time", ts, true, func(r Span) any { return nanos(r.StartUnixNanos, 0, r.IngestedAt) }}, + {"end_time", ts, true, func(r Span) any { return optionalNanos(r.EndUnixNanos) }}, + {"start_unix_nano", i, false, func(r Span) any { return r.StartUnixNanos }}, + {"end_unix_nano", i, false, func(r Span) any { return r.EndUnixNanos }}, + {"duration_ms", f, false, func(r Span) any { return r.DurationMS }}, + {"status", s, false, func(r Span) any { return r.StatusCode }}, + {"status_message", s, true, func(r Span) any { return text(r.StatusMsg) }}, + {"resource_json", s, true, func(r Span) any { return jsonText(r.ResourceJSON) }}, + {"attributes_json", s, true, func(r Span) any { return jsonText(r.AttributesJSON) }}, + {"events_json", s, true, func(r Span) any { return jsonText(r.EventsJSON) }}, + {"links_json", s, true, func(r Span) any { return jsonText(r.LinksJSON) }}, + {"trace_state", s, true, func(r Span) any { return text(r.TraceState) }}, + {"flags", i, false, func(r Span) any { return int64(r.Flags) }}, + {"scope_name", s, true, func(r Span) any { return text(r.ScopeName) }}, + {"scope_version", s, true, func(r Span) any { return text(r.ScopeVersion) }}, + {"ingested_at", ts, true, func(r Span) any { return optionalNanos(r.IngestedAt) }}, + {"ingested_unix_nano", i, false, func(r Span) any { return r.IngestedAt }}, + {"http_method", s, true, func(r Span) any { return text(r.HTTPMethod) }}, + {"http_status_code", s, true, func(r Span) any { return text(r.HTTPStatusCode) }}, + {"http_route", s, true, func(r Span) any { return text(r.HTTPRoute) }}, + {"db_system", s, true, func(r Span) any { return text(r.DBSystem) }}, + {"rpc_method", s, true, func(r Span) any { return text(r.RPCMethod) }}, + {"rpc_service", s, true, func(r Span) any { return text(r.RPCService) }}, + {"peer_service", s, true, func(r Span) any { return text(r.PeerService) }}, + {"service_version", s, true, func(r Span) any { return text(r.ServiceVersion) }}, + {"deployment_env", s, true, func(r Span) any { return text(r.DeploymentEnv) }}, + {"exception_type", s, true, func(r Span) any { return text(r.ExceptionType) }}, + {"exception_message", s, true, func(r Span) any { return text(r.ExceptionMessage) }}, + } +} + +func logParquetColumns() []parquetColumn[Log] { + s, i, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.FixedWidthTypes.Timestamp_ns + return []parquetColumn[Log]{ + {"namespace", s, false, func(r Log) any { return r.Namespace }}, + {"log_time", ts, true, func(r Log) any { return nanos(r.TimeUnixNanos, r.ObservedTimeNanos, r.IngestedAt) }}, + {"observed_time", ts, true, func(r Log) any { return nanos(r.ObservedTimeNanos, r.TimeUnixNanos, r.IngestedAt) }}, + {"time_unix_nano", i, false, func(r Log) any { return r.TimeUnixNanos }}, + {"observed_time_unix_nano", i, true, func(r Log) any { return optionalInt(r.ObservedTimeNanos) }}, + {"severity", s, false, func(r Log) any { return r.Severity }}, + {"severity_number", i, false, func(r Log) any { return int64(r.SeverityNumber) }}, + {"body", s, false, func(r Log) any { return r.Body }}, + {"service", s, true, func(r Log) any { return text(r.ServiceName) }}, + {"trace_id", s, true, func(r Log) any { return text(r.TraceID) }}, + {"span_id", s, true, func(r Log) any { return text(r.SpanID) }}, + {"flags", i, false, func(r Log) any { return int64(r.Flags) }}, + {"resource_json", s, true, func(r Log) any { return jsonText(r.ResourceJSON) }}, + {"attributes_json", s, true, func(r Log) any { return jsonText(r.AttributesJSON) }}, + {"scope_name", s, true, func(r Log) any { return text(r.ScopeName) }}, + {"scope_version", s, true, func(r Log) any { return text(r.ScopeVersion) }}, + {"ingested_at", ts, true, func(r Log) any { return optionalNanos(r.IngestedAt) }}, + {"ingested_unix_nano", i, false, func(r Log) any { return r.IngestedAt }}, + {"body_template", s, true, func(r Log) any { return text(r.BodyTemplate) }}, + } +} + +func metricParquetColumns() []parquetColumn[Metric] { + s, i, f, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Float64, arrow.FixedWidthTypes.Timestamp_ns + return []parquetColumn[Metric]{ + {"namespace", s, false, func(r Metric) any { return r.Namespace }}, + {"metric_time", ts, true, func(r Metric) any { return nanos(r.TimeUnixNanos, 0, r.IngestedAt) }}, + {"time_unix_nano", i, false, func(r Metric) any { return r.TimeUnixNanos }}, + {"name", s, false, func(r Metric) any { return r.Name }}, + {"description", s, true, func(r Metric) any { return text(r.Description) }}, + {"unit", s, true, func(r Metric) any { return text(r.Unit) }}, + {"metric_type", s, false, func(r Metric) any { return r.Type }}, + {"service", s, true, func(r Metric) any { return text(r.ServiceName) }}, + {"value", f, false, func(r Metric) any { return r.Value }}, + {"hist_bounds_json", s, true, func(r Metric) any { return jsonText(r.HistBoundsJSON) }}, + {"hist_counts_json", s, true, func(r Metric) any { return jsonText(r.HistCountsJSON) }}, + {"hist_count", i, true, func(r Metric) any { return optionalInt(r.HistCount) }}, + {"hist_sum", f, false, func(r Metric) any { return r.HistSum }}, + {"exemplars_json", s, true, func(r Metric) any { return jsonText(r.ExemplarsJSON) }}, + {"attributes_json", s, true, func(r Metric) any { return jsonText(r.AttributesJSON) }}, + {"resource_json", s, true, func(r Metric) any { return jsonText(r.ResourceJSON) }}, + {"scope_name", s, true, func(r Metric) any { return text(r.ScopeName) }}, + {"scope_version", s, true, func(r Metric) any { return text(r.ScopeVersion) }}, + {"ingested_at", ts, true, func(r Metric) any { return optionalNanos(r.IngestedAt) }}, + {"ingested_unix_nano", i, false, func(r Metric) any { return r.IngestedAt }}, + } +} diff --git a/internal/telemetry/rows.go b/internal/telemetry/rows.go new file mode 100644 index 00000000..606503f5 --- /dev/null +++ b/internal/telemetry/rows.go @@ -0,0 +1,87 @@ +// Package telemetry defines the canonical, storage-independent rows emitted by +// the OTLP decoder. These are the only telemetry write contracts in Fanout. +package telemetry + +type Span struct { + Namespace string + TraceID string + SpanID string + ParentSpanID string + ServiceName string + Name string + Kind string + StartUnixNanos int64 + EndUnixNanos int64 + DurationMS float64 + StatusCode string + StatusMsg string + ResourceJSON []byte + AttributesJSON []byte + EventsJSON []byte + LinksJSON []byte + TraceState string + Flags uint32 + ScopeName string + ScopeVersion string + IngestedAt int64 + HTTPMethod string + HTTPStatusCode string + HTTPRoute string + DBSystem string + RPCMethod string + RPCService string + PeerService string + ServiceVersion string + DeploymentEnv string + ExceptionType string + ExceptionMessage string +} + +type Log struct { + Namespace string + EventUnixNanos int64 + TimeUnixNanos int64 + ObservedTimeNanos int64 + Severity string + SeverityNumber int32 + Body string + ServiceName string + TraceID string + SpanID string + Flags uint32 + ResourceJSON []byte + AttributesJSON []byte + ScopeName string + ScopeVersion string + IngestedAt int64 + BodyTemplate string +} + +type Metric struct { + Namespace string + EventUnixNanos int64 + TimeUnixNanos int64 + Name string + Description string + Unit string + Type string + ServiceName string + Value float64 + HistBoundsJSON []byte + HistCountsJSON []byte + HistCount int64 + HistSum float64 + ExemplarsJSON []byte + AttributesJSON []byte + ResourceJSON []byte + ScopeName string + ScopeVersion string + IngestedAt int64 +} + +func NormalizeNamespace(namespace string) string { + if namespace == "" { + return "default" + } + return namespace +} diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go new file mode 100644 index 00000000..8734ae65 --- /dev/null +++ b/internal/telemetry/segment/signal_store.go @@ -0,0 +1,582 @@ +// Generic signal segments cover logs and metrics; spans add specialized indexes. +package segment + +import ( + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "reflect" + "regexp" + "sync" + + "github.com/klauspost/compress/zstd" +) + +const ( + signalMagic = "FANSIG03" + signalVersion = uint32(3) + signalHeaderSize = 64 + signalBlockSize = 32 + signalBlockRows = 2048 +) + +var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) + +type signalBlock struct { + offset uint64 + length uint32 + rows uint32 + min int64 + max int64 +} + +type signalSegment struct { + id string + path string + rows uint32 + min int64 + max int64 + fieldCount uint32 + fingerprint uint64 + blocks []signalBlock +} + +type signalManifest struct { + Version uint32 `json:"version"` + Files []string `json:"files"` +} + +// SignalStore persists one telemetry signal as immutable, independently +// compressed columns. T must be a struct containing only string, []byte, +// int32, uint32, int64, and float64 fields. +type SignalStore[T any] struct { + dir string + timeField int + codec structCodec[T] + writeMu sync.Mutex + mu sync.RWMutex + manifest signalManifest + segments []signalSegment + encoder *zstd.Encoder + decoders sync.Pool +} + +func OpenSignalStore[T any](dir, timeField string) (*SignalStore[T], error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create signal directory: %w", err) + } + codec, err := newStructCodec[T]() + if err != nil { + return nil, err + } + field, found := codec.typ.FieldByName(timeField) + if !found || field.Type.Kind() != reflect.Int64 { + return nil, fmt.Errorf("signal time field %q must be int64", timeField) + } + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithEncoderCRC(true)) + if err != nil { + return nil, fmt.Errorf("create signal encoder: %w", err) + } + s := &SignalStore[T]{dir: dir, timeField: field.Index[0], codec: codec, encoder: enc} + s.decoders.New = func() any { + dec, decErr := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + if decErr != nil { + panic(decErr) + } + return dec + } + if err := s.load(); err != nil { + enc.Close() + return nil, err + } + return s, nil +} + +func (s *SignalStore[T]) Close() error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + return s.encoder.Close() +} + +func (s *SignalStore[T]) load() error { + path := filepath.Join(s.dir, "MANIFEST.json") + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + s.manifest = signalManifest{Version: signalVersion} + return nil + } + if err != nil { + return fmt.Errorf("read signal manifest: %w", err) + } + if err := json.Unmarshal(data, &s.manifest); err != nil { + return fmt.Errorf("decode signal manifest: %w", err) + } + if s.manifest.Version != signalVersion { + return fmt.Errorf("signal manifest version %d is unsupported; expected %d", s.manifest.Version, signalVersion) + } + for _, name := range s.manifest.Files { + seg, err := openSignalSegment(filepath.Join(s.dir, name)) + if err != nil { + return fmt.Errorf("open signal segment %s: %w", name, err) + } + if seg.fieldCount != uint32(len(s.codec.fields)) || seg.fingerprint != s.codec.fingerprint { + return fmt.Errorf("signal segment %s schema does not match canonical telemetry row", name) + } + s.segments = append(s.segments, seg) + } + return nil +} + +// Append publishes rows exactly once for id. Replaying a committed transaction +// after a crash is therefore safe. +func (s *SignalStore[T]) Append(id string, rows []T) error { + if len(rows) == 0 { + return nil + } + if !segmentIDPattern.MatchString(id) { + return fmt.Errorf("invalid segment id %q", id) + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + + name := id + ".fseg" + s.mu.RLock() + for _, existing := range s.manifest.Files { + if existing == name { + s.mu.RUnlock() + return nil + } + } + current := s.manifest + s.mu.RUnlock() + + tmp := filepath.Join(s.dir, name+".tmp") + final := filepath.Join(s.dir, name) + _ = os.Remove(tmp) + seg, err := s.writeSegment(tmp, id, rows) + if err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("publish signal segment: %w", err) + } + if err := syncDir(s.dir); err != nil { + return err + } + next := current + next.Version = signalVersion + next.Files = append(append([]string(nil), current.Files...), name) + if err := writeSignalManifest(s.dir, next); err != nil { + return err + } + seg.path = final + s.mu.Lock() + s.manifest = next + s.segments = append(s.segments, seg) + s.mu.Unlock() + return nil +} + +func (s *SignalStore[T]) writeSegment(path, id string, rows []T) (signalSegment, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return signalSegment{}, fmt.Errorf("create signal segment: %w", err) + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(make([]byte, signalHeaderSize)); err != nil { + return signalSegment{}, err + } + seg := signalSegment{id: id, rows: uint32(len(rows)), min: math.MaxInt64, max: math.MinInt64, fieldCount: uint32(len(s.codec.fields)), fingerprint: s.codec.fingerprint} + offset := uint64(signalHeaderSize) + for start := 0; start < len(rows); start += signalBlockRows { + end := min(start+signalBlockRows, len(rows)) + blockMin, blockMax := int64(math.MaxInt64), int64(math.MinInt64) + for i := start; i < end; i++ { + ts := reflect.ValueOf(rows[i]).Field(s.timeField).Int() + blockMin, blockMax = min(blockMin, ts), max(blockMax, ts) + seg.min, seg.max = min(seg.min, ts), max(seg.max, ts) + } + encoded, err := s.codec.encodeBlock(s.encoder, rows[start:end]) + if err != nil { + return signalSegment{}, err + } + if _, err := f.Write(encoded); err != nil { + return signalSegment{}, err + } + seg.blocks = append(seg.blocks, signalBlock{offset: offset, length: uint32(len(encoded)), rows: uint32(end - start), min: blockMin, max: blockMax}) + offset += uint64(len(encoded)) + } + dirOffset := offset + directory := make([]byte, len(seg.blocks)*signalBlockSize) + for i, block := range seg.blocks { + entry := directory[i*signalBlockSize:] + binary.LittleEndian.PutUint64(entry[0:8], block.offset) + binary.LittleEndian.PutUint32(entry[8:12], block.length) + binary.LittleEndian.PutUint32(entry[12:16], block.rows) + binary.LittleEndian.PutUint64(entry[16:24], uint64(block.min)) + binary.LittleEndian.PutUint64(entry[24:32], uint64(block.max)) + } + if _, err := f.Write(directory); err != nil { + return signalSegment{}, err + } + var header [signalHeaderSize]byte + copy(header[0:8], signalMagic) + binary.LittleEndian.PutUint32(header[8:12], signalVersion) + binary.LittleEndian.PutUint32(header[12:16], seg.rows) + binary.LittleEndian.PutUint32(header[16:20], uint32(len(seg.blocks))) + binary.LittleEndian.PutUint32(header[20:24], uint32(len(s.codec.fields))) + binary.LittleEndian.PutUint64(header[24:32], uint64(seg.min)) + binary.LittleEndian.PutUint64(header[32:40], uint64(seg.max)) + binary.LittleEndian.PutUint64(header[40:48], dirOffset) + binary.LittleEndian.PutUint64(header[48:56], s.codec.fingerprint) + if _, err := f.WriteAt(header[:], 0); err != nil { + return signalSegment{}, err + } + if err := f.Sync(); err != nil { + return signalSegment{}, err + } + if err := f.Close(); err != nil { + return signalSegment{}, err + } + ok = true + return seg, nil +} + +// Scan visits rows in [start,end), using segment and block time pruning. +func (s *SignalStore[T]) Scan(start, end int64, visit func(T) bool) error { + s.mu.RLock() + defer s.mu.RUnlock() + dec := s.decoders.Get().(*zstd.Decoder) + defer s.decoders.Put(dec) + for _, seg := range s.segments { + if seg.max < start || seg.min >= end { + continue + } + f, err := os.Open(seg.path) + if err != nil { + return err + } + for _, block := range seg.blocks { + if block.max < start || block.min >= end { + continue + } + data := make([]byte, block.length) + if _, err := f.ReadAt(data, int64(block.offset)); err != nil { + _ = f.Close() + return err + } + rows, err := s.codec.decodeBlock(dec, data, int(block.rows)) + if err != nil { + _ = f.Close() + return fmt.Errorf("decode %s: %w", filepath.Base(seg.path), err) + } + for _, row := range rows { + ts := reflect.ValueOf(row).Field(s.timeField).Int() + if ts >= start && ts < end && !visit(row) { + _ = f.Close() + return nil + } + } + } + if err := f.Close(); err != nil { + return err + } + } + return nil +} + +func (s *SignalStore[T]) RowCount() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + var total uint64 + for _, seg := range s.segments { + total += uint64(seg.rows) + } + return total +} + +func (s *SignalStore[T]) SegmentCount() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.segments) +} + +func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.mu.RLock() + current := s.manifest + segments := append([]signalSegment(nil), s.segments...) + s.mu.RUnlock() + kept := make([]signalSegment, 0, len(segments)) + removed := make([]signalSegment, 0) + for _, seg := range segments { + if seg.max < cutoff { + removed = append(removed, seg) + } else { + kept = append(kept, seg) + } + } + if len(removed) == 0 { + return 0, nil + } + next := current + next.Files = next.Files[:0] + for _, seg := range kept { + next.Files = append(next.Files, filepath.Base(seg.path)) + } + if err := writeSignalManifest(s.dir, next); err != nil { + return 0, err + } + s.mu.Lock() + s.manifest, s.segments = next, kept + s.mu.Unlock() + var removeErr error + for _, seg := range removed { + if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { + removeErr = errors.Join(removeErr, err) + } + } + return len(removed), errors.Join(removeErr, syncDir(s.dir)) +} + +func openSignalSegment(path string) (signalSegment, error) { + f, err := os.Open(path) + if err != nil { + return signalSegment{}, err + } + defer f.Close() + var header [signalHeaderSize]byte + if _, err := io.ReadFull(f, header[:]); err != nil { + return signalSegment{}, err + } + if string(header[0:8]) != signalMagic || binary.LittleEndian.Uint32(header[8:12]) != signalVersion { + return signalSegment{}, errors.New("unsupported signal segment format") + } + seg := signalSegment{ + id: filepath.Base(path[:len(path)-len(filepath.Ext(path))]), path: path, + rows: binary.LittleEndian.Uint32(header[12:16]), fieldCount: binary.LittleEndian.Uint32(header[20:24]), + min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40])), + fingerprint: binary.LittleEndian.Uint64(header[48:56]), + } + count := int(binary.LittleEndian.Uint32(header[16:20])) + dirOffset := int64(binary.LittleEndian.Uint64(header[40:48])) + if count < 0 || dirOffset < signalHeaderSize { + return signalSegment{}, errors.New("invalid signal directory") + } + directory := make([]byte, count*signalBlockSize) + if _, err := f.ReadAt(directory, dirOffset); err != nil { + return signalSegment{}, err + } + for i := range count { + entry := directory[i*signalBlockSize:] + seg.blocks = append(seg.blocks, signalBlock{ + offset: binary.LittleEndian.Uint64(entry[0:8]), length: binary.LittleEndian.Uint32(entry[8:12]), rows: binary.LittleEndian.Uint32(entry[12:16]), + min: int64(binary.LittleEndian.Uint64(entry[16:24])), max: int64(binary.LittleEndian.Uint64(entry[24:32])), + }) + } + return seg, nil +} + +func writeSignalManifest(dir string, manifest signalManifest) error { + data, err := json.Marshal(manifest) + if err != nil { + return err + } + tmp := filepath.Join(dir, "MANIFEST.json.tmp") + final := filepath.Join(dir, "MANIFEST.json") + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + return err + } + return syncDir(dir) +} + +type fieldKind uint8 + +const ( + fieldString fieldKind = iota + fieldBytes + fieldInt64 + fieldInt32 + fieldUint32 + fieldFloat64 +) + +type codecField struct { + name string + index int + kind fieldKind +} + +type structCodec[T any] struct { + typ reflect.Type + fields []codecField + fingerprint uint64 +} + +func newStructCodec[T any]() (structCodec[T], error) { + typ := reflect.TypeOf((*T)(nil)).Elem() + if typ.Kind() != reflect.Struct { + return structCodec[T]{}, errors.New("signal row must be a struct") + } + c := structCodec[T]{typ: typ} + var fp uint64 = 1469598103934665603 + for i := range typ.NumField() { + field := typ.Field(i) + var kind fieldKind + switch { + case field.Type.Kind() == reflect.String: + kind = fieldString + case field.Type == reflect.TypeOf([]byte(nil)): + kind = fieldBytes + case field.Type.Kind() == reflect.Int64: + kind = fieldInt64 + case field.Type.Kind() == reflect.Int32: + kind = fieldInt32 + case field.Type.Kind() == reflect.Uint32: + kind = fieldUint32 + case field.Type.Kind() == reflect.Float64: + kind = fieldFloat64 + default: + return structCodec[T]{}, fmt.Errorf("unsupported field %s (%s)", field.Name, field.Type) + } + c.fields = append(c.fields, codecField{name: field.Name, index: i, kind: kind}) + for _, b := range []byte(field.Name + ":" + field.Type.String()) { + fp ^= uint64(b) + fp *= 1099511628211 + } + } + c.fingerprint = fp + return c, nil +} + +func (c structCodec[T]) encodeBlock(enc *zstd.Encoder, rows []T) ([]byte, error) { + columns := make([][]byte, len(c.fields)) + for _, row := range rows { + value := reflect.ValueOf(row) + for i, field := range c.fields { + v := value.Field(field.index) + switch field.kind { + case fieldString: + columns[i] = appendBytes(columns[i], []byte(v.String())) + case fieldBytes: + columns[i] = appendBytes(columns[i], v.Bytes()) + case fieldInt64: + columns[i] = binary.LittleEndian.AppendUint64(columns[i], uint64(v.Int())) + case fieldInt32: + columns[i] = binary.LittleEndian.AppendUint32(columns[i], uint32(v.Int())) + case fieldUint32: + columns[i] = binary.LittleEndian.AppendUint32(columns[i], uint32(v.Uint())) + case fieldFloat64: + columns[i] = binary.LittleEndian.AppendUint64(columns[i], math.Float64bits(v.Float())) + } + } + } + headerSize := 4 + len(columns)*8 + out := make([]byte, headerSize) + binary.LittleEndian.PutUint32(out[:4], uint32(len(columns))) + offset := headerSize + for i, plain := range columns { + compressed := enc.EncodeAll(plain, nil) + entry := out[4+i*8:] + binary.LittleEndian.PutUint32(entry[:4], uint32(offset)) + binary.LittleEndian.PutUint32(entry[4:8], uint32(len(compressed))) + out = append(out, compressed...) + offset += len(compressed) + } + return out, nil +} + +func (c structCodec[T]) decodeBlock(dec *zstd.Decoder, block []byte, count int) ([]T, error) { + headerSize := 4 + len(c.fields)*8 + if len(block) < headerSize || int(binary.LittleEndian.Uint32(block[:4])) != len(c.fields) { + return nil, errors.New("invalid signal block header") + } + columns := make([][]byte, len(c.fields)) + for i := range c.fields { + entry := block[4+i*8:] + offset := int(binary.LittleEndian.Uint32(entry[:4])) + length := int(binary.LittleEndian.Uint32(entry[4:8])) + if offset < headerSize || length < 0 || offset > len(block)-length { + return nil, errors.New("invalid signal column extent") + } + plain, err := dec.DecodeAll(block[offset:offset+length], nil) + if err != nil { + return nil, err + } + columns[i] = plain + } + rows := make([]T, count) + for fieldIndex, field := range c.fields { + column := columns[fieldIndex] + for row := range count { + dst := reflect.ValueOf(&rows[row]).Elem().Field(field.index) + switch field.kind { + case fieldString, fieldBytes: + value, rest, err := consumeByteView(column) + if err != nil { + return nil, err + } + column = rest + if field.kind == fieldString { + dst.SetString(string(value)) + } else { + dst.SetBytes(append([]byte(nil), value...)) + } + case fieldInt64, fieldFloat64: + if len(column) < 8 { + return nil, io.ErrUnexpectedEOF + } + bits := binary.LittleEndian.Uint64(column[:8]) + column = column[8:] + if field.kind == fieldInt64 { + dst.SetInt(int64(bits)) + } else { + dst.SetFloat(math.Float64frombits(bits)) + } + case fieldInt32, fieldUint32: + if len(column) < 4 { + return nil, io.ErrUnexpectedEOF + } + bits := binary.LittleEndian.Uint32(column[:4]) + column = column[4:] + if field.kind == fieldInt32 { + dst.SetInt(int64(int32(bits))) + } else { + dst.SetUint(uint64(bits)) + } + } + } + if len(column) != 0 { + return nil, fmt.Errorf("column %s has trailing data", field.name) + } + } + return rows, nil +} diff --git a/internal/telemetry/segment/span_columnar.go b/internal/telemetry/segment/span_columnar.go new file mode 100644 index 00000000..65168c77 --- /dev/null +++ b/internal/telemetry/segment/span_columnar.go @@ -0,0 +1,266 @@ +// Column codecs are deliberately private to the versioned segment format. +package segment + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + + "github.com/klauspost/compress/zstd" +) + +const ( + colNamespace = iota + colTraceID + colSpanID + colParentSpanID + colServiceName + colName + colKind + colStartUnixNanos + colEndUnixNanos + colDurationMS + colStatusCode + colStatusMsg + colResourceJSON + colAttributesJSON + colEventsJSON + colLinksJSON + colTraceState + colFlags + colScopeName + colScopeVersion + colIngestedAt + colHTTPMethod + colHTTPStatusCode + colHTTPRoute + colDBSystem + colRPCMethod + colRPCService + colPeerService + colServiceVersion + colDeploymentEnv + colExceptionType + colExceptionMessage + columnCount +) + +const columnarHeaderSize = 4 + columnCount*8 + +var allColumns = func() []int { + out := make([]int, columnCount) + for i := range out { + out[i] = i + } + return out +}() + +func encodeColumnarBlock(enc *zstd.Encoder, rows []Span) []byte { + columns := make([][]byte, columnCount) + for _, row := range rows { + columns[colNamespace] = appendString(columns[colNamespace], row.Namespace) + columns[colTraceID] = appendString(columns[colTraceID], row.TraceID) + columns[colSpanID] = appendString(columns[colSpanID], row.SpanID) + columns[colParentSpanID] = appendString(columns[colParentSpanID], row.ParentSpanID) + columns[colServiceName] = appendString(columns[colServiceName], row.ServiceName) + columns[colName] = appendString(columns[colName], row.Name) + columns[colKind] = appendString(columns[colKind], row.Kind) + columns[colStartUnixNanos] = binary.LittleEndian.AppendUint64(columns[colStartUnixNanos], uint64(row.StartUnixNanos)) + columns[colEndUnixNanos] = binary.LittleEndian.AppendUint64(columns[colEndUnixNanos], uint64(row.EndUnixNanos)) + columns[colDurationMS] = binary.LittleEndian.AppendUint64(columns[colDurationMS], math.Float64bits(row.DurationMS)) + columns[colStatusCode] = appendString(columns[colStatusCode], row.StatusCode) + columns[colStatusMsg] = appendString(columns[colStatusMsg], row.StatusMsg) + columns[colResourceJSON] = appendBytes(columns[colResourceJSON], row.ResourceJSON) + columns[colAttributesJSON] = appendBytes(columns[colAttributesJSON], row.AttributesJSON) + columns[colEventsJSON] = appendBytes(columns[colEventsJSON], row.EventsJSON) + columns[colLinksJSON] = appendBytes(columns[colLinksJSON], row.LinksJSON) + columns[colTraceState] = appendString(columns[colTraceState], row.TraceState) + columns[colFlags] = binary.LittleEndian.AppendUint32(columns[colFlags], row.Flags) + columns[colScopeName] = appendString(columns[colScopeName], row.ScopeName) + columns[colScopeVersion] = appendString(columns[colScopeVersion], row.ScopeVersion) + columns[colIngestedAt] = binary.LittleEndian.AppendUint64(columns[colIngestedAt], uint64(row.IngestedAt)) + columns[colHTTPMethod] = appendString(columns[colHTTPMethod], row.HTTPMethod) + columns[colHTTPStatusCode] = appendString(columns[colHTTPStatusCode], row.HTTPStatusCode) + columns[colHTTPRoute] = appendString(columns[colHTTPRoute], row.HTTPRoute) + columns[colDBSystem] = appendString(columns[colDBSystem], row.DBSystem) + columns[colRPCMethod] = appendString(columns[colRPCMethod], row.RPCMethod) + columns[colRPCService] = appendString(columns[colRPCService], row.RPCService) + columns[colPeerService] = appendString(columns[colPeerService], row.PeerService) + columns[colServiceVersion] = appendString(columns[colServiceVersion], row.ServiceVersion) + columns[colDeploymentEnv] = appendString(columns[colDeploymentEnv], row.DeploymentEnv) + columns[colExceptionType] = appendString(columns[colExceptionType], row.ExceptionType) + columns[colExceptionMessage] = appendString(columns[colExceptionMessage], row.ExceptionMessage) + } + + out := make([]byte, columnarHeaderSize) + binary.LittleEndian.PutUint32(out[0:4], columnCount) + offset := columnarHeaderSize + for id, plain := range columns { + compressed := enc.EncodeAll(plain, nil) + entry := out[4+id*8:] + binary.LittleEndian.PutUint32(entry[0:4], uint32(offset)) + binary.LittleEndian.PutUint32(entry[4:8], uint32(len(compressed))) + out = append(out, compressed...) + offset += len(compressed) + } + return out +} + +func decodeColumns(dec *zstd.Decoder, block []byte, wanted []int) (map[int][]byte, error) { + if len(block) < columnarHeaderSize { + return nil, io.ErrUnexpectedEOF + } + if got := binary.LittleEndian.Uint32(block[0:4]); got != columnCount { + return nil, fmt.Errorf("column count: got %d want %d", got, columnCount) + } + out := make(map[int][]byte, len(wanted)) + for _, id := range wanted { + if id < 0 || id >= columnCount { + return nil, fmt.Errorf("column %d out of range", id) + } + entry := block[4+id*8:] + offset := int(binary.LittleEndian.Uint32(entry[0:4])) + length := int(binary.LittleEndian.Uint32(entry[4:8])) + if offset < columnarHeaderSize || length < 0 || offset > len(block)-length { + return nil, fmt.Errorf("column %d has invalid extent %d+%d", id, offset, length) + } + plain, err := dec.DecodeAll(block[offset:offset+length], nil) + if err != nil { + return nil, fmt.Errorf("decompress column %d: %w", id, err) + } + out[id] = plain + } + return out, nil +} + +func decodeSelectedBlock(columns map[int][]byte, count int, selected []int) ([]Span, error) { + if len(selected) == 0 { + return nil, nil + } + for i, row := range selected { + if row < 0 || row >= count || (i > 0 && selected[i-1] >= row) { + return nil, errors.New("selected rows must be sorted, unique, and in range") + } + } + stringColumns := []int{ + colNamespace, colTraceID, colSpanID, colParentSpanID, colServiceName, + colName, colKind, colStatusCode, colStatusMsg, colTraceState, + colScopeName, colScopeVersion, colHTTPMethod, colHTTPStatusCode, + colHTTPRoute, colDBSystem, colRPCMethod, colRPCService, colPeerService, + colServiceVersion, colDeploymentEnv, colExceptionType, colExceptionMessage, + } + stringsByColumn := make(map[int][]string, len(stringColumns)) + for _, id := range stringColumns { + values, err := selectStrings(columns[id], count, selected) + if err != nil { + return nil, fmt.Errorf("select string column %d: %w", id, err) + } + stringsByColumn[id] = values + } + bytesByColumn := make(map[int][][]byte, 4) + for _, id := range []int{colResourceJSON, colAttributesJSON, colEventsJSON, colLinksJSON} { + values, err := selectBytes(columns[id], count, selected) + if err != nil { + return nil, fmt.Errorf("select bytes column %d: %w", id, err) + } + bytesByColumn[id] = values + } + for _, fixed := range []struct{ id, width int }{{colStartUnixNanos, 8}, {colEndUnixNanos, 8}, {colDurationMS, 8}, {colFlags, 4}, {colIngestedAt, 8}} { + if err := requireFixed(columns[fixed.id], count, fixed.width); err != nil { + return nil, err + } + } + rows := make([]Span, len(selected)) + for i, sourceRow := range selected { + rows[i] = Span{ + Namespace: stringsByColumn[colNamespace][i], TraceID: stringsByColumn[colTraceID][i], SpanID: stringsByColumn[colSpanID][i], + ParentSpanID: stringsByColumn[colParentSpanID][i], ServiceName: stringsByColumn[colServiceName][i], Name: stringsByColumn[colName][i], Kind: stringsByColumn[colKind][i], + StartUnixNanos: int64At(columns[colStartUnixNanos], sourceRow), EndUnixNanos: int64At(columns[colEndUnixNanos], sourceRow), DurationMS: float64At(columns[colDurationMS], sourceRow), + StatusCode: stringsByColumn[colStatusCode][i], StatusMsg: stringsByColumn[colStatusMsg][i], ResourceJSON: bytesByColumn[colResourceJSON][i], + AttributesJSON: bytesByColumn[colAttributesJSON][i], EventsJSON: bytesByColumn[colEventsJSON][i], LinksJSON: bytesByColumn[colLinksJSON][i], + TraceState: stringsByColumn[colTraceState][i], Flags: uint32At(columns[colFlags], sourceRow), ScopeName: stringsByColumn[colScopeName][i], + ScopeVersion: stringsByColumn[colScopeVersion][i], IngestedAt: int64At(columns[colIngestedAt], sourceRow), HTTPMethod: stringsByColumn[colHTTPMethod][i], + HTTPStatusCode: stringsByColumn[colHTTPStatusCode][i], HTTPRoute: stringsByColumn[colHTTPRoute][i], DBSystem: stringsByColumn[colDBSystem][i], + RPCMethod: stringsByColumn[colRPCMethod][i], RPCService: stringsByColumn[colRPCService][i], PeerService: stringsByColumn[colPeerService][i], + ServiceVersion: stringsByColumn[colServiceVersion][i], DeploymentEnv: stringsByColumn[colDeploymentEnv][i], ExceptionType: stringsByColumn[colExceptionType][i], + ExceptionMessage: stringsByColumn[colExceptionMessage][i], + } + } + return rows, nil +} + +func selectStrings(src []byte, count int, selected []int) ([]string, error) { + out := make([]string, len(selected)) + target := 0 + for row := 0; row < count && target < len(selected); row++ { + value, rest, err := consumeByteView(src) + if err != nil { + return nil, err + } + src = rest + if row == selected[target] { + out[target] = string(value) + target++ + } + } + if target != len(selected) { + return nil, io.ErrUnexpectedEOF + } + return out, nil +} + +func selectBytes(src []byte, count int, selected []int) ([][]byte, error) { + out := make([][]byte, len(selected)) + target := 0 + for row := 0; row < count && target < len(selected); row++ { + value, rest, err := consumeByteView(src) + if err != nil { + return nil, err + } + src = rest + if row == selected[target] { + out[target] = append([]byte(nil), value...) + target++ + } + } + if target != len(selected) { + return nil, io.ErrUnexpectedEOF + } + return out, nil +} + +func matchingStringRows(src []byte, count int, target []byte) ([]int, error) { + var out []int + for row := 0; row < count; row++ { + value, rest, err := consumeByteView(src) + if err != nil { + return nil, err + } + src = rest + if bytes.Equal(value, target) { + out = append(out, row) + } + } + return out, nil +} + +func appendBytes(dst, value []byte) []byte { + dst = binary.AppendUvarint(dst, uint64(len(value))) + return append(dst, value...) +} + +func requireFixed(src []byte, count, width int) error { + if len(src) != count*width { + return fmt.Errorf("fixed column size: got %d want %d", len(src), count*width) + } + return nil +} + +func int64At(src []byte, row int) int64 { return int64(binary.LittleEndian.Uint64(src[row*8:])) } +func float64At(src []byte, row int) float64 { + return math.Float64frombits(binary.LittleEndian.Uint64(src[row*8:])) +} +func uint32At(src []byte, row int) uint32 { return binary.LittleEndian.Uint32(src[row*4:]) } diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go new file mode 100644 index 00000000..c91673aa --- /dev/null +++ b/internal/telemetry/segment/span_store.go @@ -0,0 +1,1102 @@ +// Package segment implements Fanout's append-optimized telemetry segments. +// Immutable, checksummed segment files are published through an atomically +// replaced manifest, so a process crash exposes either the old or new commit. +package segment + +import ( + "bufio" + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/labstack/fanout/internal/telemetry" + "github.com/zeebo/xxh3" +) + +const ( + segmentMagic = "FANSEG02" + segmentVersion = uint32(2) + headerSize = 64 + blockDirSize = 32 + traceEntrySize = 16 + rowsPerBlock = 2048 + rollupBins = 32 + rollupWindow = 5 * time.Minute +) + +type Span = telemetry.Span + +type Endpoint struct { + Service string + Method string + Route string + Calls uint64 + Errors uint64 + AverageMS float64 + P95MS float64 +} + +type Aggregate struct { + Calls uint64 + Errors uint64 + DurationMS float64 +} + +type blockDir struct { + offset uint64 + length uint32 + rows uint32 + min int64 + max int64 +} + +type traceEntry struct { + hash uint64 + block uint32 + row uint32 +} + +type rollupKey struct { + bucket int64 + namespace string + service string + method string + route string +} + +type rollup struct { + key rollupKey + calls uint64 + errors uint64 + duration float64 + bins [rollupBins]uint32 +} + +type segment struct { + path string + rows uint32 + min int64 + max int64 + blocks []blockDir + traceIndex []traceEntry + rollups []rollup +} + +type manifest struct { + NextID uint64 `json:"next_id"` + Files []string `json:"files"` +} + +// Store is a set of immutable segment files referenced by one atomically +// replaced manifest. Readers see either the old commit or the complete new one. +type Store struct { + dir string + writeMu sync.Mutex + mu sync.RWMutex + manifest manifest + segments []segment + encoder *zstd.Encoder + decoders sync.Pool +} + +func Open(dir string) (*Store, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create segment directory: %w", err) + } + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) + if err != nil { + return nil, fmt.Errorf("create zstd encoder: %w", err) + } + s := &Store{dir: dir, encoder: enc} + s.decoders.New = func() any { + dec, decErr := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + if decErr != nil { + panic(decErr) + } + return dec + } + if err := s.loadManifest(); err != nil { + enc.Close() + return nil, err + } + return s, nil +} + +func (s *Store) Close() error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + return s.encoder.Close() +} + +func (s *Store) loadManifest() error { + path := filepath.Join(s.dir, "MANIFEST.json") + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + s.manifest = manifest{NextID: 1} + } else if err != nil { + return fmt.Errorf("read manifest: %w", err) + } else if err := json.Unmarshal(data, &s.manifest); err != nil { + return fmt.Errorf("decode manifest: %w", err) + } + // A crash can occur after a segment rename but before the manifest rename. + // Such an orphan is intentionally invisible, but its numeric name must still + // advance the allocator so the next append does not collide with it. + entries, err := os.ReadDir(s.dir) + if err != nil { + return fmt.Errorf("scan segment directory: %w", err) + } + for _, entry := range entries { + name := entry.Name() + if !strings.HasSuffix(name, ".fseg") { + continue + } + id, parseErr := strconv.ParseUint(strings.TrimSuffix(name, ".fseg"), 10, 64) + if parseErr == nil && id >= s.manifest.NextID { + s.manifest.NextID = id + 1 + } + } + for _, name := range s.manifest.Files { + seg, err := openSegment(filepath.Join(s.dir, name)) + if err != nil { + return fmt.Errorf("open committed segment %s: %w", name, err) + } + s.segments = append(s.segments, seg) + } + return nil +} + +// Append writes one crash-safe immutable segment and atomically publishes it. +// Rollups and the trace index are built in the same pass as block encoding. +func (s *Store) Append(rows []Span) error { + if len(rows) == 0 { + return nil + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + + s.mu.RLock() + id := s.manifest.NextID + current := s.manifest + s.mu.RUnlock() + name := fmt.Sprintf("%020d.fseg", id) + tmp := filepath.Join(s.dir, name+".tmp") + final := filepath.Join(s.dir, name) + seg, err := s.writeSegment(tmp, rows) + if err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("publish segment: %w", err) + } + if err := syncDir(s.dir); err != nil { + return err + } + + next := current + next.NextID = id + 1 + next.Files = append(append([]string(nil), current.Files...), name) + if err := writeManifest(s.dir, next); err != nil { + // The segment is an unreferenced orphan and therefore invisible after a + // restart. A later compactor can safely collect such files. + s.mu.Lock() + s.manifest.NextID = id + 1 + s.mu.Unlock() + return err + } + seg.path = final + s.mu.Lock() + s.manifest = next + s.segments = append(s.segments, seg) + s.mu.Unlock() + return nil +} + +// AppendID publishes one idempotent span segment for a durable ingest +// transaction. Replaying the same transaction after a crash is a no-op. +func (s *Store) AppendID(id string, rows []Span) error { + if len(rows) == 0 { + return nil + } + if !segmentIDPattern.MatchString(id) { + return fmt.Errorf("invalid segment id %q", id) + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + + name := id + ".fseg" + s.mu.RLock() + current := s.manifest + for _, existing := range current.Files { + if existing == name { + s.mu.RUnlock() + return nil + } + } + s.mu.RUnlock() + tmp := filepath.Join(s.dir, name+".tmp") + final := filepath.Join(s.dir, name) + _ = os.Remove(tmp) + seg, err := s.writeSegment(tmp, rows) + if err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("publish segment: %w", err) + } + if err := syncDir(s.dir); err != nil { + return err + } + next := current + next.Files = append(append([]string(nil), current.Files...), name) + if err := writeManifest(s.dir, next); err != nil { + return err + } + seg.path = final + s.mu.Lock() + s.manifest = next + s.segments = append(s.segments, seg) + s.mu.Unlock() + return nil +} + +// CompactOldest rewrites the oldest committed segments as one larger segment. +// The replacement is published with the same atomic-manifest protocol as an +// append; old files are removed only after in-flight readers release the +// snapshot they were using. +func (s *Store) CompactOldest(count int) error { + if count < 2 { + return nil + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.mu.RLock() + count = min(count, len(s.segments)) + if count < 2 { + s.mu.RUnlock() + return nil + } + old := append([]segment(nil), s.segments[:count]...) + rest := append([]segment(nil), s.segments[count:]...) + current := s.manifest + id := current.NextID + s.mu.RUnlock() + + name := fmt.Sprintf("%020d.fseg", id) + tmp, final := filepath.Join(s.dir, name+".tmp"), filepath.Join(s.dir, name) + replacement, err := s.writeCompactedSegment(tmp, old) + if err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("publish compacted segment: %w", err) + } + if err := syncDir(s.dir); err != nil { + return err + } + next := manifest{NextID: id + 1, Files: make([]string, 0, 1+len(rest))} + next.Files = append(next.Files, name) + for _, seg := range rest { + next.Files = append(next.Files, filepath.Base(seg.path)) + } + if err := writeManifest(s.dir, next); err != nil { + return err + } + replacement.path = final + + s.mu.Lock() + s.manifest = next + s.segments = append([]segment{replacement}, rest...) + var removeErr error + for _, seg := range old { + if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { + removeErr = errors.Join(removeErr, err) + } + } + s.mu.Unlock() + if removeErr != nil { + return fmt.Errorf("remove compacted segments: %w", removeErr) + } + return syncDir(s.dir) +} + +// PruneBefore removes segments whose newest event is older than cutoff. A +// boundary segment is retained intact, so retention never drops a newer row. +func (s *Store) PruneBefore(cutoff int64) (int, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.mu.RLock() + current := s.manifest + segments := append([]segment(nil), s.segments...) + s.mu.RUnlock() + kept := make([]segment, 0, len(segments)) + removed := make([]segment, 0) + for _, seg := range segments { + if seg.max < cutoff { + removed = append(removed, seg) + } else { + kept = append(kept, seg) + } + } + if len(removed) == 0 { + return 0, nil + } + next := current + next.Files = next.Files[:0] + for _, seg := range kept { + next.Files = append(next.Files, filepath.Base(seg.path)) + } + if err := writeManifest(s.dir, next); err != nil { + return 0, err + } + s.mu.Lock() + s.manifest = next + s.segments = kept + s.mu.Unlock() + var removeErr error + for _, seg := range removed { + if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { + removeErr = errors.Join(removeErr, err) + } + } + return len(removed), errors.Join(removeErr, syncDir(s.dir)) +} + +func (s *Store) writeSegment(path string, rows []Span) (segment, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return segment{}, fmt.Errorf("create segment: %w", err) + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(make([]byte, headerSize)); err != nil { + return segment{}, fmt.Errorf("reserve header: %w", err) + } + + seg := segment{rows: uint32(len(rows)), min: math.MaxInt64, max: math.MinInt64} + index := make([]traceEntry, 0, len(rows)) + rollups := make(map[rollupKey]*rollup) + var offset = uint64(headerSize) + for blockStart := 0; blockStart < len(rows); blockStart += rowsPerBlock { + blockEnd := min(blockStart+rowsPerBlock, len(rows)) + blockMin := int64(math.MaxInt64) + blockMax := int64(math.MinInt64) + blockTraces := make(map[uint64]struct{}, blockEnd-blockStart) + for _, row := range rows[blockStart:blockEnd] { + blockMin = min(blockMin, row.StartUnixNanos) + blockMax = max(blockMax, row.StartUnixNanos) + seg.min = min(seg.min, row.StartUnixNanos) + seg.max = max(seg.max, row.StartUnixNanos) + traceHash := xxh3.HashString(row.TraceID) + if _, exists := blockTraces[traceHash]; !exists { + index = append(index, traceEntry{hash: traceHash, block: uint32(len(seg.blocks))}) + blockTraces[traceHash] = struct{}{} + } + key := rollupKey{ + bucket: row.StartUnixNanos - row.StartUnixNanos%int64(rollupWindow), + namespace: row.Namespace, service: row.ServiceName, method: row.HTTPMethod, route: row.HTTPRoute, + } + r := rollups[key] + if r == nil { + r = &rollup{key: key} + rollups[key] = r + } + r.calls++ + if row.StatusCode == "ERROR" { + r.errors++ + } + r.duration += row.DurationMS + r.bins[durationBin(row.DurationMS)]++ + } + encoded := encodeColumnarBlock(s.encoder, rows[blockStart:blockEnd]) + if _, err := f.Write(encoded); err != nil { + return segment{}, fmt.Errorf("write block: %w", err) + } + seg.blocks = append(seg.blocks, blockDir{offset: offset, length: uint32(len(encoded)), rows: uint32(blockEnd - blockStart), min: blockMin, max: blockMax}) + offset += uint64(len(encoded)) + } + + sort.Slice(index, func(i, j int) bool { + if index[i].hash != index[j].hash { + return index[i].hash < index[j].hash + } + if index[i].block != index[j].block { + return index[i].block < index[j].block + } + return index[i].row < index[j].row + }) + seg.traceIndex = index + seg.rollups = make([]rollup, 0, len(rollups)) + for _, r := range rollups { + seg.rollups = append(seg.rollups, *r) + } + sort.Slice(seg.rollups, func(i, j int) bool { + a, b := seg.rollups[i].key, seg.rollups[j].key + if a.bucket != b.bucket { + return a.bucket < b.bucket + } + if a.namespace != b.namespace { + return a.namespace < b.namespace + } + if a.service != b.service { + return a.service < b.service + } + if a.method != b.method { + return a.method < b.method + } + return a.route < b.route + }) + + dirOffset := offset + directory := make([]byte, len(seg.blocks)*blockDirSize) + for i, block := range seg.blocks { + buf := directory[i*blockDirSize:] + binary.LittleEndian.PutUint64(buf[0:8], block.offset) + binary.LittleEndian.PutUint32(buf[8:12], block.length) + binary.LittleEndian.PutUint32(buf[12:16], block.rows) + binary.LittleEndian.PutUint64(buf[16:24], uint64(block.min)) + binary.LittleEndian.PutUint64(buf[24:32], uint64(block.max)) + } + if _, err := f.Write(directory); err != nil { + return segment{}, fmt.Errorf("write block directory: %w", err) + } + indexOffset, _ := f.Seek(0, io.SeekCurrent) + indexPlain := make([]byte, len(index)*traceEntrySize) + for i, entry := range index { + buf := indexPlain[i*traceEntrySize:] + binary.LittleEndian.PutUint64(buf[0:8], entry.hash) + binary.LittleEndian.PutUint32(buf[8:12], entry.block) + binary.LittleEndian.PutUint32(buf[12:16], entry.row) + } + if _, err := f.Write(s.encoder.EncodeAll(indexPlain, nil)); err != nil { + return segment{}, fmt.Errorf("write trace index: %w", err) + } + rollupOffset, _ := f.Seek(0, io.SeekCurrent) + var rollupPlain bytes.Buffer + for _, r := range seg.rollups { + if err := writeRollup(&rollupPlain, r); err != nil { + return segment{}, err + } + } + if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { + return segment{}, fmt.Errorf("write rollups: %w", err) + } + + var header [headerSize]byte + copy(header[0:8], segmentMagic) + binary.LittleEndian.PutUint32(header[8:12], segmentVersion) + binary.LittleEndian.PutUint32(header[12:16], seg.rows) + binary.LittleEndian.PutUint32(header[16:20], uint32(len(seg.blocks))) + binary.LittleEndian.PutUint32(header[20:24], uint32(len(seg.rollups))) + binary.LittleEndian.PutUint64(header[24:32], uint64(seg.min)) + binary.LittleEndian.PutUint64(header[32:40], uint64(seg.max)) + binary.LittleEndian.PutUint64(header[40:48], dirOffset) + binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) + binary.LittleEndian.PutUint64(header[56:64], uint64(rollupOffset)) + if _, err := f.WriteAt(header[:], 0); err != nil { + return segment{}, fmt.Errorf("write header: %w", err) + } + if err := f.Sync(); err != nil { + return segment{}, fmt.Errorf("sync segment: %w", err) + } + if err := f.Close(); err != nil { + return segment{}, fmt.Errorf("close segment: %w", err) + } + ok = true + return seg, nil +} + +func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return segment{}, fmt.Errorf("create compacted segment: %w", err) + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(make([]byte, headerSize)); err != nil { + return segment{}, err + } + replacement := segment{min: math.MaxInt64, max: math.MinInt64} + var offset = uint64(headerSize) + for _, input := range inputs { + source, err := os.Open(input.path) + if err != nil { + return segment{}, err + } + blockBase := uint32(len(replacement.blocks)) + for _, block := range input.blocks { + if _, err := io.CopyN(f, io.NewSectionReader(source, int64(block.offset), int64(block.length)), int64(block.length)); err != nil { + _ = source.Close() + return segment{}, fmt.Errorf("copy compressed block: %w", err) + } + replacement.blocks = append(replacement.blocks, blockDir{offset: offset, length: block.length, rows: block.rows, min: block.min, max: block.max}) + offset += uint64(block.length) + } + if err := source.Close(); err != nil { + return segment{}, err + } + for _, entry := range input.traceIndex { + entry.block += blockBase + replacement.traceIndex = append(replacement.traceIndex, entry) + } + replacement.rollups = append(replacement.rollups, input.rollups...) + replacement.rows += input.rows + replacement.min = min(replacement.min, input.min) + replacement.max = max(replacement.max, input.max) + } + sort.Slice(replacement.traceIndex, func(i, j int) bool { + a, b := replacement.traceIndex[i], replacement.traceIndex[j] + if a.hash != b.hash { + return a.hash < b.hash + } + if a.block != b.block { + return a.block < b.block + } + return a.row < b.row + }) + sort.Slice(replacement.rollups, func(i, j int) bool { + a, b := replacement.rollups[i].key, replacement.rollups[j].key + if a.bucket != b.bucket { + return a.bucket < b.bucket + } + if a.namespace != b.namespace { + return a.namespace < b.namespace + } + if a.service != b.service { + return a.service < b.service + } + if a.method != b.method { + return a.method < b.method + } + return a.route < b.route + }) + + dirOffset := offset + directory := make([]byte, len(replacement.blocks)*blockDirSize) + for i, block := range replacement.blocks { + buf := directory[i*blockDirSize:] + binary.LittleEndian.PutUint64(buf[0:8], block.offset) + binary.LittleEndian.PutUint32(buf[8:12], block.length) + binary.LittleEndian.PutUint32(buf[12:16], block.rows) + binary.LittleEndian.PutUint64(buf[16:24], uint64(block.min)) + binary.LittleEndian.PutUint64(buf[24:32], uint64(block.max)) + } + if _, err := f.Write(directory); err != nil { + return segment{}, err + } + indexOffset, _ := f.Seek(0, io.SeekCurrent) + indexPlain := make([]byte, len(replacement.traceIndex)*traceEntrySize) + for i, entry := range replacement.traceIndex { + buf := indexPlain[i*traceEntrySize:] + binary.LittleEndian.PutUint64(buf[0:8], entry.hash) + binary.LittleEndian.PutUint32(buf[8:12], entry.block) + binary.LittleEndian.PutUint32(buf[12:16], entry.row) + } + if _, err := f.Write(s.encoder.EncodeAll(indexPlain, nil)); err != nil { + return segment{}, err + } + rollupOffset, _ := f.Seek(0, io.SeekCurrent) + var rollupPlain bytes.Buffer + for _, r := range replacement.rollups { + if err := writeRollup(&rollupPlain, r); err != nil { + return segment{}, err + } + } + if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { + return segment{}, err + } + var header [headerSize]byte + copy(header[0:8], segmentMagic) + binary.LittleEndian.PutUint32(header[8:12], segmentVersion) + binary.LittleEndian.PutUint32(header[12:16], replacement.rows) + binary.LittleEndian.PutUint32(header[16:20], uint32(len(replacement.blocks))) + binary.LittleEndian.PutUint32(header[20:24], uint32(len(replacement.rollups))) + binary.LittleEndian.PutUint64(header[24:32], uint64(replacement.min)) + binary.LittleEndian.PutUint64(header[32:40], uint64(replacement.max)) + binary.LittleEndian.PutUint64(header[40:48], dirOffset) + binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) + binary.LittleEndian.PutUint64(header[56:64], uint64(rollupOffset)) + if _, err := f.WriteAt(header[:], 0); err != nil { + return segment{}, err + } + if err := f.Sync(); err != nil { + return segment{}, err + } + if err := f.Close(); err != nil { + return segment{}, err + } + ok = true + return replacement, nil +} + +// Trace performs a hash-index lookup and decompresses only the blocks that can +// contain the requested trace. The full trace ID is checked after hashing. +func (s *Store) Trace(traceID string) ([]Span, error) { + s.mu.RLock() + defer s.mu.RUnlock() + hash := xxh3.HashString(traceID) + var out []Span + for i := range s.segments { + seg := &s.segments[i] + start := sort.Search(len(seg.traceIndex), func(j int) bool { return seg.traceIndex[j].hash >= hash }) + blocks := make(map[uint32][]uint32) + for j := start; j < len(seg.traceIndex) && seg.traceIndex[j].hash == hash; j++ { + entry := seg.traceIndex[j] + blocks[entry.block] = append(blocks[entry.block], entry.row) + } + for blockID := range blocks { + rows, err := s.readTraceBlock(*seg, blockID, traceID) + if err != nil { + return nil, err + } + out = append(out, rows...) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].StartUnixNanos < out[j].StartUnixNanos }) + return out, nil +} + +// Endpoints answers Fanout's dashboard query entirely from ingestion-time +// rollups. Quantiles use the same bounded-histogram approximation style as the +// existing Fanout endpoint cache. +func (s *Store) Endpoints(namespace, service string, start, end int64, limit int) []Endpoint { + s.mu.RLock() + defer s.mu.RUnlock() + type value struct { + calls, errors uint64 + duration float64 + bins [rollupBins]uint32 + } + values := make(map[string]*value) + keys := make(map[string]rollupKey) + for i := range s.segments { + seg := &s.segments[i] + if seg.max < start || seg.min >= end { + continue + } + for _, r := range seg.rollups { + if r.key.bucket < start-start%int64(rollupWindow) || r.key.bucket >= end { + continue + } + if namespace != "" && r.key.namespace != namespace { + continue + } + if service != "" && r.key.service != service { + continue + } + key := r.key.service + "\x00" + r.key.method + "\x00" + r.key.route + v := values[key] + if v == nil { + v = &value{} + values[key] = v + keys[key] = r.key + } + v.calls += r.calls + v.errors += r.errors + v.duration += r.duration + for j := range v.bins { + v.bins[j] += r.bins[j] + } + } + } + out := make([]Endpoint, 0, len(values)) + for key, v := range values { + k := keys[key] + average := 0.0 + if v.calls > 0 { + average = v.duration / float64(v.calls) + } + out = append(out, Endpoint{Service: k.service, Method: k.method, Route: k.route, Calls: v.calls, Errors: v.errors, AverageMS: average, P95MS: histogramQuantile(v.bins, v.calls, .95)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Calls > out[j].Calls }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} + +// ScanService is the deliberately expensive raw path. It demonstrates block +// time pruning and provides a fairer comparison with a general query engine. +func (s *Store) ScanService(namespace, service string, start, end int64) (Aggregate, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var out Aggregate + wanted := []int{colNamespace, colServiceName, colStartUnixNanos, colDurationMS, colStatusCode} + namespaceNeedle, serviceNeedle, errorNeedle := []byte(namespace), []byte(service), []byte("ERROR") + for i := range s.segments { + seg := s.segments[i] + if seg.max < start || seg.min >= end { + continue + } + f, err := os.Open(seg.path) + if err != nil { + return Aggregate{}, err + } + for blockID := range seg.blocks { + block := seg.blocks[blockID] + if block.max < start || block.min >= end { + continue + } + columns, err := s.readColumns(f, block, wanted) + if err != nil { + _ = f.Close() + return Aggregate{}, err + } + if err := requireFixed(columns[colStartUnixNanos], int(block.rows), 8); err != nil { + _ = f.Close() + return Aggregate{}, err + } + if err := requireFixed(columns[colDurationMS], int(block.rows), 8); err != nil { + _ = f.Close() + return Aggregate{}, err + } + namespaceColumn := columns[colNamespace] + serviceColumn := columns[colServiceName] + statusColumn := columns[colStatusCode] + for row := range int(block.rows) { + namespaceValue, rest, err := consumeByteView(namespaceColumn) + if err != nil { + _ = f.Close() + return Aggregate{}, err + } + namespaceColumn = rest + serviceValue, rest, err := consumeByteView(serviceColumn) + if err != nil { + _ = f.Close() + return Aggregate{}, err + } + serviceColumn = rest + statusValue, rest, err := consumeByteView(statusColumn) + if err != nil { + _ = f.Close() + return Aggregate{}, err + } + statusColumn = rest + timestamp := int64At(columns[colStartUnixNanos], row) + if timestamp < start || timestamp >= end { + continue + } + if namespace != "" && !bytes.Equal(namespaceValue, namespaceNeedle) { + continue + } + if service != "" && !bytes.Equal(serviceValue, serviceNeedle) { + continue + } + out.Calls++ + out.DurationMS += float64At(columns[colDurationMS], row) + if bytes.Equal(statusValue, errorNeedle) { + out.Errors++ + } + } + } + if err := f.Close(); err != nil { + return Aggregate{}, err + } + } + return out, nil +} + +func (s *Store) readTraceBlock(seg segment, blockID uint32, traceID string) ([]Span, error) { + if int(blockID) >= len(seg.blocks) { + return nil, fmt.Errorf("block %d out of range", blockID) + } + block := seg.blocks[blockID] + f, err := os.Open(seg.path) + if err != nil { + return nil, err + } + defer f.Close() + columns, err := s.readColumns(f, block, allColumns) + if err != nil { + return nil, err + } + selected, err := matchingStringRows(columns[colTraceID], int(block.rows), []byte(traceID)) + if err != nil { + return nil, err + } + return decodeSelectedBlock(columns, int(block.rows), selected) +} + +func (s *Store) readColumns(f *os.File, block blockDir, wanted []int) (map[int][]byte, error) { + encoded := make([]byte, block.length) + if _, err := f.ReadAt(encoded, int64(block.offset)); err != nil { + return nil, err + } + dec := s.decoders.Get().(*zstd.Decoder) + columns, err := decodeColumns(dec, encoded, wanted) + s.decoders.Put(dec) + return columns, err +} + +func openSegment(path string) (segment, error) { + f, err := os.Open(path) + if err != nil { + return segment{}, err + } + defer f.Close() + var header [headerSize]byte + if _, err := io.ReadFull(f, header[:]); err != nil { + return segment{}, err + } + if string(header[0:8]) != segmentMagic { + return segment{}, errors.New("invalid segment magic") + } + if binary.LittleEndian.Uint32(header[8:12]) != segmentVersion { + return segment{}, errors.New("unsupported segment version") + } + seg := segment{path: path, rows: binary.LittleEndian.Uint32(header[12:16]), min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40]))} + blockCount := binary.LittleEndian.Uint32(header[16:20]) + rollupCount := binary.LittleEndian.Uint32(header[20:24]) + dirOffset := binary.LittleEndian.Uint64(header[40:48]) + indexOffset := binary.LittleEndian.Uint64(header[48:56]) + rollupOffset := binary.LittleEndian.Uint64(header[56:64]) + info, err := f.Stat() + if err != nil { + return segment{}, err + } + seg.blocks = make([]blockDir, blockCount) + buf := make([]byte, int(blockCount)*blockDirSize) + if _, err := f.ReadAt(buf, int64(dirOffset)); err != nil { + return segment{}, err + } + for i := range seg.blocks { + b := buf[i*blockDirSize:] + seg.blocks[i] = blockDir{offset: binary.LittleEndian.Uint64(b[0:8]), length: binary.LittleEndian.Uint32(b[8:12]), rows: binary.LittleEndian.Uint32(b[12:16]), min: int64(binary.LittleEndian.Uint64(b[16:24])), max: int64(binary.LittleEndian.Uint64(b[24:32]))} + } + indexCompressed := make([]byte, int(rollupOffset-indexOffset)) + if _, err := f.ReadAt(indexCompressed, int64(indexOffset)); err != nil { + return segment{}, err + } + dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + if err != nil { + return segment{}, err + } + indexBytes, err := dec.DecodeAll(indexCompressed, nil) + if err != nil { + dec.Close() + return segment{}, fmt.Errorf("decode trace index: %w", err) + } + if len(indexBytes)%traceEntrySize != 0 { + dec.Close() + return segment{}, fmt.Errorf("trace index size %d is not entry-aligned", len(indexBytes)) + } + seg.traceIndex = make([]traceEntry, len(indexBytes)/traceEntrySize) + for i := range seg.traceIndex { + b := indexBytes[i*traceEntrySize:] + seg.traceIndex[i] = traceEntry{hash: binary.LittleEndian.Uint64(b[0:8]), block: binary.LittleEndian.Uint32(b[8:12]), row: binary.LittleEndian.Uint32(b[12:16])} + } + rollupCompressed := make([]byte, info.Size()-int64(rollupOffset)) + if _, err := f.ReadAt(rollupCompressed, int64(rollupOffset)); err != nil { + dec.Close() + return segment{}, err + } + rollupBytes, err := dec.DecodeAll(rollupCompressed, nil) + dec.Close() + if err != nil { + return segment{}, fmt.Errorf("decode rollups: %w", err) + } + reader := bufio.NewReader(bytes.NewReader(rollupBytes)) + seg.rollups = make([]rollup, 0, rollupCount) + for range rollupCount { + r, err := readRollup(reader) + if err != nil { + return segment{}, err + } + seg.rollups = append(seg.rollups, r) + } + return seg, nil +} + +func appendString(dst []byte, value string) []byte { + dst = binary.AppendUvarint(dst, uint64(len(value))) + return append(dst, value...) +} + +func consumeByteView(src []byte) ([]byte, []byte, error) { + length, n := binary.Uvarint(src) + if n <= 0 { + return nil, nil, errors.New("invalid string length") + } + src = src[n:] + if length > uint64(len(src)) { + return nil, nil, io.ErrUnexpectedEOF + } + return src[:length], src[length:], nil +} + +func writeRollup(w io.Writer, r rollup) error { + var fixed [160]byte + binary.LittleEndian.PutUint64(fixed[0:8], uint64(r.key.bucket)) + binary.LittleEndian.PutUint64(fixed[8:16], r.calls) + binary.LittleEndian.PutUint64(fixed[16:24], r.errors) + binary.LittleEndian.PutUint64(fixed[24:32], math.Float64bits(r.duration)) + for i, count := range r.bins { + binary.LittleEndian.PutUint32(fixed[32+i*4:], count) + } + if _, err := w.Write(fixed[:]); err != nil { + return fmt.Errorf("write rollup: %w", err) + } + for _, value := range []string{r.key.namespace, r.key.service, r.key.method, r.key.route} { + if len(value) > math.MaxUint16 { + return errors.New("rollup key exceeds 65535 bytes") + } + var length [2]byte + binary.LittleEndian.PutUint16(length[:], uint16(len(value))) + if _, err := w.Write(length[:]); err != nil { + return err + } + if _, err := io.WriteString(w, value); err != nil { + return err + } + } + return nil +} + +func readRollup(r *bufio.Reader) (rollup, error) { + var fixed [160]byte + if _, err := io.ReadFull(r, fixed[:]); err != nil { + return rollup{}, err + } + out := rollup{key: rollupKey{bucket: int64(binary.LittleEndian.Uint64(fixed[0:8]))}, calls: binary.LittleEndian.Uint64(fixed[8:16]), errors: binary.LittleEndian.Uint64(fixed[16:24]), duration: math.Float64frombits(binary.LittleEndian.Uint64(fixed[24:32]))} + for i := range out.bins { + out.bins[i] = binary.LittleEndian.Uint32(fixed[32+i*4:]) + } + for _, target := range []*string{&out.key.namespace, &out.key.service, &out.key.method, &out.key.route} { + var length [2]byte + if _, err := io.ReadFull(r, length[:]); err != nil { + return rollup{}, err + } + value := make([]byte, binary.LittleEndian.Uint16(length[:])) + if _, err := io.ReadFull(r, value); err != nil { + return rollup{}, err + } + *target = string(value) + } + return out, nil +} + +func durationBin(ms float64) int { + if ms <= 0 { + return 0 + } + bin := int(math.Log2(ms*1000 + 1)) + return min(bin, rollupBins-1) +} + +func histogramQuantile(bins [rollupBins]uint32, total uint64, q float64) float64 { + if total == 0 { + return 0 + } + target := uint64(math.Ceil(float64(total) * q)) + var seen uint64 + for i, count := range bins { + seen += uint64(count) + if seen >= target { + return (math.Pow(2, float64(i+1)) - 1) / 1000 + } + } + return (math.Pow(2, rollupBins) - 1) / 1000 +} + +func writeManifest(dir string, m manifest) error { + data, err := json.Marshal(m) + if err != nil { + return err + } + tmp := filepath.Join(dir, "MANIFEST.json.tmp") + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, filepath.Join(dir, "MANIFEST.json")); err != nil { + return err + } + return syncDir(dir) +} + +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +// DiskBytes returns the committed segment and manifest size. +func (s *Store) DiskBytes() (int64, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var total int64 + for _, name := range append(append([]string(nil), s.manifest.Files...), "MANIFEST.json") { + info, err := os.Stat(filepath.Join(s.dir, name)) + if errors.Is(err, os.ErrNotExist) && name == "MANIFEST.json" { + continue + } + if err != nil { + return 0, err + } + total += info.Size() + } + return total, nil +} + +// RowCount returns the number of committed rows. +func (s *Store) RowCount() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + var total uint64 + for i := range s.segments { + total += uint64(s.segments[i].rows) + } + return total +} + +func (s *Store) SegmentCount() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.segments) +} + +// EqualSpan compares every persisted field. It is intended for POC recovery +// and format-roundtrip validation, where nil and empty JSON are distinct. +func EqualSpan(a, b Span) bool { + return reflect.DeepEqual(a, b) +} diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go new file mode 100644 index 00000000..ff15523a --- /dev/null +++ b/internal/telemetry/segment/span_store_test.go @@ -0,0 +1,168 @@ +// Tests use the internal package to exercise crash boundaries and corruption. +package segment + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestStoreCommitReopenAndQueries(t *testing.T) { + dir := t.TempDir() + base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() + rows := []Span{ + {Namespace: "default", TraceID: "trace-a", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/users/:id", StartUnixNanos: base, EndUnixNanos: base + int64(10*time.Millisecond), DurationMS: 10, StatusCode: "OK", AttributesJSON: []byte(`{"tenant":"a"}`)}, + {Namespace: "default", TraceID: "trace-a", SpanID: "2", ParentSpanID: "1", ServiceName: "db", StartUnixNanos: base + int64(time.Millisecond), EndUnixNanos: base + int64(6*time.Millisecond), DurationMS: 5, StatusCode: "OK"}, + {Namespace: "default", TraceID: "trace-b", SpanID: "3", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/users/:id", StartUnixNanos: base + int64(time.Minute), EndUnixNanos: base + int64(time.Minute+50*time.Millisecond), DurationMS: 50, StatusCode: "ERROR"}, + } + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := store.Append(rows[:2]); err != nil { + t.Fatal(err) + } + if err := store.Append(rows[2:]); err != nil { + t.Fatal(err) + } + if got := store.RowCount(); got != 3 { + t.Fatalf("row count = %d", got) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + // Files left by an interrupted append are not present in the committed + // manifest and must not become visible after recovery. + if err := os.WriteFile(filepath.Join(dir, "999.fseg.tmp"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + store, err = Open(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if got := store.RowCount(); got != 3 { + t.Fatalf("reopened row count = %d", got) + } + trace, err := store.Trace("trace-a") + if err != nil { + t.Fatal(err) + } + if len(trace) != 2 || !EqualSpan(trace[0], rows[0]) || !EqualSpan(trace[1], rows[1]) { + t.Fatalf("trace result = %#v", trace) + } + endpoints := store.Endpoints("default", "api", base, base+int64(5*time.Minute), 10) + if len(endpoints) != 1 || endpoints[0].Calls != 2 || endpoints[0].Errors != 1 { + t.Fatalf("endpoint result = %#v", endpoints) + } + agg, err := store.ScanService("default", "api", base, base+int64(5*time.Minute)) + if err != nil { + t.Fatal(err) + } + if agg.Calls != 2 || agg.Errors != 1 || agg.DurationMS != 60 { + t.Fatalf("aggregate = %#v", agg) + } +} + +func TestStoreSkipsOrphanAndAdvancesSegmentID(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + base := time.Now().UnixNano() + row := Span{TraceID: "trace", SpanID: "span", StartUnixNanos: base} + if err := store.Append([]Span{row}); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + // Simulate a crash after publishing segment 2 but before its manifest commit. + orphan := filepath.Join(dir, "00000000000000000002.fseg") + committed := filepath.Join(dir, "00000000000000000001.fseg") + data, err := os.ReadFile(committed) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(orphan, data, 0o644); err != nil { + t.Fatal(err) + } + store, err = Open(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if got := store.RowCount(); got != 1 { + t.Fatalf("orphan became visible: row count = %d", got) + } + if err := store.Append([]Span{row}); err != nil { + t.Fatalf("append after orphan: %v", err) + } + if got := store.RowCount(); got != 2 { + t.Fatalf("row count after append = %d", got) + } + if _, err := os.Stat(filepath.Join(dir, "00000000000000000003.fseg")); err != nil { + t.Fatalf("allocator did not advance past orphan: %v", err) + } +} + +func TestStoreCompactionPreservesRowsAndIndexes(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + base := time.Now().UnixNano() + for batch := range 3 { + rows := []Span{ + {TraceID: "shared", SpanID: string(rune('a' + batch)), ServiceName: "api", StartUnixNanos: base + int64(batch), StatusCode: "OK"}, + {TraceID: "other", SpanID: string(rune('x' + batch)), ServiceName: "worker", StartUnixNanos: base + int64(batch+10), StatusCode: "ERROR"}, + } + if err := store.Append(rows); err != nil { + t.Fatal(err) + } + } + if err := store.CompactOldest(2); err != nil { + t.Fatal(err) + } + if got := store.SegmentCount(); got != 2 { + t.Fatalf("segments after compaction = %d", got) + } + if got := store.RowCount(); got != 6 { + t.Fatalf("rows after compaction = %d", got) + } + trace, err := store.Trace("shared") + if err != nil { + t.Fatal(err) + } + if len(trace) != 3 { + t.Fatalf("trace rows after compaction = %d", len(trace)) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = Open(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if got := store.RowCount(); got != 6 { + t.Fatalf("reopened compacted rows = %d", got) + } +} + +func TestStoreRejectsCorruptCommittedSegment(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "broken.fseg"), []byte("bad"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "MANIFEST.json"), []byte(`{"next_id":2,"files":["broken.fseg"]}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Open(dir); err == nil { + t.Fatal("Open succeeded with a corrupt committed segment") + } +} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go new file mode 100644 index 00000000..1e5e32d4 --- /dev/null +++ b/internal/telemetry/store/compaction.go @@ -0,0 +1,177 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +type compactionMarker struct { + ID string `json:"id"` + Inputs []string `json:"inputs"` + MaxNanos int64 `json:"max_nanos"` +} + +// CompactParquet combines the oldest small atomic batches into larger files. +// A durable marker makes the multi-signal swap recoverable after a crash. +func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int) (int, error) { + if db == nil || maxBatches < 2 { + return 0, nil + } + r.mu.Lock() + defer r.mu.Unlock() + if len(r.manifest.Batches) < 8 { + return 0, nil + } + count := min(maxBatches, len(r.manifest.Batches)) + selected := append([]batchMetadata(nil), r.manifest.Batches[:count]...) + marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano())} + for _, batch := range selected { + marker.Inputs = append(marker.Inputs, batch.ID) + marker.MaxNanos = max(marker.MaxNanos, batch.MaxNanos) + } + stageDir := filepath.Join(r.root, marker.ID) + if err := os.Mkdir(stageDir, 0o755); err != nil { + return 0, err + } + for _, signal := range []string{"spans", "logs", "metrics"} { + var inputs []string + for _, id := range marker.Inputs { + path := filepath.Join(r.Parquet.Dir(), signal, id+".parquet") + if _, err := os.Stat(path); err == nil { + inputs = append(inputs, path) + } else if !errors.Is(err, os.ErrNotExist) { + return 0, err + } + } + if len(inputs) == 0 { + continue + } + quoted := make([]string, len(inputs)) + for i, path := range inputs { + quoted[i] = sqlQuote(path) + } + output := filepath.Join(stageDir, signal+".parquet") + stmt := fmt.Sprintf("COPY (SELECT * FROM read_parquet([%s], union_by_name=true)) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, ROW_GROUP_SIZE 122880)", strings.Join(quoted, ","), sqlQuote(output)) + if _, err := db.ExecContext(ctx, stmt); err != nil { + return 0, fmt.Errorf("compact %s parquet: %w", signal, err) + } + } + data, err := json.Marshal(marker) + if err != nil { + return 0, err + } + if err := writeDurableFile(filepath.Join(r.root, "COMPACTION.json"), data); err != nil { + return 0, err + } + if err := syncDirectory(r.root); err != nil { + return 0, err + } + if err := r.completeCompaction(marker); err != nil { + return 0, err + } + return count, nil +} + +func (r *Repository) recoverCompaction() error { + data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var marker compactionMarker + if err := json.Unmarshal(data, &marker); err != nil { + return err + } + return r.completeCompaction(marker) +} + +func (r *Repository) completeCompaction(marker compactionMarker) error { + stageDir := filepath.Join(r.root, marker.ID) + for _, signal := range []string{"spans", "logs", "metrics"} { + dir := filepath.Join(r.Parquet.Dir(), signal) + for _, id := range marker.Inputs { + input := filepath.Join(dir, id+".parquet") + retired := input + ".retired-" + marker.ID + if _, err := os.Stat(input); err == nil { + if err := os.Rename(input, retired); err != nil { + return err + } + } + } + stage := filepath.Join(stageDir, signal+".parquet") + final := filepath.Join(dir, marker.ID+".parquet") + if _, err := os.Stat(stage); err == nil { + if err := os.Rename(stage, final); err != nil { + return err + } + } + } + if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { + return err + } + inputSet := make(map[string]struct{}, len(marker.Inputs)) + for _, id := range marker.Inputs { + inputSet[id] = struct{}{} + } + kept := make([]batchMetadata, 0, len(r.manifest.Batches)-len(marker.Inputs)+1) + for _, batch := range r.manifest.Batches { + if _, compacted := inputSet[batch.ID]; !compacted && batch.ID != marker.ID { + kept = append(kept, batch) + } + } + kept = append(kept, batchMetadata{ID: marker.ID, MaxNanos: marker.MaxNanos}) + next := repositoryManifest{Version: 1, Batches: kept} + if err := writeRepositoryManifest(r.root, next); err != nil { + return err + } + r.manifest = next + for _, signal := range []string{"spans", "logs", "metrics"} { + for _, id := range marker.Inputs { + _ = os.Remove(filepath.Join(r.Parquet.Dir(), signal, id+".parquet.retired-"+marker.ID)) + } + } + _ = os.RemoveAll(stageDir) + if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(r.root) +} + +func writeDurableFile(path string, data []byte) error { + tmp := path + ".tmp" + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + +func syncParquetDirectories(root string) error { + var err error + for _, signal := range []string{"spans", "logs", "metrics"} { + err = errors.Join(err, syncDirectory(filepath.Join(root, signal))) + } + return err +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go new file mode 100644 index 00000000..6a0c08c5 --- /dev/null +++ b/internal/telemetry/store/repository.go @@ -0,0 +1,401 @@ +// Package store owns Fanout's authoritative telemetry commit path: a +// replayable ingest WAL, immutable hot segments, and open Parquet files. +package store + +import ( + "bytes" + "encoding/gob" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/klauspost/compress/zstd" + "github.com/labstack/fanout/internal/telemetry" + "github.com/labstack/fanout/internal/telemetry/segment" +) + +type Batch struct { + ID string + Spans []telemetry.Span + Logs []telemetry.Log + Metrics []telemetry.Metric +} + +type batchMetadata struct { + ID string `json:"id"` + MaxNanos int64 `json:"max_nanos"` +} + +type repositoryManifest struct { + Version uint32 `json:"version"` + Batches []batchMetadata `json:"batches"` +} + +type Repository struct { + mu sync.RWMutex + root string + walDir string + Spans *segment.Store + Logs *segment.SignalStore[telemetry.Log] + Metrics *segment.SignalStore[telemetry.Metric] + Parquet *telemetry.ParquetStore + manifest repositoryManifest +} + +func Open(root string) (*Repository, error) { + walDir := filepath.Join(root, "wal") + for _, dir := range []string{root, walDir, filepath.Join(root, "hot", "spans"), filepath.Join(root, "hot", "logs"), filepath.Join(root, "hot", "metrics")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + } + spans, err := segment.Open(filepath.Join(root, "hot", "spans")) + if err != nil { + return nil, err + } + logs, err := segment.OpenSignalStore[telemetry.Log](filepath.Join(root, "hot", "logs"), "EventUnixNanos") + if err != nil { + _ = spans.Close() + return nil, err + } + metricsStore, err := segment.OpenSignalStore[telemetry.Metric](filepath.Join(root, "hot", "metrics"), "EventUnixNanos") + if err != nil { + _ = logs.Close() + _ = spans.Close() + return nil, err + } + parquet, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) + if err != nil { + _ = metricsStore.Close() + _ = logs.Close() + _ = spans.Close() + return nil, err + } + r := &Repository{root: root, walDir: walDir, Spans: spans, Logs: logs, Metrics: metricsStore, Parquet: parquet} + if err := r.loadManifest(); err != nil { + _ = r.Close() + return nil, fmt.Errorf("load telemetry manifest: %w", err) + } + if err := r.recoverCompaction(); err != nil { + _ = r.Close() + return nil, fmt.Errorf("recover parquet compaction: %w", err) + } + if err := r.recover(); err != nil { + _ = r.Close() + return nil, fmt.Errorf("recover telemetry WAL: %w", err) + } + return r, nil +} + +func (r *Repository) Close() error { + return errors.Join(r.Spans.Close(), r.Logs.Close(), r.Metrics.Close()) +} + +func (r *Repository) ReadLock() func() { + r.mu.RLock() + return r.mu.RUnlock +} + +// PruneHot removes acceleration segments older than cutoff. Parquet remains +// authoritative for longer retention and SQL queries. +func (r *Repository) PruneHot(cutoff int64) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + spans, spanErr := r.Spans.PruneBefore(cutoff) + logs, logErr := r.Logs.PruneBefore(cutoff) + metricRows, metricErr := r.Metrics.PruneBefore(cutoff) + return spans + logs + metricRows, errors.Join(spanErr, logErr, metricErr) +} + +// PruneParquet removes complete ingest batches older than cutoff. A batch that +// straddles the boundary is retained intact, so retention never removes newer +// telemetry from another signal in the same atomic commit. +func (r *Repository) PruneParquet(cutoff int64) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + kept := make([]batchMetadata, 0, len(r.manifest.Batches)) + removed := 0 + var removeErr error + for _, batch := range r.manifest.Batches { + if batch.MaxNanos <= 0 || batch.MaxNanos >= cutoff { + kept = append(kept, batch) + continue + } + batchOK := true + for _, signal := range []string{"spans", "logs", "metrics"} { + path := filepath.Join(r.Parquet.Dir(), signal, batch.ID+".parquet") + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + removeErr = errors.Join(removeErr, err) + batchOK = false + } + } + if batchOK { + removed++ + } else { + kept = append(kept, batch) + } + } + if removed == 0 { + return 0, removeErr + } + if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { + return 0, errors.Join(removeErr, err) + } + next := repositoryManifest{Version: 1, Batches: kept} + if err := writeRepositoryManifest(r.root, next); err != nil { + return 0, errors.Join(removeErr, err) + } + r.manifest = next + return removed, removeErr +} + +// Commit durably records a batch and publishes its three signal projections +// exactly once. A crash at any point leaves the WAL for replay on next boot. +func (r *Repository) Commit(batch Batch) error { + if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { + return errors.New("telemetry batch requires a safe ID") + } + normalizeBatch(&batch) + if err := r.writeWAL(batch); err != nil { + return err + } + r.mu.Lock() + err := r.apply(batch) + if err == nil { + err = r.recordBatch(batch) + } + r.mu.Unlock() + if err != nil { + return err + } + if err := os.Remove(filepath.Join(r.walDir, batch.ID+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove committed telemetry WAL: %w", err) + } + return syncDirectory(r.walDir) +} + +func (r *Repository) apply(batch Batch) error { + if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { + return fmt.Errorf("commit span segment: %w", err) + } + if err := r.Logs.Append(batch.ID, batch.Logs); err != nil { + return fmt.Errorf("commit log segment: %w", err) + } + if err := r.Metrics.Append(batch.ID, batch.Metrics); err != nil { + return fmt.Errorf("commit metric segment: %w", err) + } + if err := r.Parquet.WriteSpans(batch.ID, batch.Spans); err != nil { + return fmt.Errorf("commit span parquet: %w", err) + } + if err := r.Parquet.WriteLogs(batch.ID, batch.Logs); err != nil { + return fmt.Errorf("commit log parquet: %w", err) + } + if err := r.Parquet.WriteMetrics(batch.ID, batch.Metrics); err != nil { + return fmt.Errorf("commit metric parquet: %w", err) + } + return nil +} + +func (r *Repository) writeWAL(batch Batch) error { + final := filepath.Join(r.walDir, batch.ID+".wal") + if _, err := os.Stat(final); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + var plain bytes.Buffer + if err := gob.NewEncoder(&plain).Encode(batch); err != nil { + return err + } + enc, err := zstd.NewWriter(nil, zstd.WithEncoderCRC(true), zstd.WithEncoderConcurrency(1)) + if err != nil { + return err + } + data := enc.EncodeAll(plain.Bytes(), nil) + enc.Close() + tmp := final + ".tmp" + _ = os.Remove(tmp) + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + return err + } + return syncDirectory(r.walDir) +} + +func (r *Repository) recover() error { + entries, err := os.ReadDir(r.walDir) + if err != nil { + return err + } + var names []string + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".wal") { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + if err != nil { + return err + } + defer dec.Close() + for _, name := range names { + data, err := os.ReadFile(filepath.Join(r.walDir, name)) + if err != nil { + return err + } + plain, err := dec.DecodeAll(data, nil) + if err != nil { + return fmt.Errorf("decode %s: %w", name, err) + } + var batch Batch + if err := gob.NewDecoder(bytes.NewReader(plain)).Decode(&batch); err != nil { + return fmt.Errorf("read %s: %w", name, err) + } + if err := r.apply(batch); err != nil { + return fmt.Errorf("replay %s: %w", name, err) + } + if err := r.recordBatch(batch); err != nil { + return fmt.Errorf("record replayed %s: %w", name, err) + } + if err := os.Remove(filepath.Join(r.walDir, name)); err != nil { + return err + } + } + return syncDirectory(r.walDir) +} + +func (r *Repository) loadManifest() error { + path := filepath.Join(r.root, "MANIFEST.json") + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + r.manifest = repositoryManifest{Version: 1} + return writeRepositoryManifest(r.root, r.manifest) + } + if err != nil { + return err + } + if err := json.Unmarshal(data, &r.manifest); err != nil { + return err + } + if r.manifest.Version != 1 { + return fmt.Errorf("unsupported telemetry manifest version %d", r.manifest.Version) + } + return nil +} + +func (r *Repository) recordBatch(batch Batch) error { + for _, existing := range r.manifest.Batches { + if existing.ID == batch.ID { + return nil + } + } + next := r.manifest + next.Batches = append(append([]batchMetadata(nil), r.manifest.Batches...), batchMetadata{ID: batch.ID, MaxNanos: batchMaxNanos(batch)}) + if err := writeRepositoryManifest(r.root, next); err != nil { + return err + } + r.manifest = next + return nil +} + +func batchMaxNanos(batch Batch) int64 { + var maxNanos int64 + for _, row := range batch.Spans { + maxNanos = max(maxNanos, max(row.StartUnixNanos, row.IngestedAt)) + } + for _, row := range batch.Logs { + maxNanos = max(maxNanos, max(row.EventUnixNanos, row.IngestedAt)) + } + for _, row := range batch.Metrics { + maxNanos = max(maxNanos, max(row.EventUnixNanos, row.IngestedAt)) + } + return maxNanos +} + +func writeRepositoryManifest(root string, manifest repositoryManifest) error { + data, err := json.Marshal(manifest) + if err != nil { + return err + } + tmp := filepath.Join(root, "MANIFEST.json.tmp") + final := filepath.Join(root, "MANIFEST.json") + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + if err := os.Rename(tmp, final); err != nil { + return err + } + return syncDirectory(root) +} + +func normalizeBatch(batch *Batch) { + for i := range batch.Spans { + batch.Spans[i].Namespace = telemetry.NormalizeNamespace(batch.Spans[i].Namespace) + } + for i := range batch.Logs { + batch.Logs[i].Namespace = telemetry.NormalizeNamespace(batch.Logs[i].Namespace) + if batch.Logs[i].EventUnixNanos == 0 { + batch.Logs[i].EventUnixNanos = firstNonzero(batch.Logs[i].TimeUnixNanos, batch.Logs[i].ObservedTimeNanos, batch.Logs[i].IngestedAt) + } + } + for i := range batch.Metrics { + batch.Metrics[i].Namespace = telemetry.NormalizeNamespace(batch.Metrics[i].Namespace) + if batch.Metrics[i].EventUnixNanos == 0 { + batch.Metrics[i].EventUnixNanos = firstNonzero(batch.Metrics[i].TimeUnixNanos, batch.Metrics[i].IngestedAt) + } + } +} + +func firstNonzero(values ...int64) int64 { + for _, value := range values { + if value != 0 { + return value + } + } + return 0 +} + +func syncDirectory(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + if err := f.Sync(); err != nil && !errors.Is(err, io.ErrClosedPipe) { + return err + } + return nil +} diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go new file mode 100644 index 00000000..697cd4ce --- /dev/null +++ b/internal/telemetry/store/repository_test.go @@ -0,0 +1,176 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + + _ "github.com/duckdb/duckdb-go/v2" + "github.com/labstack/fanout/internal/telemetry" +) + +func testBatch() Batch { + return Batch{ + ID: "0198f4a0-test-batch", + Spans: []telemetry.Span{{Namespace: "", TraceID: "trace-1", SpanID: "span-1", ServiceName: "api", Name: "GET /", StartUnixNanos: 100, EndUnixNanos: 200, DurationMS: .0001, StatusCode: "OK", IngestedAt: 300}}, + Logs: []telemetry.Log{{TimeUnixNanos: 110, Severity: "INFO", Body: "ready", ServiceName: "api", TraceID: "trace-1", IngestedAt: 300}}, + Metrics: []telemetry.Metric{{TimeUnixNanos: 120, Name: "requests", Type: "sum", ServiceName: "api", Value: 1, IngestedAt: 300}}, + } +} + +func TestRepositoryCommitIsIdempotentAndQueryable(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + if got := repository.Spans.RowCount(); got != 1 { + t.Fatalf("span rows = %d", got) + } + if got := repository.Logs.RowCount(); got != 1 { + t.Fatalf("log rows = %d", got) + } + if got := repository.Metrics.RowCount(); got != 1 { + t.Fatalf("metric rows = %d", got) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, signal := range []string{"spans", "logs", "metrics"} { + var count int + pattern := filepath.ToSlash(filepath.Join(dir, "parquet", signal, "*.parquet")) + if err := db.QueryRowContext(context.Background(), "SELECT count(*) FROM read_parquet(?)", pattern).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("%s parquet rows = %d", signal, count) + } + } +} + +func TestRepositoryReplaysDurableWAL(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + if err := repository.writeWAL(batch); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + recovered, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if recovered.Spans.RowCount() != 1 || recovered.Logs.RowCount() != 1 || recovered.Metrics.RowCount() != 1 { + t.Fatal("WAL recovery did not restore every signal") + } + entries, err := filepath.Glob(filepath.Join(dir, "wal", "*.wal")) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("committed WAL files remain: %v", entries) + } +} + +func TestRepositoryPrunesOnlyCompleteExpiredParquetBatches(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + old := testBatch() + old.ID = "old-batch" + old.Spans[0].IngestedAt, old.Logs[0].IngestedAt, old.Metrics[0].IngestedAt = 100, 100, 100 + newer := testBatch() + newer.ID = "new-batch" + newer.Spans[0].IngestedAt, newer.Logs[0].IngestedAt, newer.Metrics[0].IngestedAt = 1000, 1000, 1000 + if err := repository.Commit(old); err != nil { + t.Fatal(err) + } + if err := repository.Commit(newer); err != nil { + t.Fatal(err) + } + removed, err := repository.PruneParquet(500) + if err != nil { + t.Fatal(err) + } + if removed != 1 { + t.Fatalf("removed batches = %d, want 1", removed) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, "old-batch.parquet")); !os.IsNotExist(err) { + t.Fatalf("expired %s file remains: %v", signal, err) + } + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, "new-batch.parquet")); err != nil { + t.Fatalf("new %s file missing: %v", signal, err) + } + } +} + +func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range 8 { + batch := testBatch() + batch.ID = fmt.Sprintf("batch-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + compacted, err := repository.CompactParquet(context.Background(), db, 64) + if err != nil { + t.Fatal(err) + } + if compacted != 8 { + t.Fatalf("compacted batches = %d, want 8", compacted) + } + stats, err := repository.Parquet.Stats() + if err != nil { + t.Fatal(err) + } + if stats["spans"].Files != 1 { + t.Fatalf("span files = %d, want 1", stats["spans"].Files) + } + pattern := filepath.ToSlash(filepath.Join(dir, "parquet", "spans", "*.parquet")) + var rows int + if err := db.QueryRow("SELECT count(*) FROM read_parquet(?)", pattern).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 8 { + t.Fatalf("compacted rows = %d, want 8", rows) + } +} diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go new file mode 100644 index 00000000..e9b17d01 --- /dev/null +++ b/internal/telemetry/store/writer.go @@ -0,0 +1,152 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/labstack/fanout/internal/metrics" + "github.com/labstack/fanout/internal/telemetry" +) + +const flushQueueDepth = 4 + +type Writer struct { + repository *Repository + interval time.Duration + batchSize int + spans <-chan telemetry.Span + logs <-chan telemetry.Log + metricRows <-chan telemetry.Metric + bufSpans []telemetry.Span + bufLogs []telemetry.Log + bufMetrics []telemetry.Metric + done chan struct{} +} + +func NewWriter(repository *Repository, interval time.Duration, batchSize int, spans <-chan telemetry.Span, logs <-chan telemetry.Log, metricRows <-chan telemetry.Metric) *Writer { + return &Writer{repository: repository, interval: interval, batchSize: batchSize, spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{})} +} + +func (w *Writer) Wait() { <-w.done } + +func (w *Writer) Run(ctx context.Context) error { + defer close(w.done) + flushes := make(chan Batch, flushQueueDepth) + workerDone := make(chan error, 1) + go w.flushWorker(flushes, workerDone) + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + spans, logs, metricRows := w.spans, w.logs, w.metricRows + finish := func() error { + w.drain(&spans, &logs, &metricRows) + w.flush(flushes) + close(flushes) + return <-workerDone + } + for { + select { + case row, ok := <-spans: + if !ok { + spans = nil + } else { + w.bufSpans = append(w.bufSpans, row) + metrics.RecordIngest("spans", 1) + } + case row, ok := <-logs: + if !ok { + logs = nil + } else { + w.bufLogs = append(w.bufLogs, row) + metrics.RecordIngest("logs", 1) + } + case row, ok := <-metricRows: + if !ok { + metricRows = nil + } else { + w.bufMetrics = append(w.bufMetrics, row) + metrics.RecordIngest("metrics", 1) + } + case <-ticker.C: + w.flush(flushes) + case <-ctx.Done(): + return finish() + } + if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) >= w.batchSize { + w.flush(flushes) + } + if spans == nil && logs == nil && metricRows == nil { + return finish() + } + } +} + +func (w *Writer) flush(out chan<- Batch) { + if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) == 0 { + return + } + batch := Batch{ID: uuid.NewString(), Spans: append([]telemetry.Span(nil), w.bufSpans...), Logs: append([]telemetry.Log(nil), w.bufLogs...), Metrics: append([]telemetry.Metric(nil), w.bufMetrics...)} + w.bufSpans = w.bufSpans[:0] + w.bufLogs = w.bufLogs[:0] + w.bufMetrics = w.bufMetrics[:0] + out <- batch +} + +func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { + var joined error + for batch := range in { + var err error + for attempt := 0; attempt < 3; attempt++ { + err = w.repository.Commit(batch) + if err == nil { + break + } + time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond) + } + if err != nil { + joined = errors.Join(joined, fmt.Errorf("commit batch %s: %w", batch.ID, err)) + } + } + done <- joined +} + +func (w *Writer) drain(spans *<-chan telemetry.Span, logs *<-chan telemetry.Log, metricRows *<-chan telemetry.Metric) { + for { + drained := false + select { + case row, ok := <-*spans: + if ok { + w.bufSpans = append(w.bufSpans, row) + drained = true + } else { + *spans = nil + } + default: + } + select { + case row, ok := <-*logs: + if ok { + w.bufLogs = append(w.bufLogs, row) + drained = true + } else { + *logs = nil + } + default: + } + select { + case row, ok := <-*metricRows: + if ok { + w.bufMetrics = append(w.bufMetrics, row) + drained = true + } else { + *metricRows = nil + } + default: + } + if !drained { + return + } + } +} diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index df1511ad..efd002ec 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -22,8 +22,8 @@ as a refusal to start rather than as a default nobody chose. | `storage.duckdb.max_connections` | `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | | `storage.duckdb.memory` | `FANOUT_DUCKDB_MEMORY` | string | — | | `storage.duckdb.threads` | `FANOUT_DUCKDB_THREADS` | integer | — | +| `storage.hot_retention` | `FANOUT_HOT_RETENTION` | duration | `24h` | | `storage.maintenance_interval` | `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | -| `storage.merge_interval` | `FANOUT_MERGE_INTERVAL` | duration | `1m` | | `storage.retention_days` | `FANOUT_RETENTION_DAYS` | integer | `30` | | `storage.rollup_interval` | `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | | `storage.rollup_skip_to_latest` | `FANOUT_ROLLUP_SKIP_TO_LATEST` | boolean | `false` | @@ -32,7 +32,7 @@ as a refusal to start rather than as a default nobody chose. ### `storage.duckdb.max_connections` -Caps the DuckDB connection pool. A value of 1 serializes everything through one handle; the machine-sized default lets read queries run concurrently with each other and with ingest flushes. Two things make >1 safe: the DuckLake SQLite catalog is opened in WAL mode (enableCatalogWAL), so readers don't collide with the single writer and a crashed writer can't leave the catalog permanently locked; and write commits are serialized by the shared write gate (Duck.WriteGate, wired into the writer via UseWriteGate in cmd/fanout/main.go, enforced at startup). Without the WAL mode, pool >1 fails with "database is locked". Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. +Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. ### `storage.duckdb.memory` @@ -42,13 +42,13 @@ Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected mem Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. -### `storage.maintenance_interval` +### `storage.hot_retention` -Throttles the DuckLake maintenance cycle (retention deletes + compaction). Default 1h. Lower it to compact more aggressively, or for soak tests that need to observe file-count staying bounded within minutes rather than hours. +Controls how long the custom indexed segments are retained. Older telemetry remains queryable in Parquet through DuckDB. -### `storage.merge_interval` +### `storage.maintenance_interval` -The cadence for the cheap, frequent DuckLake file compaction pass (ducklake_merge_adjacent_files only — it consolidates the newest small parquet files and deletes nothing). Run often (default 1m) it keeps the queryable file count continuously low, which is what bounds rollup/query scan latency — WITHOUT the churn, deletion race, or catalog cost of the full hourly maintenance pass (expire + cleanup). 0 disables it. +Controls hot-segment pruning, Parquet retention and compaction, and query-cache checkpointing. ### `storage.rollup_skip_to_latest` From 45bcae284cda59bcc4f8ecc14b0c866d22c9b6a4 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 14:25:43 -0700 Subject: [PATCH 02/31] fix(storage): harden telemetry persistence Fail fast on legacy DuckLake data instead of hiding it. Drain and publish compaction without blocking reads or commits, and surface failed writes immediately. --- internal/api/health.go | 8 +- internal/metrics/metrics.go | 2 +- internal/observability/logs.go | 62 ++++++++++-- internal/observability/service_test.go | 65 ++++++++++++ internal/observability/trace.go | 106 +++++++++++++++++++- internal/query/duck.go | 6 +- internal/telemetry/segment/signal_store.go | 32 ++++-- internal/telemetry/segment/span_store.go | 2 +- internal/telemetry/store/compaction.go | 51 +++++++++- internal/telemetry/store/repository.go | 53 ++++++++-- internal/telemetry/store/repository_test.go | 68 ++++++++++++- internal/telemetry/store/writer.go | 51 +++++++--- internal/telemetry/store/writer_test.go | 37 +++++++ 13 files changed, 485 insertions(+), 58 deletions(-) create mode 100644 internal/telemetry/store/writer_test.go diff --git a/internal/api/health.go b/internal/api/health.go index ca218c07..73bbb973 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -117,8 +117,8 @@ func (h *HealthHandler) checkDuckDB() CheckResult { } // diskDegradedPct is the free-space fraction below which the data dir reports -// degraded. A full disk silently fails Parquet flushes (the writer drops rows -// once its retry buffer overflows), so we surface pressure before that point. +// degraded. A full disk fails Parquet flushes and stops ingest, so we surface +// pressure before that point. const diskDegradedPct = 10.0 func (h *HealthHandler) checkDataDir() CheckResult { @@ -161,7 +161,7 @@ func diskSpaceResult(freeBytes, totalBytes uint64) CheckResult { } if freePct < diskDegradedPct { res.Status = "degraded" - res.Error = fmt.Sprintf("low disk space: %.1f%% free — ingest flushes drop rows when the disk fills", freePct) + res.Error = fmt.Sprintf("low disk space: %.1f%% free — ingest stops when telemetry cannot commit", freePct) } return res } @@ -190,7 +190,7 @@ func (h *HealthHandler) checkTelemetry() CheckResult { defer cancel() var one int - err := h.duck.DB.QueryRowContext(ctx, "SELECT 1 FROM lake.spans LIMIT 1").Scan(&one) + err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM lake.spans LIMIT 1") if err != nil && err != sql.ErrNoRows { return CheckResult{ Status: "unhealthy", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ded6483c..25821f00 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -76,7 +76,7 @@ var ( RowsDropped = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "fanout_rows_dropped_total", - Help: "Total rows dropped due to retry buffer overflow", + Help: "Total rows left uncommitted after a permanent flush failure", }, []string{"signal"}) WriteGateWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 4e8a5e37..172b0b5a 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -1,6 +1,7 @@ package observability import ( + "container/heap" "context" "fmt" "sort" @@ -10,6 +11,54 @@ import ( "github.com/labstack/fanout/internal/telemetry" ) +// newestLogHeap keeps its oldest entry at the root so a full-window scan only +// retains the newest bounded result set. +type newestLogHeap []LogEntry + +func (h newestLogHeap) Len() int { return len(h) } +func (h newestLogHeap) Less(i, j int) bool { return h[i].Time.Before(h[j].Time) } +func (h newestLogHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *newestLogHeap) Push(value any) { *h = append(*h, value.(LogEntry)) } +func (h *newestLogHeap) Pop() any { + old := *h + value := old[len(old)-1] + *h = old[:len(old)-1] + return value +} + +// earliestLogHeap keeps its newest entry at the root so trace correlation can +// retain the earliest bounded result set even when batches arrive out of order. +type earliestLogHeap []LogEntry + +func (h earliestLogHeap) Len() int { return len(h) } +func (h earliestLogHeap) Less(i, j int) bool { return h[i].Time.After(h[j].Time) } +func (h earliestLogHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *earliestLogHeap) Push(value any) { *h = append(*h, value.(LogEntry)) } +func (h *earliestLogHeap) Pop() any { + old := *h + value := old[len(old)-1] + *h = old[:len(old)-1] + return value +} + +func retainNewest(entries *newestLogHeap, entry LogEntry, limit int) { + if entries.Len() < limit { + heap.Push(entries, entry) + } else if entry.Time.After((*entries)[0].Time) { + (*entries)[0] = entry + heap.Fix(entries, 0) + } +} + +func retainEarliest(entries *earliestLogHeap, entry LogEntry, limit int) { + if entries.Len() < limit { + heap.Push(entries, entry) + } else if entry.Time.Before((*entries)[0].Time) { + (*entries)[0] = entry + heap.Fix(entries, 0) + } +} + func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, search string, limit int) (Result[Logs], error) { scope, err := s.normalizeScope(scope) if err != nil { @@ -27,7 +76,8 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear severity string } buckets := make(map[bucketKey]int64) - unlock := s.repository.ReadLock() + entries := newestLogHeap{} + matched := 0 err = s.repository.Logs.Scan(scope.Start.UnixNano(), scope.End.UnixNano(), func(row telemetry.Log) bool { select { case <-ctx.Done(): @@ -42,7 +92,8 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear return true } entryTime := time.Unix(0, row.EventUnixNanos).UTC() - data.Entries = append(data.Entries, LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) + matched++ + retainNewest(&entries, LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}, limit) bucketSeverity := strings.ToUpper(row.Severity) if bucketSeverity == "" { bucketSeverity = "UNSPECIFIED" @@ -51,17 +102,14 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear buckets[bucketKey{time: bucketNanos, severity: bucketSeverity}]++ return true }) - unlock() if err != nil { return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) } if err := ctx.Err(); err != nil { return Result[Logs]{}, err } + data.Entries = append(data.Entries, entries...) sort.Slice(data.Entries, func(i, j int) bool { return data.Entries[i].Time.After(data.Entries[j].Time) }) - if len(data.Entries) > limit { - data.Entries = data.Entries[:limit] - } for key, count := range buckets { data.Buckets = append(data.Buckets, LogBucket{Time: time.Unix(0, key.time).UTC(), Severity: key.severity, Count: count}) } @@ -72,7 +120,7 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear return data.Buckets[i].Time.Before(data.Buckets[j].Time) }) return Result[Logs]{ - Schema: LogsSchema, Summary: fmt.Sprintf("%d logs matched the selected telemetry window", len(data.Entries)), + Schema: LogsSchema, Summary: fmt.Sprintf("%d logs matched the selected telemetry window", matched), Data: data, Provenance: s.provenanceFor(scope, "fanout_segments"), }, nil } diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index e2437623..64777641 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -274,4 +274,69 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { } } +func TestLogsRetainsOnlyNewestLimit(t *testing.T) { + svc, _ := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + logs := make([]telemetry.Log, 100) + for i := range logs { + logs[i] = telemetry.Log{Namespace: "prod", TimeUnixNanos: start.Add(time.Duration(i) * time.Millisecond).UnixNano(), Severity: "INFO", Body: "entry"} + } + if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { + t.Fatal(err) + } + result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "", "", "", 5) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Entries) != 5 || !result.Data.Entries[0].Time.Equal(start.Add(99*time.Millisecond)) || !result.Data.Entries[4].Time.Equal(start.Add(95*time.Millisecond)) { + t.Fatalf("newest bounded entries = %#v", result.Data.Entries) + } +} + +func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { + svc, _ := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + batches := []telemetrystore.Batch{ + {ID: "trace-latest", Spans: []telemetry.Span{{Namespace: "prod", TraceID: "trace-order", SpanID: "root", StartUnixNanos: start.UnixNano(), DurationMS: 1}}, Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(30 * time.Millisecond).UnixNano(), Body: "latest"}}}, + {ID: "trace-earliest", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(10 * time.Millisecond).UnixNano(), Body: "earliest"}}}, + {ID: "trace-middle", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), Body: "middle"}}}, + } + for _, batch := range batches { + if err := svc.repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "trace-order", "", 2) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Logs) != 2 || result.Data.Logs[0].Body != "earliest" || result.Data.Logs[1].Body != "middle" { + t.Fatalf("trace logs = %#v", result.Data.Logs) + } +} + +func TestTraceFallsBackToParquetWhenHotSegmentsMiss(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). + WithArgs("cold-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). + AddRow("root", "", "checkout", "pay", "SERVER", start, 25.0, "ERROR", "declined")) + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("cold-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "cold-trace", "root")) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "cold-trace", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Spans) != 1 || len(result.Data.Logs) != 1 || !result.Data.HasError { + t.Fatalf("cold trace detail = %#v", result.Data) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + var _ DB = (*sql.DB)(nil) diff --git a/internal/observability/trace.go b/internal/observability/trace.go index fe0ed87f..d649e3e8 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -19,6 +19,21 @@ ORDER BY MAX(CASE WHEN upper(status) IN ('ERROR', 'STATUS_CODE_ERROR') THEN 1 EL MAX(end_time) - MIN(start_time) DESC LIMIT 1` +const traceSpansQuery = ` +SELECT span_id, coalesce(parent_span_id, ''), service, operation, kind, start_time, + duration_ms, status, coalesce(status_message, '') +FROM spans +WHERE trace_id = ? AND start_time >= ? AND start_time < ? AND (? = '' OR namespace = ?) +ORDER BY start_time ASC, duration_ms DESC +LIMIT ?` + +const traceLogsQuery = ` +SELECT time, severity, coalesce(service, ''), body, coalesce(trace_id, ''), coalesce(span_id, '') +FROM logs +WHERE trace_id = ? AND time >= ? AND time < ? AND (? = '' OR namespace = ?) +ORDER BY time ASC +LIMIT ?` + func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service string, limit int) (Result[TraceDetail], error) { scope, err := s.normalizeScope(scope) if err != nil { @@ -47,6 +62,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin rows.Close() } + dataSource := "fanout_segments" data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} if traceID != "" { unlock := s.repository.ReadLock() @@ -55,6 +71,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin unlock() return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } + unlock() startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() for _, row := range storedSpans { if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || @@ -97,23 +114,104 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin } sort.Strings(data.Services) + traceLogs := earliestLogHeap{} readErr = s.repository.Logs.Scan(startNanos, endNanos, func(row telemetry.Log) bool { + select { + case <-ctx.Done(): + return false + default: + } if row.TraceID != traceID || (scope.Namespace != "" && row.Namespace != scope.Namespace) { return true } - data.Logs = append(data.Logs, LogEntry{Time: time.Unix(0, row.EventUnixNanos).UTC(), Severity: row.Severity, Service: row.ServiceName, Body: redactLogBody(row.Body), TraceID: row.TraceID, SpanID: row.SpanID}) - return len(data.Logs) < limit + retainEarliest(&traceLogs, LogEntry{Time: time.Unix(0, row.EventUnixNanos).UTC(), Severity: row.Severity, Service: row.ServiceName, Body: redactLogBody(row.Body), TraceID: row.TraceID, SpanID: row.SpanID}, limit) + return true }) - unlock() if readErr != nil { return Result[TraceDetail]{}, fmt.Errorf("read trace logs: %w", readErr) } + if err := ctx.Err(); err != nil { + return Result[TraceDetail]{}, err + } + data.Logs = append(data.Logs, traceLogs...) sort.Slice(data.Logs, func(i, j int) bool { return data.Logs[i].Time.Before(data.Logs[j].Time) }) } + if traceID != "" && len(data.Spans) == 0 { + data, err = s.traceFromParquet(ctx, scope, traceID, limit) + if err != nil { + return Result[TraceDetail]{}, err + } + dataSource = "parquet" + } summary := "No traces found in this telemetry window" if traceID != "" { summary = fmt.Sprintf("Trace %s contains %d spans across %d services", traceID, len(data.Spans), len(data.Services)) } - return Result[TraceDetail]{Schema: TraceSchema, Summary: summary, Data: data, Provenance: s.provenanceFor(scope, "fanout_segments")}, nil + return Result[TraceDetail]{Schema: TraceSchema, Summary: summary, Data: data, Provenance: s.provenanceFor(scope, dataSource)}, nil +} + +func (s *Service) traceFromParquet(ctx context.Context, scope Scope, traceID string, limit int) (TraceDetail, error) { + data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} + rows, err := s.db.QueryContext(ctx, traceSpansQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) + if err != nil { + return TraceDetail{}, fmt.Errorf("query trace parquet spans: %w", err) + } + for rows.Next() { + var span TraceSpan + if err := rows.Scan(&span.SpanID, &span.ParentSpanID, &span.Service, &span.Operation, &span.Kind, &span.Start, &span.DurationMS, &span.Status, &span.StatusMessage); err != nil { + rows.Close() + return TraceDetail{}, fmt.Errorf("scan trace parquet span: %w", err) + } + data.Spans = append(data.Spans, span) + } + if err := rows.Err(); err != nil { + rows.Close() + return TraceDetail{}, fmt.Errorf("iterate trace parquet spans: %w", err) + } + rows.Close() + + serviceSet := make(map[string]struct{}) + var first, last time.Time + for _, span := range data.Spans { + if first.IsZero() || span.Start.Before(first) { + first = span.Start + } + if end := span.Start.Add(time.Duration(span.DurationMS * float64(time.Millisecond))); end.After(last) { + last = end + } + if strings.Contains(strings.ToUpper(span.Status), "ERROR") { + data.HasError = true + } + if span.Service != "" { + serviceSet[span.Service] = struct{}{} + } + } + if !first.IsZero() { + data.DurationMS = last.Sub(first).Seconds() * 1000 + } + for service := range serviceSet { + data.Services = append(data.Services, service) + } + sort.Strings(data.Services) + + rows, err = s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) + if err != nil { + return TraceDetail{}, fmt.Errorf("query trace parquet logs: %w", err) + } + for rows.Next() { + var entry LogEntry + if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { + rows.Close() + return TraceDetail{}, fmt.Errorf("scan trace parquet log: %w", err) + } + entry.Body = redactLogBody(entry.Body) + data.Logs = append(data.Logs, entry) + } + if err := rows.Err(); err != nil { + rows.Close() + return TraceDetail{}, fmt.Errorf("iterate trace parquet logs: %w", err) + } + rows.Close() + return data, nil } diff --git a/internal/query/duck.go b/internal/query/duck.go index 321e0304..51fbb8fb 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -418,14 +418,14 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { var pruneErr error if d.repository != nil { _, pruneErr = d.repository.PruneHot(cutoff) - d.parquetMu.Lock() var parquetErr error if d.cfg.RetentionDays > 0 { + d.parquetMu.Lock() _, parquetErr = d.repository.PruneParquet(time.Now().Add(-time.Duration(d.cfg.RetentionDays) * 24 * time.Hour).UnixNano()) + d.parquetMu.Unlock() } compactStart := time.Now() - compacted, compactErr := d.repository.CompactParquet(ctx, d.DB, 64) - d.parquetMu.Unlock() + compacted, compactErr := d.repository.CompactParquetBacklog(ctx, d.DB, 64, &d.parquetMu) compactResult := metrics.TelemetryNoop if compactErr != nil { compactResult = metrics.TelemetryError diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index 8734ae65..fb9608f2 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -256,43 +256,55 @@ func (s *SignalStore[T]) writeSegment(path, id string, rows []T) (signalSegment, // Scan visits rows in [start,end), using segment and block time pruning. func (s *SignalStore[T]) Scan(start, end int64, visit func(T) bool) error { + type openedSegment struct { + segment signalSegment + file *os.File + } + var opened []openedSegment s.mu.RLock() - defer s.mu.RUnlock() - dec := s.decoders.Get().(*zstd.Decoder) - defer s.decoders.Put(dec) for _, seg := range s.segments { if seg.max < start || seg.min >= end { continue } f, err := os.Open(seg.path) if err != nil { + s.mu.RUnlock() + for _, item := range opened { + _ = item.file.Close() + } return err } + opened = append(opened, openedSegment{segment: seg, file: f}) + } + s.mu.RUnlock() + defer func() { + for _, item := range opened { + _ = item.file.Close() + } + }() + dec := s.decoders.Get().(*zstd.Decoder) + defer s.decoders.Put(dec) + for _, item := range opened { + seg, f := item.segment, item.file for _, block := range seg.blocks { if block.max < start || block.min >= end { continue } data := make([]byte, block.length) if _, err := f.ReadAt(data, int64(block.offset)); err != nil { - _ = f.Close() return err } rows, err := s.codec.decodeBlock(dec, data, int(block.rows)) if err != nil { - _ = f.Close() return fmt.Errorf("decode %s: %w", filepath.Base(seg.path), err) } for _, row := range rows { ts := reflect.ValueOf(row).Field(s.timeField).Int() if ts >= start && ts < end && !visit(row) { - _ = f.Close() return nil } } } - if err := f.Close(); err != nil { - return err - } } return nil } @@ -333,7 +345,7 @@ func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { return 0, nil } next := current - next.Files = next.Files[:0] + next.Files = make([]string, 0, len(kept)) for _, seg := range kept { next.Files = append(next.Files, filepath.Base(seg.path)) } diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index c91673aa..14e271f3 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -358,7 +358,7 @@ func (s *Store) PruneBefore(cutoff int64) (int, error) { return 0, nil } next := current - next.Files = next.Files[:0] + next.Files = make([]string, 0, len(kept)) for _, seg := range kept { next.Files = append(next.Files, filepath.Base(seg.path)) } diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 1e5e32d4..f172001e 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" ) @@ -20,17 +21,20 @@ type compactionMarker struct { // CompactParquet combines the oldest small atomic batches into larger files. // A durable marker makes the multi-signal swap recoverable after a crash. -func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int) (int, error) { +func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { if db == nil || maxBatches < 2 { return 0, nil } - r.mu.Lock() - defer r.mu.Unlock() + r.compactionMu.Lock() + defer r.compactionMu.Unlock() + r.mu.RLock() if len(r.manifest.Batches) < 8 { + r.mu.RUnlock() return 0, nil } count := min(maxBatches, len(r.manifest.Batches)) selected := append([]batchMetadata(nil), r.manifest.Batches[:count]...) + r.mu.RUnlock() marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano())} for _, batch := range selected { marker.Inputs = append(marker.Inputs, batch.ID) @@ -40,6 +44,12 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if err := os.Mkdir(stageDir, 0o755); err != nil { return 0, err } + recoverable := false + defer func() { + if !recoverable { + _ = os.RemoveAll(stageDir) + } + }() for _, signal := range []string{"spans", "logs", "metrics"} { var inputs []string for _, id := range marker.Inputs { @@ -62,6 +72,9 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if _, err := db.ExecContext(ctx, stmt); err != nil { return 0, fmt.Errorf("compact %s parquet: %w", signal, err) } + if err := syncFile(output); err != nil { + return 0, fmt.Errorf("sync compacted %s parquet: %w", signal, err) + } } data, err := json.Marshal(marker) if err != nil { @@ -73,12 +86,44 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if err := syncDirectory(r.root); err != nil { return 0, err } + recoverable = true + if publishLock != nil { + publishLock.Lock() + defer publishLock.Unlock() + } + r.mu.Lock() + defer r.mu.Unlock() if err := r.completeCompaction(marker); err != nil { return 0, err } return count, nil } +// CompactParquetBacklog drains every currently eligible compaction group so a +// maintenance interval cannot create files faster than it retires them. +func (r *Repository) CompactParquetBacklog(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { + total := 0 + for { + compacted, err := r.CompactParquet(ctx, db, maxBatches, publishLock) + total += compacted + if err != nil || compacted == 0 { + return total, err + } + } +} + +func syncFile(path string) error { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + func (r *Repository) recoverCompaction() error { data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) if errors.Is(err, os.ErrNotExist) { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 6a0c08c5..85b1e6bb 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -9,11 +9,13 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "sort" "strings" "sync" + "time" "github.com/klauspost/compress/zstd" "github.com/labstack/fanout/internal/telemetry" @@ -38,17 +40,24 @@ type repositoryManifest struct { } type Repository struct { - mu sync.RWMutex - root string - walDir string - Spans *segment.Store - Logs *segment.SignalStore[telemetry.Log] - Metrics *segment.SignalStore[telemetry.Metric] - Parquet *telemetry.ParquetStore - manifest repositoryManifest + mu sync.RWMutex + compactionMu sync.Mutex + root string + walDir string + Spans *segment.Store + Logs *segment.SignalStore[telemetry.Log] + Metrics *segment.SignalStore[telemetry.Metric] + Parquet *telemetry.ParquetStore + manifest repositoryManifest } func Open(root string) (*Repository, error) { + legacyCatalog := filepath.Join(root, "ducklake.sqlite") + if _, err := os.Stat(legacyCatalog); err == nil { + return nil, fmt.Errorf("legacy DuckLake catalog %s is unsupported; start Fanout with a clean storage.data_dir", legacyCatalog) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("inspect legacy DuckLake catalog: %w", err) + } walDir := filepath.Join(root, "wal") for _, dir := range []string{root, walDir, filepath.Join(root, "hot", "spans"), filepath.Join(root, "hot", "logs"), filepath.Join(root, "hot", "metrics")} { if err := os.MkdirAll(dir, 0o755); err != nil { @@ -266,11 +275,17 @@ func (r *Repository) recover() error { } plain, err := dec.DecodeAll(data, nil) if err != nil { - return fmt.Errorf("decode %s: %w", name, err) + if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { + return quarantineErr + } + continue } var batch Batch if err := gob.NewDecoder(bytes.NewReader(plain)).Decode(&batch); err != nil { - return fmt.Errorf("read %s: %w", name, err) + if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { + return quarantineErr + } + continue } if err := r.apply(batch); err != nil { return fmt.Errorf("replay %s: %w", name, err) @@ -285,6 +300,24 @@ func (r *Repository) recover() error { return syncDirectory(r.walDir) } +func quarantineWAL(dir, name string, cause error) error { + source := filepath.Join(dir, name) + target := source + ".corrupt" + if _, err := os.Stat(target); err == nil { + target = fmt.Sprintf("%s.%d", target, time.Now().UnixNano()) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect WAL quarantine target: %w", err) + } + if err := os.Rename(source, target); err != nil { + return fmt.Errorf("quarantine corrupt WAL %s: %w", name, err) + } + if err := syncDirectory(dir); err != nil { + return fmt.Errorf("sync WAL quarantine: %w", err) + } + slog.Error("quarantined corrupt telemetry WAL", "file", filepath.Base(target), "error", cause) + return nil +} + func (r *Repository) loadManifest() error { path := filepath.Join(r.root, "MANIFEST.json") data, err := os.ReadFile(path) diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 697cd4ce..76c90e45 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" _ "github.com/duckdb/duckdb-go/v2" @@ -95,6 +96,39 @@ func TestRepositoryReplaysDurableWAL(t *testing.T) { } } +func TestRepositoryRejectsLegacyDuckLakeCatalog(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "ducklake.sqlite"), []byte("legacy"), 0o600); err != nil { + t.Fatal(err) + } + _, err := Open(dir) + if err == nil || !strings.Contains(err.Error(), "clean storage.data_dir") { + t.Fatalf("Open error = %v, want explicit clean-data-dir failure", err) + } +} + +func TestRepositoryQuarantinesCorruptWAL(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "wal", "poison.wal"), []byte("not-zstd"), 0o600); err != nil { + t.Fatal(err) + } + recovered, err := Open(dir) + if err != nil { + t.Fatalf("Open with poison WAL: %v", err) + } + defer recovered.Close() + if _, err := os.Stat(filepath.Join(dir, "wal", "poison.wal.corrupt")); err != nil { + t.Fatalf("quarantined WAL missing: %v", err) + } +} + func TestRepositoryPrunesOnlyCompleteExpiredParquetBatches(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -151,7 +185,7 @@ func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { t.Fatal(err) } defer db.Close() - compacted, err := repository.CompactParquet(context.Background(), db, 64) + compacted, err := repository.CompactParquet(context.Background(), db, 64, nil) if err != nil { t.Fatal(err) } @@ -174,3 +208,35 @@ func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { t.Fatalf("compacted rows = %d, want 8", rows) } } + +func TestRepositoryCompactionDrainsBacklog(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range 72 { + batch := testBatch() + batch.ID = fmt.Sprintf("backlog-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + compacted, err := repository.CompactParquetBacklog(context.Background(), db, 64, nil) + if err != nil { + t.Fatal(err) + } + if compacted <= 64 { + t.Fatalf("compacted batches = %d, want multiple groups", compacted) + } + if len(repository.manifest.Batches) != 1 { + t.Fatalf("manifest batches = %d, want 1", len(repository.manifest.Batches)) + } +} diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index e9b17d01..9cb0f27b 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -2,8 +2,8 @@ package store import ( "context" - "errors" "fmt" + "log/slog" "time" "github.com/google/uuid" @@ -13,8 +13,12 @@ import ( const flushQueueDepth = 4 +type batchCommitter interface { + Commit(Batch) error +} + type Writer struct { - repository *Repository + repository batchCommitter interval time.Duration batchSize int spans <-chan telemetry.Span @@ -42,7 +46,9 @@ func (w *Writer) Run(ctx context.Context) error { spans, logs, metricRows := w.spans, w.logs, w.metricRows finish := func() error { w.drain(&spans, &logs, &metricRows) - w.flush(flushes) + if err := w.flush(flushes, workerDone); err != nil { + return err + } close(flushes) return <-workerDone } @@ -70,12 +76,18 @@ func (w *Writer) Run(ctx context.Context) error { metrics.RecordIngest("metrics", 1) } case <-ticker.C: - w.flush(flushes) + if err := w.flush(flushes, workerDone); err != nil { + return err + } case <-ctx.Done(): return finish() + case err := <-workerDone: + return err } if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) >= w.batchSize { - w.flush(flushes) + if err := w.flush(flushes, workerDone); err != nil { + return err + } } if spans == nil && logs == nil && metricRows == nil { return finish() @@ -83,19 +95,23 @@ func (w *Writer) Run(ctx context.Context) error { } } -func (w *Writer) flush(out chan<- Batch) { +func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) == 0 { - return + return nil } batch := Batch{ID: uuid.NewString(), Spans: append([]telemetry.Span(nil), w.bufSpans...), Logs: append([]telemetry.Log(nil), w.bufLogs...), Metrics: append([]telemetry.Metric(nil), w.bufMetrics...)} - w.bufSpans = w.bufSpans[:0] - w.bufLogs = w.bufLogs[:0] - w.bufMetrics = w.bufMetrics[:0] - out <- batch + select { + case out <- batch: + w.bufSpans = w.bufSpans[:0] + w.bufLogs = w.bufLogs[:0] + w.bufMetrics = w.bufMetrics[:0] + return nil + case err := <-workerDone: + return err + } } func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { - var joined error for batch := range in { var err error for attempt := 0; attempt < 3; attempt++ { @@ -106,10 +122,17 @@ func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond) } if err != nil { - joined = errors.Join(joined, fmt.Errorf("commit batch %s: %w", batch.ID, err)) + commitErr := fmt.Errorf("commit batch %s: %w", batch.ID, err) + metrics.FlushErrors.WithLabelValues("batch").Inc() + metrics.RowsDropped.WithLabelValues("spans").Add(float64(len(batch.Spans))) + metrics.RowsDropped.WithLabelValues("logs").Add(float64(len(batch.Logs))) + metrics.RowsDropped.WithLabelValues("metrics").Add(float64(len(batch.Metrics))) + slog.Error("telemetry batch commit failed permanently", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) + done <- commitErr + return } } - done <- joined + done <- nil } func (w *Writer) drain(spans *<-chan telemetry.Span, logs *<-chan telemetry.Log, metricRows *<-chan telemetry.Metric) { diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go new file mode 100644 index 00000000..7e270bc2 --- /dev/null +++ b/internal/telemetry/store/writer_test.go @@ -0,0 +1,37 @@ +package store + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/labstack/fanout/internal/telemetry" +) + +type failingCommitter struct{} + +func (failingCommitter) Commit(Batch) error { return errors.New("disk full") } + +func TestWriterSurfacesPermanentCommitFailure(t *testing.T) { + spans := make(chan telemetry.Span, 1) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} + close(spans) + close(logs) + close(metricRows) + w := &Writer{ + repository: failingCommitter{}, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + } + start := time.Now() + err := w.Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("Run error = %v, want permanent commit failure", err) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("commit failure surfaced after %s", elapsed) + } +} From d382a301d97ea597479c490e8fa5dd0e1803edfa Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 15:00:20 -0700 Subject: [PATCH 03/31] fix(storage): harden maintenance concurrency Keep failed ingest batches backpressured until durable commit. Use day-partitioned leveled compaction so retention remains enforceable without rewriting the retained corpus. Hold Parquet snapshot locks through row iteration and bound hot-segment descriptors. --- internal/observability/namespace_test.go | 2 +- .../observability/performance_rollup_test.go | 2 +- internal/observability/service.go | 13 +- internal/observability/service_test.go | 6 +- internal/observability/trace.go | 3 - internal/query/duck.go | 78 +++++++++-- internal/query/duck_test.go | 43 ++++++ internal/queryrows/rows.go | 32 +++++ internal/telemetry/segment/signal_store.go | 71 ++++++---- .../telemetry/segment/signal_store_test.go | 68 ++++++++++ internal/telemetry/store/compaction.go | 79 ++++++++++-- internal/telemetry/store/repository.go | 51 ++++++-- internal/telemetry/store/repository_test.go | 122 +++++++++++++++++- internal/telemetry/store/writer.go | 36 ++++-- internal/telemetry/store/writer_test.go | 39 ++++-- 15 files changed, 544 insertions(+), 101 deletions(-) create mode 100644 internal/queryrows/rows.go create mode 100644 internal/telemetry/segment/signal_store_test.go diff --git a/internal/observability/namespace_test.go b/internal/observability/namespace_test.go index 0b9092c7..03c2940d 100644 --- a/internal/observability/namespace_test.go +++ b/internal/observability/namespace_test.go @@ -38,7 +38,7 @@ CREATE TABLE service_rollup ( t.Fatalf("insert service rollups: %v", err) } - svc := New(db, newTestRepository(t)) + svc := New(SQLDB(db), newTestRepository(t)) result, err := svc.Overview(context.Background(), Scope{Start: stamp.Add(-time.Minute), End: stamp.Add(time.Minute)}, 100) if err != nil { t.Fatalf("Overview: %v", err) diff --git a/internal/observability/performance_rollup_test.go b/internal/observability/performance_rollup_test.go index 831da3fa..0123ad67 100644 --- a/internal/observability/performance_rollup_test.go +++ b/internal/observability/performance_rollup_test.go @@ -90,7 +90,7 @@ FROM (VALUES ` + seed.values + `) t(ms)` t.Fatalf("seed endpoint rollup state: %v", err) } - svc := New(db, newTestRepository(t)) + svc := New(SQLDB(db), newTestRepository(t)) svc.endpointMature.Store(true) var cachedCalls, totalCachedCalls int64 var minBucket, maxBucket time.Time diff --git a/internal/observability/service.go b/internal/observability/service.go index 37f3e90d..6c7ee585 100644 --- a/internal/observability/service.go +++ b/internal/observability/service.go @@ -1,8 +1,6 @@ package observability import ( - "context" - "database/sql" "errors" "fmt" "strings" @@ -10,6 +8,7 @@ import ( "time" appid "github.com/labstack/fanout/internal/id" + "github.com/labstack/fanout/internal/queryrows" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -25,11 +24,7 @@ var ( ErrInvalidLimit = errors.New("invalid observability query limit") ) -// DB is the narrow database surface used by the query kernel. *sql.DB -// satisfies it, while tests can use sqlmock without a storage-specific fake. -type DB interface { - QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) -} +type DB = queryrows.Queryer type Service struct { db DB @@ -45,6 +40,10 @@ func New(db DB, repository *telemetrystore.Repository) *Service { return &Service{db: db, repository: repository, now: time.Now} } +// SQLDB adapts a standard database/sql queryer for tests and callers that do +// not need storage-engine row-lifetime hooks. +func SQLDB(db queryrows.SQLQueryer) DB { return queryrows.SQLAdapter{DB: db} } + func (s *Service) normalizeScope(scope Scope) (Scope, error) { now := s.now().UTC() if scope.End.IsZero() { diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 64777641..c8a1db91 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -2,13 +2,13 @@ package observability import ( "context" - "database/sql" "errors" "regexp" "testing" "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/fanout/internal/queryrows" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -30,7 +30,7 @@ func newMockService(t *testing.T) (*Service, sqlmock.Sqlmock) { t.Fatalf("sqlmock.New: %v", err) } t.Cleanup(func() { _ = db.Close() }) - svc := New(db, newTestRepository(t)) + svc := New(SQLDB(db), newTestRepository(t)) svc.now = func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) } return svc, mock } @@ -339,4 +339,4 @@ func TestTraceFallsBackToParquetWhenHotSegmentsMiss(t *testing.T) { } } -var _ DB = (*sql.DB)(nil) +var _ DB = queryrows.SQLAdapter{} diff --git a/internal/observability/trace.go b/internal/observability/trace.go index d649e3e8..abde8673 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -65,13 +65,10 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin dataSource := "fanout_segments" data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} if traceID != "" { - unlock := s.repository.ReadLock() storedSpans, readErr := s.repository.Spans.Trace(traceID) if readErr != nil { - unlock() return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } - unlock() startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() for _, row := range storedSpans { if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || diff --git a/internal/query/duck.go b/internal/query/duck.go index 51fbb8fb..c253808c 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -18,6 +18,7 @@ import ( "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/metrics" "github.com/labstack/fanout/internal/query/writegate" + "github.com/labstack/fanout/internal/queryrows" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -344,6 +345,12 @@ func (d *Duck) RunRollups(ctx context.Context) { slog.Info("startup rollup complete", "rows", rows, "duration", time.Since(start)) } d.updateParquetStats() + maintenanceDone := make(chan struct{}) + go func() { + defer close(maintenanceDone) + d.runMaintenanceLoop(ctx) + }() + defer func() { <-maintenanceDone }() ticker := time.NewTicker(d.cfg.RollupInterval) defer ticker.Stop() @@ -397,13 +404,33 @@ func (d *Duck) rollupOnce(ctx context.Context) (int, error) { affected += n } - if err := d.runRepositoryMaintenance(ctx); err != nil { - slog.Warn("telemetry maintenance failed", "err", err) - } - return int(affected), errors.Join(errs...) } +func (d *Duck) runMaintenanceLoop(ctx context.Context) { + every := d.cfg.MaintenanceInterval + if every <= 0 { + every = time.Hour + } + run := func() { + if err := d.runRepositoryMaintenance(ctx); err != nil && ctx.Err() == nil { + slog.Warn("telemetry maintenance failed", "err", err) + } + d.updateParquetStats() + } + run() + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ticker.C: + run() + case <-ctx.Done(): + return + } + } +} + func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { every := d.cfg.MaintenanceInterval if every <= 0 { @@ -1339,13 +1366,38 @@ FROM messaging_edges;` // QueryContext executes a read against immutable Parquet files and DuckDB's // local rollup cache. -func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { +func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { d.parquetMu.RLock() rows, err := d.DB.QueryContext(ctx, query, args...) - d.parquetMu.RUnlock() - return rows, err + if err != nil { + d.parquetMu.RUnlock() + return nil, err + } + return &lockedRows{Rows: rows, unlock: d.parquetMu.RUnlock}, nil +} + +type lockedRows struct { + *sql.Rows + unlockOnce sync.Once + unlock func() } +func (r *lockedRows) Close() error { + err := r.Rows.Close() + r.release() + return err +} + +func (r *lockedRows) Next() bool { + ok := r.Rows.Next() + if !ok { + r.release() + } + return ok +} + +func (r *lockedRows) release() { r.unlockOnce.Do(r.unlock) } + // QueryRowScan executes a single-row query against immutable Parquet files and // DuckDB's local rollup cache. func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args ...any) error { @@ -1379,7 +1431,7 @@ HAVING SUM(spans) > 0 ORDER BY p95_ms DESC LIMIT 100; `, windowMinutes) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace) + rows, err := d.QueryContext(ctx, q, namespace, namespace) if err != nil { return nil, err } @@ -1417,7 +1469,7 @@ WHERE time >= now() - INTERVAL %d MINUTE ORDER BY time DESC LIMIT %d; `, windowMinutes, limit) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace, pattern) + rows, err := d.QueryContext(ctx, q, namespace, namespace, pattern) if err != nil { return nil, err } @@ -1449,7 +1501,7 @@ WHERE bucket >= now() - INTERVAL %d MINUTE GROUP BY bucket ORDER BY bucket ASC; `, windowMinutes) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace) + rows, err := d.QueryContext(ctx, q, namespace, namespace) if err != nil { return nil, err } @@ -1481,7 +1533,7 @@ GROUP BY service ORDER BY spans_per_minute DESC LIMIT 20; `, windowMinutes, windowMinutes) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace) + rows, err := d.QueryContext(ctx, q, namespace, namespace) if err != nil { return nil, err } @@ -1514,7 +1566,7 @@ GROUP BY body ORDER BY count DESC LIMIT %d; `, windowMinutes, limit) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace) + rows, err := d.QueryContext(ctx, q, namespace, namespace) if err != nil { return nil, err } @@ -1556,7 +1608,7 @@ HAVING errors > 0 ORDER BY errors DESC LIMIT %d; `, windowMinutes, limit) - rows, err := d.DB.QueryContext(ctx, q, namespace, namespace) + rows, err := d.QueryContext(ctx, q, namespace, namespace) if err != nil { return nil, err } diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 4beea3f0..cb5d73eb 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -148,6 +148,49 @@ func TestNewDuckUsesSingleConnectionPool(t *testing.T) { } } +func TestQueryContextHoldsParquetLockUntilRowsFinish(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT 1").WillReturnRows(sqlmock.NewRows([]string{"value"}).AddRow(1)) + d := &Duck{DB: db} + rows, err := d.QueryContext(context.Background(), "SELECT 1") + if err != nil { + t.Fatalf("QueryContext: %v", err) + } + lockAcquired := make(chan struct{}) + go func() { + d.parquetMu.Lock() + close(lockAcquired) + d.parquetMu.Unlock() + }() + select { + case <-lockAcquired: + t.Fatal("Parquet write lock acquired while query rows were still open") + case <-time.After(25 * time.Millisecond): + } + if !rows.Next() { + t.Fatalf("rows.Next() = false: %v", rows.Err()) + } + var value int + if err := rows.Scan(&value); err != nil { + t.Fatalf("rows.Scan: %v", err) + } + if rows.Next() { + t.Fatal("rows.Next() returned an unexpected second row") + } + select { + case <-lockAcquired: + case <-time.After(time.Second): + t.Fatal("Parquet write lock remained blocked after row iteration completed") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestDuckDSN(t *testing.T) { tests := []struct { name string diff --git a/internal/queryrows/rows.go b/internal/queryrows/rows.go new file mode 100644 index 00000000..265489c2 --- /dev/null +++ b/internal/queryrows/rows.go @@ -0,0 +1,32 @@ +package queryrows + +import ( + "context" + "database/sql" +) + +// Rows is the database row-stream surface used by query consumers. Keeping it +// as an interface lets storage engines attach resource-lifetime behavior (for +// example, retaining a Parquet snapshot lock until iteration is complete). +type Rows interface { + Close() error + Columns() ([]string, error) + Err() error + Next() bool + Scan(dest ...any) error +} + +type Queryer interface { + QueryContext(ctx context.Context, query string, args ...any) (Rows, error) +} + +// SQLQueryer is implemented by *sql.DB and sqlmock's database handle. +type SQLQueryer interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +type SQLAdapter struct{ DB SQLQueryer } + +func (a SQLAdapter) QueryContext(ctx context.Context, query string, args ...any) (Rows, error) { + return a.DB.QueryContext(ctx, query, args...) +} diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index fb9608f2..107c4e9c 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -51,6 +51,11 @@ type signalManifest struct { Files []string `json:"files"` } +type signalFile interface { + ReadAt([]byte, int64) (int, error) + Close() error +} + // SignalStore persists one telemetry signal as immutable, independently // compressed columns. T must be a struct containing only string, []byte, // int32, uint32, int64, and float64 fields. @@ -64,6 +69,7 @@ type SignalStore[T any] struct { segments []signalSegment encoder *zstd.Encoder decoders sync.Pool + openFile func(string) (signalFile, error) } func OpenSignalStore[T any](dir, timeField string) (*SignalStore[T], error) { @@ -82,7 +88,7 @@ func OpenSignalStore[T any](dir, timeField string) (*SignalStore[T], error) { if err != nil { return nil, fmt.Errorf("create signal encoder: %w", err) } - s := &SignalStore[T]{dir: dir, timeField: field.Index[0], codec: codec, encoder: enc} + s := &SignalStore[T]{dir: dir, timeField: field.Index[0], codec: codec, encoder: enc, openFile: func(path string) (signalFile, error) { return os.Open(path) }} s.decoders.New = func() any { dec, decErr := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) if decErr != nil { @@ -256,54 +262,73 @@ func (s *SignalStore[T]) writeSegment(path, id string, rows []T) (signalSegment, // Scan visits rows in [start,end), using segment and block time pruning. func (s *SignalStore[T]) Scan(start, end int64, visit func(T) bool) error { - type openedSegment struct { - segment signalSegment - file *os.File - } - var opened []openedSegment s.mu.RLock() + segments := make([]signalSegment, 0, len(s.segments)) for _, seg := range s.segments { if seg.max < start || seg.min >= end { continue } - f, err := os.Open(seg.path) - if err != nil { - s.mu.RUnlock() - for _, item := range opened { - _ = item.file.Close() - } - return err - } - opened = append(opened, openedSegment{segment: seg, file: f}) + segments = append(segments, seg) } s.mu.RUnlock() - defer func() { - for _, item := range opened { - _ = item.file.Close() - } - }() dec := s.decoders.Get().(*zstd.Decoder) defer s.decoders.Put(dec) - for _, item := range opened { - seg, f := item.segment, item.file + for _, seg := range segments { + // Revalidate and open while holding the metadata read lock. Pruning must + // acquire the write lock before unlinking a retired segment, so once this + // descriptor is open the scan can safely release the lock and decode it. + s.mu.RLock() + active := false + for _, current := range s.segments { + if current.path == seg.path { + active = true + break + } + } + if !active { + s.mu.RUnlock() + continue + } + openFile := s.openFile + if openFile == nil { + openFile = func(path string) (signalFile, error) { return os.Open(path) } + } + f, err := openFile(seg.path) + s.mu.RUnlock() + if err != nil { + return err + } + stop := false for _, block := range seg.blocks { if block.max < start || block.min >= end { continue } data := make([]byte, block.length) if _, err := f.ReadAt(data, int64(block.offset)); err != nil { + _ = f.Close() return err } rows, err := s.codec.decodeBlock(dec, data, int(block.rows)) if err != nil { + _ = f.Close() return fmt.Errorf("decode %s: %w", filepath.Base(seg.path), err) } for _, row := range rows { ts := reflect.ValueOf(row).Field(s.timeField).Int() if ts >= start && ts < end && !visit(row) { - return nil + stop = true + break } } + if stop { + break + } + } + if err := f.Close(); err != nil { + return err + } + if stop { + return nil } } return nil diff --git a/internal/telemetry/segment/signal_store_test.go b/internal/telemetry/segment/signal_store_test.go new file mode 100644 index 00000000..979b1e12 --- /dev/null +++ b/internal/telemetry/segment/signal_store_test.go @@ -0,0 +1,68 @@ +package segment + +import ( + "os" + "sync/atomic" + "testing" + + "github.com/labstack/fanout/internal/telemetry" +) + +type countedSignalFile struct { + *os.File + open *atomic.Int64 +} + +func (f *countedSignalFile) Close() error { + f.open.Add(-1) + return f.File.Close() +} + +func TestSignalStoreScanKeepsFileDescriptorsBounded(t *testing.T) { + store, err := OpenSignalStore[telemetry.Log](t.TempDir(), "EventUnixNanos") + if err != nil { + t.Fatalf("OpenSignalStore: %v", err) + } + defer store.Close() + for i := range 32 { + if err := store.Append(stringID(i), []telemetry.Log{{EventUnixNanos: int64(i + 1)}}); err != nil { + t.Fatalf("Append(%d): %v", i, err) + } + } + var open, maximum atomic.Int64 + store.openFile = func(path string) (signalFile, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + current := open.Add(1) + for { + previous := maximum.Load() + if current <= previous || maximum.CompareAndSwap(previous, current) { + break + } + } + return &countedSignalFile{File: file, open: &open}, nil + } + visited := 0 + if err := store.Scan(0, 100, func(telemetry.Log) bool { + visited++ + return true + }); err != nil { + t.Fatalf("Scan: %v", err) + } + if visited != 32 { + t.Fatalf("visited = %d, want 32", visited) + } + if got := maximum.Load(); got != 1 { + t.Fatalf("maximum simultaneous descriptors = %d, want 1", got) + } + if got := open.Load(); got != 0 { + t.Fatalf("descriptors left open = %d, want 0", got) + } +} + +func stringID(i int) string { + const digits = "0123456789abcdef" + return "batch-" + string([]byte{digits[(i>>4)&15], digits[i&15]}) +} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index f172001e..2a3f1958 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path/filepath" "strings" @@ -14,11 +15,15 @@ import ( ) type compactionMarker struct { - ID string `json:"id"` - Inputs []string `json:"inputs"` - MaxNanos int64 `json:"max_nanos"` + ID string `json:"id"` + Inputs []string `json:"inputs"` + MinNanos int64 `json:"min_nanos"` + MaxNanos int64 `json:"max_nanos"` + Generation uint32 `json:"generation"` } +const minCompactionInputs = 8 + // CompactParquet combines the oldest small atomic batches into larger files. // A durable marker makes the multi-signal swap recoverable after a crash. func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { @@ -28,18 +33,22 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches r.compactionMu.Lock() defer r.compactionMu.Unlock() r.mu.RLock() - if len(r.manifest.Batches) < 8 { - r.mu.RUnlock() + selected := selectCompactionBatches(r.manifest.Batches, maxBatches) + r.mu.RUnlock() + if len(selected) < minCompactionInputs { return 0, nil } - count := min(maxBatches, len(r.manifest.Batches)) - selected := append([]batchMetadata(nil), r.manifest.Batches[:count]...) - r.mu.RUnlock() - marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano())} + marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinNanos: math.MaxInt64, Generation: selected[0].Generation + 1} for _, batch := range selected { marker.Inputs = append(marker.Inputs, batch.ID) + if batch.MinNanos > 0 { + marker.MinNanos = min(marker.MinNanos, batch.MinNanos) + } marker.MaxNanos = max(marker.MaxNanos, batch.MaxNanos) } + if marker.MinNanos == math.MaxInt64 { + marker.MinNanos = 0 + } stageDir := filepath.Join(r.root, marker.ID) if err := os.Mkdir(stageDir, 0o755); err != nil { return 0, err @@ -96,7 +105,55 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if err := r.completeCompaction(marker); err != nil { return 0, err } - return count, nil + return len(selected), nil +} + +type compactionKey struct { + day int64 + generation uint32 +} + +// selectCompactionBatches implements a leveled, day-partitioned compaction +// policy. An output can only be merged with peers from the same day and +// generation, so maintenance never folds the complete retained corpus into +// one perpetually young file. At most minCompactionInputs-1 files remain at +// each level for a day. +func selectCompactionBatches(batches []batchMetadata, maxBatches int) []batchMetadata { + if maxBatches < minCompactionInputs { + return nil + } + counts := make(map[compactionKey]int) + for _, batch := range batches { + if batch.MaxNanos <= 0 { + continue + } + key := compactionKey{day: batch.MaxNanos / int64(24*time.Hour), generation: batch.Generation} + counts[key]++ + } + var chosen compactionKey + found := false + for key, count := range counts { + if count < minCompactionInputs { + continue + } + if !found || key.day < chosen.day || (key.day == chosen.day && key.generation < chosen.generation) { + chosen, found = key, true + } + } + if !found { + return nil + } + selected := make([]batchMetadata, 0, min(maxBatches, counts[chosen])) + for _, batch := range batches { + key := compactionKey{day: batch.MaxNanos / int64(24*time.Hour), generation: batch.Generation} + if batch.MaxNanos > 0 && key == chosen { + selected = append(selected, batch) + if len(selected) == maxBatches { + break + } + } + } + return selected } // CompactParquetBacklog drains every currently eligible compaction group so a @@ -173,7 +230,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { kept = append(kept, batch) } } - kept = append(kept, batchMetadata{ID: marker.ID, MaxNanos: marker.MaxNanos}) + kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation}) next := repositoryManifest{Version: 1, Batches: kept} if err := writeRepositoryManifest(r.root, next); err != nil { return err diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 85b1e6bb..bf325cf6 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "math" "os" "path/filepath" "sort" @@ -30,8 +31,10 @@ type Batch struct { } type batchMetadata struct { - ID string `json:"id"` - MaxNanos int64 `json:"max_nanos"` + ID string `json:"id"` + MinNanos int64 `json:"min_nanos"` + MaxNanos int64 `json:"max_nanos"` + Generation uint32 `json:"generation"` } type repositoryManifest struct { @@ -41,6 +44,7 @@ type repositoryManifest struct { type Repository struct { mu sync.RWMutex + commitMu sync.Mutex compactionMu sync.Mutex root string walDir string @@ -106,16 +110,9 @@ func (r *Repository) Close() error { return errors.Join(r.Spans.Close(), r.Logs.Close(), r.Metrics.Close()) } -func (r *Repository) ReadLock() func() { - r.mu.RLock() - return r.mu.RUnlock -} - // PruneHot removes acceleration segments older than cutoff. Parquet remains // authoritative for longer retention and SQL queries. func (r *Repository) PruneHot(cutoff int64) (int, error) { - r.mu.Lock() - defer r.mu.Unlock() spans, spanErr := r.Spans.PruneBefore(cutoff) logs, logErr := r.Logs.PruneBefore(cutoff) metricRows, metricErr := r.Metrics.PruneBefore(cutoff) @@ -174,12 +171,17 @@ func (r *Repository) Commit(batch Batch) error { if err := r.writeWAL(batch); err != nil { return err } - r.mu.Lock() + // Commits are serialized, but their segment and Parquet fsyncs do not hold + // the repository metadata lock. Each projection has its own atomic publish + // protocol; the WAL keeps a partially applied transaction replayable. + r.commitMu.Lock() + defer r.commitMu.Unlock() err := r.apply(batch) if err == nil { + r.mu.Lock() err = r.recordBatch(batch) + r.mu.Unlock() } - r.mu.Unlock() if err != nil { return err } @@ -344,7 +346,7 @@ func (r *Repository) recordBatch(batch Batch) error { } } next := r.manifest - next.Batches = append(append([]batchMetadata(nil), r.manifest.Batches...), batchMetadata{ID: batch.ID, MaxNanos: batchMaxNanos(batch)}) + next.Batches = append(append([]batchMetadata(nil), r.manifest.Batches...), batchMetadata{ID: batch.ID, MinNanos: batchMinNanos(batch), MaxNanos: batchMaxNanos(batch)}) if err := writeRepositoryManifest(r.root, next); err != nil { return err } @@ -366,6 +368,31 @@ func batchMaxNanos(batch Batch) int64 { return maxNanos } +func batchMinNanos(batch Batch) int64 { + minNanos := int64(math.MaxInt64) + include := func(value int64) { + if value > 0 { + minNanos = min(minNanos, value) + } + } + for _, row := range batch.Spans { + include(row.StartUnixNanos) + include(row.IngestedAt) + } + for _, row := range batch.Logs { + include(row.EventUnixNanos) + include(row.IngestedAt) + } + for _, row := range batch.Metrics { + include(row.EventUnixNanos) + include(row.IngestedAt) + } + if minNanos == math.MaxInt64 { + return 0 + } + return minNanos +} + func writeRepositoryManifest(root string, manifest repositoryManifest) error { data, err := json.Marshal(manifest) if err != nil { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 76c90e45..cb223733 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" _ "github.com/duckdb/duckdb-go/v2" "github.com/labstack/fanout/internal/telemetry" @@ -65,6 +66,42 @@ func TestRepositoryCommitIsIdempotentAndQueryable(t *testing.T) { } } +func TestRepositoryCommitIODoesNotHoldMetadataLock(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + batch := testBatch() + batch.ID = "lock-scope-batch" + repository.mu.Lock() + committed := make(chan error, 1) + go func() { committed <- repository.Commit(batch) }() + parquet := filepath.Join(dir, "parquet", "spans", batch.ID+".parquet") + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(parquet); err == nil { + break + } + if time.Now().After(deadline) { + repository.mu.Unlock() + t.Fatal("commit projection I/O remained blocked by repository metadata lock") + } + time.Sleep(time.Millisecond) + } + select { + case err := <-committed: + repository.mu.Unlock() + t.Fatalf("Commit returned before metadata publication lock was released: %v", err) + default: + } + repository.mu.Unlock() + if err := <-committed; err != nil { + t.Fatal(err) + } +} + func TestRepositoryReplaysDurableWAL(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -236,7 +273,88 @@ func TestRepositoryCompactionDrainsBacklog(t *testing.T) { if compacted <= 64 { t.Fatalf("compacted batches = %d, want multiple groups", compacted) } - if len(repository.manifest.Batches) != 1 { - t.Fatalf("manifest batches = %d, want 1", len(repository.manifest.Batches)) + if len(repository.manifest.Batches) != 2 { + t.Fatalf("manifest batches = %d, want 2 bounded level-1 outputs", len(repository.manifest.Batches)) + } + for _, batch := range repository.manifest.Batches { + if batch.Generation != 1 { + t.Fatalf("batch %s generation = %d, want 1", batch.ID, batch.Generation) + } + } + again, err := repository.CompactParquet(context.Background(), db, 64, nil) + if err != nil { + t.Fatal(err) + } + if again != 0 { + t.Fatalf("second compaction rewrote %d already-compacted inputs, want 0", again) + } +} + +func TestRepositoryCompactionPreservesRetentionPartitions(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + oldTime := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC).UnixNano() + newTime := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC).UnixNano() + for day, timestamp := range []int64{oldTime, newTime} { + for i := range 8 { + batch := testBatchAt(timestamp) + batch.ID = fmt.Sprintf("day-%d-batch-%d", day, i) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + } + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := repository.CompactParquetBacklog(context.Background(), db, 64, nil); err != nil { + t.Fatal(err) + } + if len(repository.manifest.Batches) != 2 { + t.Fatalf("manifest batches = %d, want one output per day", len(repository.manifest.Batches)) } + var oldID, newID string + for _, batch := range repository.manifest.Batches { + switch batch.MaxNanos { + case oldTime: + oldID = batch.ID + case newTime: + newID = batch.ID + } + } + removed, err := repository.PruneParquet(time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC).UnixNano()) + if err != nil { + t.Fatal(err) + } + if removed != 1 || oldID == "" || newID == "" { + t.Fatalf("removed=%d oldID=%q newID=%q, want exactly the old partition", removed, oldID, newID) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, oldID+".parquet")); !os.IsNotExist(err) { + t.Fatalf("expired compacted %s file remains: %v", signal, err) + } + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, newID+".parquet")); err != nil { + t.Fatalf("current compacted %s file missing: %v", signal, err) + } + } +} + +func testBatchAt(timestamp int64) Batch { + batch := testBatch() + batch.Spans[0].StartUnixNanos = timestamp + batch.Spans[0].EndUnixNanos = timestamp + 1 + batch.Spans[0].IngestedAt = timestamp + batch.Logs[0].TimeUnixNanos = timestamp + batch.Logs[0].EventUnixNanos = timestamp + batch.Logs[0].IngestedAt = timestamp + batch.Metrics[0].TimeUnixNanos = timestamp + batch.Metrics[0].EventUnixNanos = timestamp + batch.Metrics[0].IngestedAt = timestamp + return batch } diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 9cb0f27b..7ebb1a3c 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -2,7 +2,6 @@ package store import ( "context" - "fmt" "log/slog" "time" @@ -27,6 +26,7 @@ type Writer struct { bufSpans []telemetry.Span bufLogs []telemetry.Log bufMetrics []telemetry.Metric + retryDelay func(int) time.Duration done chan struct{} } @@ -113,28 +113,36 @@ func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { for batch := range in { - var err error - for attempt := 0; attempt < 3; attempt++ { - err = w.repository.Commit(batch) + for attempt := 0; ; attempt++ { + err := w.repository.Commit(batch) if err == nil { break } - time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond) - } - if err != nil { - commitErr := fmt.Errorf("commit batch %s: %w", batch.ID, err) metrics.FlushErrors.WithLabelValues("batch").Inc() - metrics.RowsDropped.WithLabelValues("spans").Add(float64(len(batch.Spans))) - metrics.RowsDropped.WithLabelValues("logs").Add(float64(len(batch.Logs))) - metrics.RowsDropped.WithLabelValues("metrics").Add(float64(len(batch.Metrics))) - slog.Error("telemetry batch commit failed permanently", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) - done <- commitErr - return + // Log immediately and then at powers of two so a persistent storage + // outage stays visible without producing an unbounded log storm. The + // batch remains at the head of this bounded worker queue, applying + // backpressure until the same durable transaction commits. + if attempt == 0 || attempt&(attempt-1) == 0 { + slog.Warn("telemetry batch commit failed; retrying", "batch_id", batch.ID, "attempt", attempt+1, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) + } + delay := defaultCommitRetryDelay(attempt) + if w.retryDelay != nil { + delay = w.retryDelay(attempt) + } + if delay > 0 { + time.Sleep(delay) + } } } done <- nil } +func defaultCommitRetryDelay(attempt int) time.Duration { + shift := min(attempt, 6) + return min(100*time.Millisecond*time.Duration(1< 2*time.Second { - t.Fatalf("commit failure surfaced after %s", elapsed) + if len(committer.batches) != 1 || len(committer.batches[0].Spans) != 1 { + t.Fatalf("committed batches = %#v, want original batch exactly once", committer.batches) } } From 5a7a5fb305f2d7c2c3daf244b27487e70da38935 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 15:31:19 -0700 Subject: [PATCH 04/31] fix(storage): make compaction recovery safe Persist staged directory entries before publishing the recovery marker. Validate every required signal output and restore retired inputs instead of publishing an incomplete compaction. --- internal/telemetry/store/compaction.go | 88 ++++++++++++++++++++- internal/telemetry/store/repository_test.go | 73 +++++++++++++++++ 2 files changed, 157 insertions(+), 4 deletions(-) diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 2a3f1958..aea848e0 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -17,6 +17,7 @@ import ( type compactionMarker struct { ID string `json:"id"` Inputs []string `json:"inputs"` + Signals []string `json:"signals"` MinNanos int64 `json:"min_nanos"` MaxNanos int64 `json:"max_nanos"` Generation uint32 `json:"generation"` @@ -24,6 +25,8 @@ type compactionMarker struct { const minCompactionInputs = 8 +var parquetSignals = [...]string{"spans", "logs", "metrics"} + // CompactParquet combines the oldest small atomic batches into larger files. // A durable marker makes the multi-signal swap recoverable after a crash. func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { @@ -59,7 +62,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches _ = os.RemoveAll(stageDir) } }() - for _, signal := range []string{"spans", "logs", "metrics"} { + for _, signal := range parquetSignals { var inputs []string for _, id := range marker.Inputs { path := filepath.Join(r.Parquet.Dir(), signal, id+".parquet") @@ -72,6 +75,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if len(inputs) == 0 { continue } + marker.Signals = append(marker.Signals, signal) quoted := make([]string, len(inputs)) for i, path := range inputs { quoted[i] = sqlQuote(path) @@ -85,6 +89,12 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches return 0, fmt.Errorf("sync compacted %s parquet: %w", signal, err) } } + if len(marker.Signals) == 0 { + return 0, errors.New("compaction selected batches without parquet inputs") + } + if err := syncDirectory(stageDir); err != nil { + return 0, fmt.Errorf("sync compaction staging directory: %w", err) + } data, err := json.Marshal(marker) if err != nil { return 0, err @@ -198,7 +208,10 @@ func (r *Repository) recoverCompaction() error { func (r *Repository) completeCompaction(marker compactionMarker) error { stageDir := filepath.Join(r.root, marker.ID) - for _, signal := range []string{"spans", "logs", "metrics"} { + if err := r.validateCompactionOutputs(marker, stageDir); err != nil { + return errors.Join(err, r.restoreCompactionInputs(marker)) + } + for _, signal := range marker.Signals { dir := filepath.Join(r.Parquet.Dir(), signal) for _, id := range marker.Inputs { input := filepath.Join(dir, id+".parquet") @@ -207,6 +220,8 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { if err := os.Rename(input, retired); err != nil { return err } + } else if !errors.Is(err, os.ErrNotExist) { + return err } } stage := filepath.Join(stageDir, signal+".parquet") @@ -236,7 +251,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { return err } r.manifest = next - for _, signal := range []string{"spans", "logs", "metrics"} { + for _, signal := range marker.Signals { for _, id := range marker.Inputs { _ = os.Remove(filepath.Join(r.Parquet.Dir(), signal, id+".parquet.retired-"+marker.ID)) } @@ -248,6 +263,71 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { return syncDirectory(r.root) } +func (r *Repository) validateCompactionOutputs(marker compactionMarker, stageDir string) error { + if len(marker.Signals) == 0 { + return errors.New("compaction marker has no required signals") + } + for _, signal := range marker.Signals { + stage := filepath.Join(stageDir, signal+".parquet") + final := filepath.Join(r.Parquet.Dir(), signal, marker.ID+".parquet") + stageExists, err := pathExists(stage) + if err != nil { + return fmt.Errorf("inspect staged %s output: %w", signal, err) + } + finalExists, err := pathExists(final) + if err != nil { + return fmt.Errorf("inspect final %s output: %w", signal, err) + } + if !stageExists && !finalExists { + return fmt.Errorf("compaction %s is missing required %s output", marker.ID, signal) + } + } + return nil +} + +func (r *Repository) restoreCompactionInputs(marker compactionMarker) error { + var restoreErr error + for _, signal := range marker.Signals { + dir := filepath.Join(r.Parquet.Dir(), signal) + for _, id := range marker.Inputs { + input := filepath.Join(dir, id+".parquet") + retired := input + ".retired-" + marker.ID + retiredExists, err := pathExists(retired) + if err != nil { + restoreErr = errors.Join(restoreErr, fmt.Errorf("inspect retired %s input %s: %w", signal, id, err)) + continue + } + if !retiredExists { + continue + } + inputExists, err := pathExists(input) + if err != nil { + restoreErr = errors.Join(restoreErr, fmt.Errorf("inspect active %s input %s: %w", signal, id, err)) + continue + } + if inputExists { + restoreErr = errors.Join(restoreErr, fmt.Errorf("restore retired %s input %s: active input already exists", signal, id)) + continue + } + if err := os.Rename(retired, input); err != nil { + restoreErr = errors.Join(restoreErr, fmt.Errorf("restore retired %s input %s: %w", signal, id, err)) + } + } + } + return errors.Join(restoreErr, syncParquetDirectories(r.Parquet.Dir())) +} + +func pathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + func writeDurableFile(path string, data []byte) error { tmp := path + ".tmp" file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) @@ -272,7 +352,7 @@ func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", func syncParquetDirectories(root string) error { var err error - for _, signal := range []string{"spans", "logs", "metrics"} { + for _, signal := range parquetSignals { err = errors.Join(err, syncDirectory(filepath.Join(root, signal))) } return err diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index cb223733..ead528cb 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -3,6 +3,7 @@ package store import ( "context" "database/sql" + "encoding/json" "fmt" "os" "path/filepath" @@ -345,6 +346,78 @@ func TestRepositoryCompactionPreservesRetentionPartitions(t *testing.T) { } } +func TestRepositoryCompactionRecoveryRestoresRetiredInputsWhenStageMissing(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + marker := compactionMarker{ + ID: "compact-recovery", + Inputs: []string{"recovery-a", "recovery-b"}, + Signals: parquetSignals[:], + MinNanos: 100, + MaxNanos: 120, + Generation: 1, + } + for _, id := range marker.Inputs { + batch := testBatch() + batch.ID = id + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + stageDir := filepath.Join(dir, marker.ID) + if err := os.Mkdir(stageDir, 0o755); err != nil { + t.Fatal(err) + } + for _, signal := range marker.Signals { + if signal != "spans" { + data, err := os.ReadFile(filepath.Join(dir, "parquet", signal, marker.Inputs[0]+".parquet")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stageDir, signal+".parquet"), data, 0o644); err != nil { + t.Fatal(err) + } + } + for _, id := range marker.Inputs { + input := filepath.Join(dir, "parquet", signal, id+".parquet") + if err := os.Rename(input, input+".retired-"+marker.ID); err != nil { + t.Fatal(err) + } + } + } + data, err := json.Marshal(marker) + if err != nil { + t.Fatal(err) + } + if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { + t.Fatal(err) + } + if err := repository.recoverCompaction(); err == nil || !strings.Contains(err.Error(), "missing required spans output") { + t.Fatalf("recover compaction error = %v, want missing spans output", err) + } + for _, signal := range marker.Signals { + for _, id := range marker.Inputs { + input := filepath.Join(dir, "parquet", signal, id+".parquet") + if _, err := os.Stat(input); err != nil { + t.Fatalf("restored %s input %s: %v", signal, id, err) + } + if _, err := os.Stat(input + ".retired-" + marker.ID); !os.IsNotExist(err) { + t.Fatalf("retired %s input %s remains: %v", signal, id, err) + } + } + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, marker.ID+".parquet")); !os.IsNotExist(err) { + t.Fatalf("unexpected compacted %s output: %v", signal, err) + } + } + if len(repository.manifest.Batches) != len(marker.Inputs) { + t.Fatalf("manifest batches = %d, want original %d", len(repository.manifest.Batches), len(marker.Inputs)) + } +} + func testBatchAt(timestamp int64) Batch { batch := testBatch() batch.Spans[0].StartUnixNanos = timestamp From f1463b1ac51f9e6de474679c1ab704a7eef60142 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 15:44:01 -0700 Subject: [PATCH 05/31] fix(storage): close recovery and query gaps Prevent compacted WAL replay, bound writer shutdown and poison-batch retries, tier cold log reads through Parquet, and serialize DuckDB maintenance with rollups and context-aware query locks. --- internal/observability/logs.go | 97 +++++++++++++++----- internal/observability/service_test.go | 34 ++++++- internal/observability/trace.go | 8 +- internal/query/duck.go | 42 ++++++++- internal/query/duck_test.go | 55 ++++++++++++ internal/telemetry/segment/signal_store.go | 16 ++++ internal/telemetry/store/compaction.go | 22 ++++- internal/telemetry/store/repository.go | 56 ++++++++++-- internal/telemetry/store/repository_test.go | 57 ++++++++++++ internal/telemetry/store/writer.go | 98 ++++++++++++++++----- internal/telemetry/store/writer_test.go | 63 +++++++++++++ 11 files changed, 489 insertions(+), 59 deletions(-) diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 172b0b5a..d98b2fc7 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -59,6 +59,17 @@ func retainEarliest(entries *earliestLogHeap, entry LogEntry, limit int) { } } +var coldLogsQuery = ` +SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, + coalesce(trace_id, ''), coalesce(span_id, '') +FROM logs +WHERE time >= ? AND time < ? + AND (? = '' OR namespace = ?) + AND (? = '' OR service = ?) + AND (? = '' OR lower(severity) = lower(?)) + AND (? = '' OR contains(lower(` + redactLogBodySQL("body") + `), lower(?))) +ORDER BY time ASC` + func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, search string, limit int) (Result[Logs], error) { scope, err := s.normalizeScope(scope) if err != nil { @@ -78,32 +89,68 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear buckets := make(map[bucketKey]int64) entries := newestLogHeap{} matched := 0 - err = s.repository.Logs.Scan(scope.Start.UnixNano(), scope.End.UnixNano(), func(row telemetry.Log) bool { - select { - case <-ctx.Done(): - return false - default: - } - if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { - return true - } - body := redactLogBody(row.Body) - if search != "" && !strings.Contains(strings.ToLower(body), search) { - return true - } - entryTime := time.Unix(0, row.EventUnixNanos).UTC() + accumulate := func(entry LogEntry) { matched++ - retainNewest(&entries, LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}, limit) - bucketSeverity := strings.ToUpper(row.Severity) + retainNewest(&entries, entry, limit) + bucketSeverity := strings.ToUpper(entry.Severity) if bucketSeverity == "" { bucketSeverity = "UNSPECIFIED" } - bucketNanos := entryTime.Truncate(5 * time.Minute).UnixNano() + bucketNanos := entry.Time.Truncate(5 * time.Minute).UnixNano() buckets[bucketKey{time: bucketNanos, severity: bucketSeverity}]++ - return true - }) - if err != nil { - return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) + } + startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() + hotStart, coldEnd := startNanos, startNanos + oldestHot, _, hasHot := s.repository.Logs.Bounds() + if !hasHot { + coldEnd, hotStart = endNanos, endNanos + } else if oldestHot > startNanos { + coldEnd = min(oldestHot, endNanos) + hotStart = coldEnd + } + usedCold := coldEnd > startNanos + if usedCold { + rows, queryErr := s.db.QueryContext(ctx, coldLogsQuery, + scope.Start, time.Unix(0, coldEnd).UTC(), + scope.Namespace, scope.Namespace, service, service, severity, severity, search, search) + if queryErr != nil { + return Result[Logs]{}, fmt.Errorf("query cold logs: %w", queryErr) + } + for rows.Next() { + var entry LogEntry + if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { + rows.Close() + return Result[Logs]{}, fmt.Errorf("scan cold log: %w", err) + } + accumulate(entry) + } + if err := rows.Err(); err != nil { + rows.Close() + return Result[Logs]{}, fmt.Errorf("iterate cold logs: %w", err) + } + rows.Close() + } + if hotStart < endNanos { + err = s.repository.Logs.Scan(hotStart, endNanos, func(row telemetry.Log) bool { + select { + case <-ctx.Done(): + return false + default: + } + if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { + return true + } + body := redactLogBody(row.Body) + if search != "" && !strings.Contains(strings.ToLower(body), search) { + return true + } + entryTime := time.Unix(0, row.EventUnixNanos).UTC() + accumulate(LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) + return true + }) + if err != nil { + return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) + } } if err := ctx.Err(); err != nil { return Result[Logs]{}, err @@ -119,8 +166,14 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear } return data.Buckets[i].Time.Before(data.Buckets[j].Time) }) + dataSource := "fanout_segments" + if usedCold && hotStart < endNanos { + dataSource = "fanout_segments+parquet" + } else if usedCold { + dataSource = "parquet" + } return Result[Logs]{ Schema: LogsSchema, Summary: fmt.Sprintf("%d logs matched the selected telemetry window", matched), - Data: data, Provenance: s.provenanceFor(scope, "fanout_segments"), + Data: data, Provenance: s.provenanceFor(scope, dataSource), }, nil } diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index c8a1db91..f2877e84 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -293,6 +293,35 @@ func TestLogsRetainsOnlyNewestLimit(t *testing.T) { } } +func TestLogsFallsBackToParquetOutsideHotRetention(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "cold-logs", Logs: []telemetry.Log{{ + Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", + ServiceName: "checkout", Body: "token=secret", TraceID: "trace-cold", + }}}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(coldLogsQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(time.Minute), "ERROR", "checkout", "token=[REDACTED]", "trace-cold", "")) + result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "checkout", "error", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Entries) != 1 || result.Data.Entries[0].Body != "token=[REDACTED]" || result.Provenance.DataSource != "parquet" { + t.Fatalf("cold logs result = %#v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { svc, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) @@ -300,17 +329,18 @@ func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { {ID: "trace-latest", Spans: []telemetry.Span{{Namespace: "prod", TraceID: "trace-order", SpanID: "root", StartUnixNanos: start.UnixNano(), DurationMS: 1}}, Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(30 * time.Millisecond).UnixNano(), Body: "latest"}}}, {ID: "trace-earliest", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(10 * time.Millisecond).UnixNano(), Body: "earliest"}}}, {ID: "trace-middle", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), Body: "middle"}}}, + {ID: "trace-outside-span", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(2 * time.Minute).UnixNano(), Body: "unrelated later event"}}}, } for _, batch := range batches { if err := svc.repository.Commit(batch); err != nil { t.Fatal(err) } } - result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "trace-order", "", 2) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "trace-order", "", 10) if err != nil { t.Fatal(err) } - if len(result.Data.Logs) != 2 || result.Data.Logs[0].Body != "earliest" || result.Data.Logs[1].Body != "middle" { + if len(result.Data.Logs) != 3 || result.Data.Logs[0].Body != "earliest" || result.Data.Logs[1].Body != "middle" || result.Data.Logs[2].Body != "latest" { t.Fatalf("trace logs = %#v", result.Data.Logs) } } diff --git a/internal/observability/trace.go b/internal/observability/trace.go index abde8673..cce48055 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -112,7 +112,13 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin sort.Strings(data.Services) traceLogs := earliestLogHeap{} - readErr = s.repository.Logs.Scan(startNanos, endNanos, func(row telemetry.Log) bool { + logStart, logEnd := startNanos, endNanos + if !first.IsZero() { + const correlationMargin = time.Second + logStart = max(logStart, first.Add(-correlationMargin).UnixNano()) + logEnd = min(logEnd, last.Add(correlationMargin).UnixNano()) + } + readErr = s.repository.Logs.Scan(logStart, logEnd, func(row telemetry.Log) bool { select { case <-ctx.Done(): return false diff --git a/internal/query/duck.go b/internal/query/duck.go index c253808c..b712a3ee 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -463,6 +463,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { pruneErr = errors.Join(pruneErr, parquetErr, compactErr) } var cacheErr error + unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) if d.cfg.RetentionDays > 0 { for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { @@ -471,6 +472,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { } } _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") + unlockMaintenance() err := errors.Join(pruneErr, cacheErr, checkpointErr) maintenanceResult := metrics.TelemetrySuccess if err != nil { @@ -506,6 +508,10 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupService, true, watermark, sourceMax) } }() + if err := d.lockParquetRead(ctx); err != nil { + return 0, err + } + defer d.parquetMu.RUnlock() // Serialize against other writers (edge rollup, maintenance, ingest flushes). // The write gate is always acquired before a connection to keep lock ordering @@ -627,6 +633,10 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupEndpoint, true, watermark, sourceMax) } }() + if err := d.lockParquetRead(ctx); err != nil { + return 0, err + } + defer d.parquetMu.RUnlock() unlock := d.writeGate.Lock(writegate.WriteRollupEndpoint) defer unlock() @@ -747,6 +757,10 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupEdge, true, watermark, sourceMax) } }() + if err := d.lockParquetRead(ctx); err != nil { + return 0, err + } + defer d.parquetMu.RUnlock() unlock := d.writeGate.Lock(writegate.WriteRollupEdge) defer unlock() @@ -1367,7 +1381,9 @@ FROM messaging_edges;` // QueryContext executes a read against immutable Parquet files and DuckDB's // local rollup cache. func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { - d.parquetMu.RLock() + if err := d.lockParquetRead(ctx); err != nil { + return nil, err + } rows, err := d.DB.QueryContext(ctx, query, args...) if err != nil { d.parquetMu.RUnlock() @@ -1401,11 +1417,33 @@ func (r *lockedRows) release() { r.unlockOnce.Do(r.unlock) } // QueryRowScan executes a single-row query against immutable Parquet files and // DuckDB's local rollup cache. func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args ...any) error { - d.parquetMu.RLock() + if err := d.lockParquetRead(ctx); err != nil { + return err + } defer d.parquetMu.RUnlock() return d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) } +func (d *Duck) lockParquetRead(ctx context.Context) error { + for { + if d.parquetMu.TryRLock() { + return nil + } + timer := time.NewTimer(time.Millisecond) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return ctx.Err() + case <-timer.C: + } + } +} + // ---- Queries for API ---- type LatencyRow struct { diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index cb5d73eb..bb1c41d1 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/metrics" + "github.com/labstack/fanout/internal/query/writegate" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -191,6 +192,60 @@ func TestQueryContextHoldsParquetLockUntilRowsFinish(t *testing.T) { } } +func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { + d := &Duck{} + d.parquetMu.Lock() + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + started := time.Now() + err := d.QueryRowScan(ctx, []any{new(int)}, "SELECT 1") + d.parquetMu.Unlock() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("QueryRowScan error = %v, want deadline exceeded", err) + } + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + t.Fatalf("QueryRowScan ignored lock deadline for %s", elapsed) + } +} + +func TestRollupReadLockHonorsContext(t *testing.T) { + d := &Duck{} + d.parquetMu.Lock() + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + _, err := d.refreshServiceRollup(ctx) + d.parquetMu.Unlock() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("refreshServiceRollup error = %v, want deadline exceeded", err) + } +} + +func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, cfg: config.Config{MaintenanceInterval: time.Nanosecond}} + release := d.writeGate.Lock(writegate.WriteRollupService) + done := make(chan error, 1) + go func() { done <- d.runRepositoryMaintenance(context.Background()) }() + select { + case err := <-done: + release() + t.Fatalf("maintenance bypassed write gate: %v", err) + case <-time.After(25 * time.Millisecond): + } + release() + if err := <-done; err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestDuckDSN(t *testing.T) { tests := []struct { name string diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index 107c4e9c..15ef6432 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -344,6 +344,22 @@ func (s *SignalStore[T]) RowCount() uint64 { return total } +// Bounds returns the oldest and newest event timestamps currently covered by +// the hot acceleration tier. +func (s *SignalStore[T]) Bounds() (int64, int64, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.segments) == 0 { + return 0, 0, false + } + minTime, maxTime := s.segments[0].min, s.segments[0].max + for _, segment := range s.segments[1:] { + minTime = min(minTime, segment.min) + maxTime = max(maxTime, segment.max) + } + return minTime, maxTime, true +} + func (s *SignalStore[T]) SegmentCount() int { s.mu.RLock() defer s.mu.RUnlock() diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index aea848e0..19f09276 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -21,6 +21,7 @@ type compactionMarker struct { MinNanos int64 `json:"min_nanos"` MaxNanos int64 `json:"max_nanos"` Generation uint32 `json:"generation"` + Sources []string `json:"sources"` } const minCompactionInputs = 8 @@ -44,6 +45,11 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinNanos: math.MaxInt64, Generation: selected[0].Generation + 1} for _, batch := range selected { marker.Inputs = append(marker.Inputs, batch.ID) + if len(batch.Sources) == 0 { + marker.Sources = append(marker.Sources, batch.ID) + } else { + marker.Sources = append(marker.Sources, batch.Sources...) + } if batch.MinNanos > 0 { marker.MinNanos = min(marker.MinNanos, batch.MinNanos) } @@ -110,8 +116,6 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches publishLock.Lock() defer publishLock.Unlock() } - r.mu.Lock() - defer r.mu.Unlock() if err := r.completeCompaction(marker); err != nil { return 0, err } @@ -207,6 +211,10 @@ func (r *Repository) recoverCompaction() error { } func (r *Repository) completeCompaction(marker compactionMarker) error { + r.commitMu.Lock() + defer r.commitMu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() stageDir := filepath.Join(r.root, marker.ID) if err := r.validateCompactionOutputs(marker, stageDir); err != nil { return errors.Join(err, r.restoreCompactionInputs(marker)) @@ -235,6 +243,14 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { return err } + for _, source := range marker.Sources { + if err := os.Remove(filepath.Join(r.walDir, source+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove compacted input WAL %s: %w", source, err) + } + } + if err := syncDirectory(r.walDir); err != nil { + return fmt.Errorf("sync compacted input WAL removals: %w", err) + } inputSet := make(map[string]struct{}, len(marker.Inputs)) for _, id := range marker.Inputs { inputSet[id] = struct{}{} @@ -245,7 +261,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { kept = append(kept, batch) } } - kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation}) + kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation, Sources: append([]string(nil), marker.Sources...)}) next := repositoryManifest{Version: 1, Batches: kept} if err := writeRepositoryManifest(r.root, next); err != nil { return err diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index bf325cf6..c36d27af 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -35,6 +35,10 @@ type batchMetadata struct { MinNanos int64 `json:"min_nanos"` MaxNanos int64 `json:"max_nanos"` Generation uint32 `json:"generation"` + // Sources retains the original ingest batch IDs folded into a compacted + // output. It is a retention-bounded replay ledger: a stale WAL can never + // resurrect rows already present in this output. + Sources []string `json:"sources,omitempty"` } type repositoryManifest struct { @@ -123,6 +127,8 @@ func (r *Repository) PruneHot(cutoff int64) (int, error) { // straddles the boundary is retained intact, so retention never removes newer // telemetry from another signal in the same atomic commit. func (r *Repository) PruneParquet(cutoff int64) (int, error) { + r.commitMu.Lock() + defer r.commitMu.Unlock() r.mu.Lock() defer r.mu.Unlock() kept := make([]batchMetadata, 0, len(r.manifest.Batches)) @@ -168,14 +174,18 @@ func (r *Repository) Commit(batch Batch) error { return errors.New("telemetry batch requires a safe ID") } normalizeBatch(&batch) - if err := r.writeWAL(batch); err != nil { - return err - } // Commits are serialized, but their segment and Parquet fsyncs do not hold // the repository metadata lock. Each projection has its own atomic publish // protocol; the WAL keeps a partially applied transaction replayable. r.commitMu.Lock() defer r.commitMu.Unlock() + consumed := r.batchConsumedLocked(batch.ID) + if consumed { + return r.removeWAL(batch.ID) + } + if err := r.writeWAL(batch); err != nil { + return err + } err := r.apply(batch) if err == nil { r.mu.Lock() @@ -185,10 +195,7 @@ func (r *Repository) Commit(batch Batch) error { if err != nil { return err } - if err := os.Remove(filepath.Join(r.walDir, batch.ID+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove committed telemetry WAL: %w", err) - } - return syncDirectory(r.walDir) + return r.removeWAL(batch.ID) } func (r *Repository) apply(batch Batch) error { @@ -289,6 +296,12 @@ func (r *Repository) recover() error { } continue } + if r.batchConsumed(batch.ID) { + if err := r.removeWAL(batch.ID); err != nil { + return fmt.Errorf("remove consumed replay %s: %w", name, err) + } + continue + } if err := r.apply(batch); err != nil { return fmt.Errorf("replay %s: %w", name, err) } @@ -344,6 +357,11 @@ func (r *Repository) recordBatch(batch Batch) error { if existing.ID == batch.ID { return nil } + for _, source := range existing.Sources { + if source == batch.ID { + return nil + } + } } next := r.manifest next.Batches = append(append([]batchMetadata(nil), r.manifest.Batches...), batchMetadata{ID: batch.ID, MinNanos: batchMinNanos(batch), MaxNanos: batchMaxNanos(batch)}) @@ -354,6 +372,30 @@ func (r *Repository) recordBatch(batch Batch) error { return nil } +func (r *Repository) batchConsumed(id string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + return r.batchConsumedLocked(id) +} + +func (r *Repository) batchConsumedLocked(id string) bool { + for _, batch := range r.manifest.Batches { + for _, source := range batch.Sources { + if source == id { + return true + } + } + } + return false +} + +func (r *Repository) removeWAL(id string) error { + if err := os.Remove(filepath.Join(r.walDir, id+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove committed telemetry WAL: %w", err) + } + return syncDirectory(r.walDir) +} + func batchMaxNanos(batch Batch) int64 { var maxNanos int64 for _, row := range batch.Spans { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index ead528cb..4777c41f 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -247,6 +247,63 @@ func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { } } +func TestRepositorySkipsStaleWALForCompactedBatch(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + var stale Batch + for i := range 8 { + batch := testBatch() + batch.ID = fmt.Sprintf("replay-source-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + if i == 0 { + stale = batch + } + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + if _, err := repository.CompactParquet(context.Background(), db, 64, nil); err != nil { + db.Close() + t.Fatal(err) + } + if err := repository.writeWAL(stale); err != nil { + db.Close() + t.Fatal(err) + } + if err := repository.Close(); err != nil { + db.Close() + t.Fatal(err) + } + recovered, err := Open(dir) + if err != nil { + db.Close() + t.Fatal(err) + } + defer recovered.Close() + defer db.Close() + if _, err := os.Stat(filepath.Join(dir, "wal", stale.ID+".wal")); !os.IsNotExist(err) { + t.Fatalf("consumed WAL remains after recovery: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "parquet", "spans", stale.ID+".parquet")); !os.IsNotExist(err) { + t.Fatalf("consumed source parquet was resurrected: %v", err) + } + pattern := filepath.ToSlash(filepath.Join(dir, "parquet", "spans", "*.parquet")) + var rows int + if err := db.QueryRow("SELECT count(*) FROM read_parquet(?)", pattern).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 8 { + t.Fatalf("rows after stale WAL recovery = %d, want 8", rows) + } +} + func TestRepositoryCompactionDrainsBacklog(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 7ebb1a3c..dd7620b8 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -10,24 +10,29 @@ import ( "github.com/labstack/fanout/internal/telemetry" ) -const flushQueueDepth = 4 +const ( + flushQueueDepth = 4 + commitRetryLimit = 8 + writerShutdownGrace = 5 * time.Second +) type batchCommitter interface { Commit(Batch) error } type Writer struct { - repository batchCommitter - interval time.Duration - batchSize int - spans <-chan telemetry.Span - logs <-chan telemetry.Log - metricRows <-chan telemetry.Metric - bufSpans []telemetry.Span - bufLogs []telemetry.Log - bufMetrics []telemetry.Metric - retryDelay func(int) time.Duration - done chan struct{} + repository batchCommitter + interval time.Duration + batchSize int + spans <-chan telemetry.Span + logs <-chan telemetry.Log + metricRows <-chan telemetry.Metric + bufSpans []telemetry.Span + bufLogs []telemetry.Log + bufMetrics []telemetry.Metric + retryDelay func(int) time.Duration + shutdownGrace time.Duration + done chan struct{} } func NewWriter(repository *Repository, interval time.Duration, batchSize int, spans <-chan telemetry.Span, logs <-chan telemetry.Log, metricRows <-chan telemetry.Metric) *Writer { @@ -40,7 +45,9 @@ func (w *Writer) Run(ctx context.Context) error { defer close(w.done) flushes := make(chan Batch, flushQueueDepth) workerDone := make(chan error, 1) - go w.flushWorker(flushes, workerDone) + workerCtx, cancelWorker := context.WithCancel(context.Background()) + defer cancelWorker() + go w.flushWorker(workerCtx, flushes, workerDone) ticker := time.NewTicker(w.interval) defer ticker.Stop() spans, logs, metricRows := w.spans, w.logs, w.metricRows @@ -52,6 +59,15 @@ func (w *Writer) Run(ctx context.Context) error { close(flushes) return <-workerDone } + finishBounded := func() error { + grace := w.shutdownGrace + if grace <= 0 { + grace = writerShutdownGrace + } + timer := time.AfterFunc(grace, cancelWorker) + defer timer.Stop() + return finish() + } for { select { case row, ok := <-spans: @@ -80,7 +96,7 @@ func (w *Writer) Run(ctx context.Context) error { return err } case <-ctx.Done(): - return finish() + return finishBounded() case err := <-workerDone: return err } @@ -111,18 +127,32 @@ func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { } } -func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { - for batch := range in { - for attempt := 0; ; attempt++ { +func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- error) { + for { + var batch Batch + select { + case <-ctx.Done(): + done <- ctx.Err() + return + case next, ok := <-in: + if !ok { + done <- nil + return + } + batch = next + } + committed := false + var lastErr error + for attempt := 0; attempt < commitRetryLimit; attempt++ { err := w.repository.Commit(batch) if err == nil { + committed = true break } + lastErr = err metrics.FlushErrors.WithLabelValues("batch").Inc() // Log immediately and then at powers of two so a persistent storage - // outage stays visible without producing an unbounded log storm. The - // batch remains at the head of this bounded worker queue, applying - // backpressure until the same durable transaction commits. + // outage stays visible without producing an unbounded log storm. if attempt == 0 || attempt&(attempt-1) == 0 { slog.Warn("telemetry batch commit failed; retrying", "batch_id", batch.ID, "attempt", attempt+1, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) } @@ -130,12 +160,36 @@ func (w *Writer) flushWorker(in <-chan Batch, done chan<- error) { if w.retryDelay != nil { delay = w.retryDelay(attempt) } + if attempt+1 == commitRetryLimit { + break + } if delay > 0 { - time.Sleep(delay) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + done <- ctx.Err() + return + case <-timer.C: + } } } + if !committed { + recordDroppedBatch(batch) + slog.Error("telemetry batch permanently failed; dropping after bounded retries", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) + } } - done <- nil +} + +func recordDroppedBatch(batch Batch) { + metrics.RowsDropped.WithLabelValues("spans").Add(float64(len(batch.Spans))) + metrics.RowsDropped.WithLabelValues("logs").Add(float64(len(batch.Logs))) + metrics.RowsDropped.WithLabelValues("metrics").Add(float64(len(batch.Metrics))) } func defaultCommitRetryDelay(attempt int) time.Duration { diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index d79b86d2..c4f510b7 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -52,3 +52,66 @@ func TestWriterRetainsBatchAcrossCommitFailures(t *testing.T) { t.Fatalf("committed batches = %#v, want original batch exactly once", committer.batches) } } + +func TestWriterDropsPoisonBatchAfterBoundedRetries(t *testing.T) { + spans := make(chan telemetry.Span, 1) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} + close(spans) + close(logs) + close(metricRows) + committer := &recoveringCommitter{failures: commitRetryLimit + 1} + w := &Writer{ + repository: committer, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return 0 }, + } + if err := w.Run(context.Background()); err != nil { + t.Fatalf("Run error = %v", err) + } + if committer.calls != commitRetryLimit { + t.Fatalf("Commit calls = %d, want %d", committer.calls, commitRetryLimit) + } + if len(committer.batches) != 0 { + t.Fatalf("committed poison batches = %#v", committer.batches) + } +} + +func TestWriterCancellationInterruptsCommitBackoff(t *testing.T) { + spans := make(chan telemetry.Span, 1) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} + committer := &recoveringCommitter{failures: commitRetryLimit + 1} + w := &Writer{ + repository: committer, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return time.Hour }, shutdownGrace: 25 * time.Millisecond, + } + ctx, cancel := context.WithCancel(context.Background()) + finished := make(chan error, 1) + go func() { finished <- w.Run(ctx) }() + deadline := time.Now().Add(time.Second) + for { + committer.mu.Lock() + calls := committer.calls + committer.mu.Unlock() + if calls > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("commit attempt did not start") + } + time.Sleep(time.Millisecond) + } + cancel() + select { + case err := <-finished: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run error = %v, want context canceled", err) + } + case <-time.After(time.Second): + t.Fatal("writer shutdown remained blocked in retry backoff") + } +} From 2cad290e1330d21d11d34ef6d52dadfda4d12ea3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 17:00:25 -0700 Subject: [PATCH 06/31] fix(storage): close remaining hot/cold and durability review gaps Split hot and cold reads on a durable prune watermark so pruning can never open a silent gap between the tiers, and fall back to Parquet whenever a trace's scope crosses that watermark. Replace the writer-preferring parquet mutex with a reader-first gate so queued maintenance cannot stall unrelated reads. Stage every batch in the WAL before the asynchronous commit handoff, and drop an unstageable batch with visible accounting instead of shutting the server down. Validate span segment section offsets so a torn file fails with an error instead of a boot-loop panic. Bound the compaction replay ledger to one generation, backfill zero span start times the way Parquet does, and drop the redundant maintenance throttle that halved the effective cadence. Claude-Session: https://claude.ai/code/session_01Cxec3QnsbwcU1dqaFCtnTf --- internal/observability/logs.go | 50 +++---- internal/observability/service_test.go | 81 +++++++++++ internal/observability/trace.go | 17 ++- internal/query/duck.go | 27 ++-- internal/query/duck_test.go | 83 +++++++++++ internal/query/edge_backlog_test.go | 5 +- internal/query/parquet_gate.go | 79 +++++++++++ internal/query/rollup_test.go | 32 ++--- internal/query/rollup_watermark_test.go | 21 ++- internal/telemetry/segment/span_store.go | 7 + internal/telemetry/segment/span_store_test.go | 43 ++++++ internal/telemetry/store/compaction.go | 26 +++- internal/telemetry/store/repository.go | 103 +++++++++++++- internal/telemetry/store/repository_test.go | 69 +++++++++ internal/telemetry/store/writer.go | 21 +++ internal/telemetry/store/writer_test.go | 133 ++++++++++++++++++ 16 files changed, 697 insertions(+), 100 deletions(-) create mode 100644 internal/query/parquet_gate.go diff --git a/internal/observability/logs.go b/internal/observability/logs.go index d98b2fc7..2da63f35 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -100,14 +100,28 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear buckets[bucketKey{time: bucketNanos, severity: bucketSeverity}]++ } startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() - hotStart, coldEnd := startNanos, startNanos - oldestHot, _, hasHot := s.repository.Logs.Bounds() - if !hasHot { - coldEnd, hotStart = endNanos, endNanos - } else if oldestHot > startNanos { - coldEnd = min(oldestHot, endNanos) - hotStart = coldEnd + hotCutoff, err := s.repository.ScanHotLogs(startNanos, endNanos, func(row telemetry.Log) bool { + select { + case <-ctx.Done(): + return false + default: + } + if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { + return true + } + body := redactLogBody(row.Body) + if search != "" && !strings.Contains(strings.ToLower(body), search) { + return true + } + entryTime := time.Unix(0, row.EventUnixNanos).UTC() + accumulate(LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) + return true + }) + if err != nil { + return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) } + coldEnd := min(max(hotCutoff, startNanos), endNanos) + hotStart := max(hotCutoff, startNanos) usedCold := coldEnd > startNanos if usedCold { rows, queryErr := s.db.QueryContext(ctx, coldLogsQuery, @@ -130,28 +144,6 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear } rows.Close() } - if hotStart < endNanos { - err = s.repository.Logs.Scan(hotStart, endNanos, func(row telemetry.Log) bool { - select { - case <-ctx.Done(): - return false - default: - } - if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { - return true - } - body := redactLogBody(row.Body) - if search != "" && !strings.Contains(strings.ToLower(body), search) { - return true - } - entryTime := time.Unix(0, row.EventUnixNanos).UTC() - accumulate(LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) - return true - }) - if err != nil { - return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) - } - } if err := ctx.Err(); err != nil { return Result[Logs]{}, err } diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index f2877e84..1b7938b4 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -322,6 +322,47 @@ func TestLogsFallsBackToParquetOutsideHotRetention(t *testing.T) { } } +func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + cutoff := start.Add(250 * time.Millisecond) + end := start.Add(time.Second) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ + {Namespace: "prod", TimeUnixNanos: start.Add(150 * time.Millisecond).UnixNano(), Body: "newer-old", Severity: "INFO"}, + {Namespace: "prod", TimeUnixNanos: start.Add(300 * time.Millisecond).UnixNano(), Body: "newer-hot", Severity: "INFO"}, + }}); err != nil { + t.Fatal(err) + } + if err := svc.repository.Commit(telemetrystore.Batch{ID: "overlap-late", Logs: []telemetry.Log{ + {Namespace: "prod", TimeUnixNanos: start.Add(100 * time.Millisecond).UnixNano(), Body: "late-old", Severity: "INFO"}, + {Namespace: "prod", TimeUnixNanos: start.Add(200 * time.Millisecond).UnixNano(), Body: "late-boundary", Severity: "INFO"}, + }}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(coldLogsQuery)). + WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(100*time.Millisecond), "INFO", "", "late-old", "", ""). + AddRow(start.Add(150*time.Millisecond), "INFO", "", "newer-old", "", ""). + AddRow(start.Add(200*time.Millisecond), "INFO", "", "late-boundary", "", "")) + result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Entries) != 4 || result.Summary != "4 logs matched the selected telemetry window" { + t.Fatalf("boundary logs = %#v", result) + } + if result.Data.Entries[0].Body != "newer-hot" || result.Provenance.DataSource != "fanout_segments+parquet" { + t.Fatalf("boundary result = %#v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { svc, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) @@ -369,4 +410,44 @@ func TestTraceFallsBackToParquetWhenHotSegmentsMiss(t *testing.T) { } } +func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + cutoff := start.Add(30 * time.Minute) + end := start.Add(time.Hour) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ + Namespace: "prod", TraceID: "split-trace", SpanID: "root", ServiceName: "frontend", + StartUnixNanos: start.Add(10 * time.Minute).UnixNano(), DurationMS: 100, StatusCode: "ERROR", + }}}); err != nil { + t.Fatal(err) + } + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-hot-child", Spans: []telemetry.Span{{ + Namespace: "prod", TraceID: "split-trace", SpanID: "child", ParentSpanID: "root", ServiceName: "backend", + StartUnixNanos: start.Add(50 * time.Minute).UnixNano(), DurationMS: 25, StatusCode: "OK", + }}}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). + WithArgs("split-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). + AddRow("root", "", "frontend", "request", "SERVER", start.Add(10*time.Minute), 100.0, "ERROR", "failed"). + AddRow("child", "root", "backend", "work", "CLIENT", start.Add(50*time.Minute), 25.0, "OK", "")) + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("split-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "split-trace", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Spans) != 2 || !result.Data.HasError || len(result.Data.Services) != 2 || result.Provenance.DataSource != "parquet" { + t.Fatalf("straddling trace = %#v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + var _ DB = queryrows.SQLAdapter{} diff --git a/internal/observability/trace.go b/internal/observability/trace.go index cce48055..497e42c1 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -64,11 +64,13 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin dataSource := "fanout_segments" data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} + hotCutoff := int64(0) if traceID != "" { - storedSpans, readErr := s.repository.Spans.Trace(traceID) + storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID) if readErr != nil { return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } + hotCutoff = spanCutoff startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() for _, row := range storedSpans { if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || @@ -118,7 +120,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin logStart = max(logStart, first.Add(-correlationMargin).UnixNano()) logEnd = min(logEnd, last.Add(correlationMargin).UnixNano()) } - readErr = s.repository.Logs.Scan(logStart, logEnd, func(row telemetry.Log) bool { + logCutoff, scanErr := s.repository.ScanHotLogs(logStart, logEnd, func(row telemetry.Log) bool { select { case <-ctx.Done(): return false @@ -130,16 +132,21 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin retainEarliest(&traceLogs, LogEntry{Time: time.Unix(0, row.EventUnixNanos).UTC(), Severity: row.Severity, Service: row.ServiceName, Body: redactLogBody(row.Body), TraceID: row.TraceID, SpanID: row.SpanID}, limit) return true }) - if readErr != nil { - return Result[TraceDetail]{}, fmt.Errorf("read trace logs: %w", readErr) + if scanErr != nil { + return Result[TraceDetail]{}, fmt.Errorf("read trace logs: %w", scanErr) } + hotCutoff = max(hotCutoff, logCutoff) if err := ctx.Err(); err != nil { return Result[TraceDetail]{}, err } data.Logs = append(data.Logs, traceLogs...) sort.Slice(data.Logs, func(i, j int) bool { return data.Logs[i].Time.Before(data.Logs[j].Time) }) } - if traceID != "" && len(data.Spans) == 0 { + // Any scope crossing the durable hot prune watermark may contain early trace + // spans that have aged out while a late suffix remains hot. Parquet is the + // authoritative complete trace in that case; a zero-span hot miss uses the + // same path. + if traceID != "" && (len(data.Spans) == 0 || scope.Start.UnixNano() < hotCutoff) { data, err = s.traceFromParquet(ctx, scope, traceID, limit) if err != nil { return Result[TraceDetail]{}, err diff --git a/internal/query/duck.go b/internal/query/duck.go index b712a3ee..fa6b8cf0 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -23,19 +23,18 @@ import ( ) type Duck struct { - DB *sql.DB - cfg config.Config - lastMaintenance time.Time - repository *telemetrystore.Repository + DB *sql.DB + cfg config.Config + repository *telemetrystore.Repository // rollupLagNanos holds the rollup watermark back from the max ingested // timestamp so late/out-of-order commits aren't skipped. Zero disables the // lag (no trailing window). rollupLagNanos int64 // writeGate serializes writes to the rebuildable DuckDB rollup cache. writeGate writegate.WriteGate - // parquetMu prevents retention from unlinking a file while DuckDB is opening - // the immutable files selected for a new query. - parquetMu sync.RWMutex + // parquetMu pins immutable files for active DuckDB readers. Its reader-first + // gate keeps a queued maintenance publish from stalling unrelated new reads. + parquetMu parquetReadGate // maintHealthMu guards the maintenance health fields below, which the // readiness probe reads while the maintenance pass writes them. maintHealthMu sync.Mutex @@ -432,14 +431,6 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { } func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { - every := d.cfg.MaintenanceInterval - if every <= 0 { - every = time.Hour - } - if !d.lastMaintenance.IsZero() && time.Since(d.lastMaintenance) < every { - metrics.RecordTelemetryOperation(metrics.TelemetryMaintenance, metrics.TelemetryThrottled, 0) - return nil - } start := time.Now() cutoff := time.Now().Add(-d.cfg.HotRetention).UnixNano() var pruneErr error @@ -479,11 +470,11 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { maintenanceResult = metrics.TelemetryError } metrics.RecordTelemetryOperation(metrics.TelemetryMaintenance, maintenanceResult, time.Since(start).Seconds()) - d.lastMaintenance = time.Now() + finished := time.Now() d.maintHealthMu.Lock() - d.lastMaintenanceAt = d.lastMaintenance + d.lastMaintenanceAt = finished if err == nil { - d.lastMaintenanceOK = d.lastMaintenance + d.lastMaintenanceOK = finished } d.lastMaintenanceErr = err d.maintHealthMu.Unlock() diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index bb1c41d1..41990bca 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -208,6 +208,65 @@ func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { } } +func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectQuery("SELECT 1").WillReturnRows(sqlmock.NewRows([]string{"value"}).AddRow(1)) + d := &Duck{DB: db} + d.parquetMu.RLock() + writerAcquired := make(chan struct{}) + releaseWriter := make(chan struct{}) + writerDone := make(chan struct{}) + go func() { + d.parquetMu.Lock() + close(writerAcquired) + <-releaseWriter + d.parquetMu.Unlock() + close(writerDone) + }() + deadline := time.Now().Add(time.Second) + for { + d.parquetMu.mu.Lock() + waiting := d.parquetMu.waitingWriters + d.parquetMu.mu.Unlock() + if waiting > 0 { + break + } + if time.Now().After(deadline) { + d.parquetMu.RUnlock() + t.Fatal("maintenance writer did not begin waiting") + } + time.Sleep(time.Millisecond) + } + var value int + if err := d.QueryRowScan(context.Background(), []any{&value}, "SELECT 1"); err != nil { + d.parquetMu.RUnlock() + t.Fatalf("new read blocked behind waiting maintenance: %v", err) + } + if value != 1 { + d.parquetMu.RUnlock() + t.Fatalf("value = %d, want 1", value) + } + d.parquetMu.RUnlock() + select { + case <-writerAcquired: + case <-time.After(time.Second): + t.Fatal("maintenance writer did not run after readers drained") + } + close(releaseWriter) + select { + case <-writerDone: + case <-time.After(time.Second): + t.Fatal("maintenance writer did not release") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestRollupReadLockHonorsContext(t *testing.T) { d := &Duck{} d.parquetMu.Lock() @@ -414,3 +473,27 @@ func TestFailedRollupStillPublishesLag(t *testing.T) { }) } } + +func TestMaintenanceRunsOnEveryTick(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, cfg: config.Config{MaintenanceInterval: time.Hour}} + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatal(err) + } + first := d.lastMaintenanceAt + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatal(err) + } + if !d.lastMaintenanceAt.After(first) { + t.Fatal("second maintenance pass was throttled; the ticker is the only intended throttle") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index feca7f1b..ae3e49e1 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -27,9 +27,8 @@ func TestEdgeRollupBacklog(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30, DuckDBMemory: "1GB"}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30, DuckDBMemory: "1GB"}, } ctx := context.Background() diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go new file mode 100644 index 00000000..3dd13a2c --- /dev/null +++ b/internal/query/parquet_gate.go @@ -0,0 +1,79 @@ +package query + +import "sync" + +// parquetReadGate protects Parquet file publication without sync.RWMutex's +// writer preference. Maintenance may wait behind a long reader, but merely +// queuing that maintenance must never block unrelated API reads or readiness +// probes. Once readers drain, the writer publishes while new readers wait. +type parquetReadGate struct { + once sync.Once + mu sync.Mutex + changed *sync.Cond + readers int + writer bool + waitingWriters int +} + +func (g *parquetReadGate) init() { + g.once.Do(func() { g.changed = sync.NewCond(&g.mu) }) +} + +func (g *parquetReadGate) TryRLock() bool { + g.init() + g.mu.Lock() + defer g.mu.Unlock() + if g.writer { + return false + } + g.readers++ + return true +} + +func (g *parquetReadGate) RLock() { + g.init() + g.mu.Lock() + for g.writer { + g.changed.Wait() + } + g.readers++ + g.mu.Unlock() +} + +func (g *parquetReadGate) RUnlock() { + g.init() + g.mu.Lock() + g.readers-- + if g.readers < 0 { + g.mu.Unlock() + panic("query: parquetReadGate RUnlock without RLock") + } + if g.readers == 0 { + g.changed.Broadcast() + } + g.mu.Unlock() +} + +func (g *parquetReadGate) Lock() { + g.init() + g.mu.Lock() + g.waitingWriters++ + for g.writer || g.readers > 0 { + g.changed.Wait() + } + g.waitingWriters-- + g.writer = true + g.mu.Unlock() +} + +func (g *parquetReadGate) Unlock() { + g.init() + g.mu.Lock() + if !g.writer { + g.mu.Unlock() + panic("query: parquetReadGate Unlock without Lock") + } + g.writer = false + g.changed.Broadcast() + g.mu.Unlock() +} diff --git a/internal/query/rollup_test.go b/internal/query/rollup_test.go index 7f4013a6..de00ec22 100644 --- a/internal/query/rollup_test.go +++ b/internal/query/rollup_test.go @@ -19,9 +19,8 @@ func TestRollupOnceRebuildsAffectedServiceBuckets(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() @@ -90,7 +89,7 @@ func TestRollupOnceRebuildsAffectedEndpointBuckets(t *testing.T) { if err := CreateViews(db); err != nil { t.Fatalf("CreateViews failed: %v", err) } - d := &Duck{DB: db, cfg: config.Config{RetentionDays: 30}, lastMaintenance: time.Now()} + d := &Duck{DB: db, cfg: config.Config{RetentionDays: 30}} ctx := context.Background() bucket := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute) @@ -141,9 +140,8 @@ func TestRollupOnceRebuildsAffectedEdgeBuckets(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() @@ -231,9 +229,8 @@ func TestRollupOnceIgnoresRowsWithoutBucketTimestamp(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() @@ -306,9 +303,8 @@ func TestRollupOnceMessagingEdgeCountsConsumedMessages(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() @@ -378,9 +374,8 @@ func TestRollupOnceDropsCallEdgeParentOutsideWindow(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() @@ -502,9 +497,8 @@ func TestRollupOnceChunksWideBacklog(t *testing.T) { } d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), + DB: db, + cfg: config.Config{RetentionDays: 30}, } ctx := context.Background() diff --git a/internal/query/rollup_watermark_test.go b/internal/query/rollup_watermark_test.go index 5fa35808..be7cf8b6 100644 --- a/internal/query/rollup_watermark_test.go +++ b/internal/query/rollup_watermark_test.go @@ -23,10 +23,9 @@ func TestRollupWatermarkPicksUpLateLowIngestedRow(t *testing.T) { const lag = 2 * time.Second d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), - rollupLagNanos: lag.Nanoseconds(), + DB: db, + cfg: config.Config{RetentionDays: 30}, + rollupLagNanos: lag.Nanoseconds(), } ctx := context.Background() @@ -89,10 +88,9 @@ func TestRollupWatermarkLagSurvivesChunkedCatchUp(t *testing.T) { const lag = 2 * time.Second d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), - rollupLagNanos: lag.Nanoseconds(), + DB: db, + cfg: config.Config{RetentionDays: 30}, + rollupLagNanos: lag.Nanoseconds(), } ctx := context.Background() @@ -152,10 +150,9 @@ func TestEdgeRollupWatermarkPicksUpLateChild(t *testing.T) { const lag = 2 * time.Second d := &Duck{ - DB: db, - cfg: config.Config{RetentionDays: 30}, - lastMaintenance: time.Now(), - rollupLagNanos: lag.Nanoseconds(), + DB: db, + cfg: config.Config{RetentionDays: 30}, + rollupLagNanos: lag.Nanoseconds(), } ctx := context.Background() bucket := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute) diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index 14e271f3..95a0b7ee 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -880,6 +880,13 @@ func openSegment(path string) (segment, error) { if err != nil { return segment{}, err } + // Header offsets come from disk; a torn or bit-rotted segment must fail + // with an error naming the file, never drive a negative or huge make(). + size := uint64(info.Size()) + dirEnd := dirOffset + uint64(blockCount)*blockDirSize + if dirOffset < headerSize || dirEnd > indexOffset || indexOffset > rollupOffset || rollupOffset > size { + return segment{}, fmt.Errorf("segment %s has corrupt section offsets", filepath.Base(path)) + } seg.blocks = make([]blockDir, blockCount) buf := make([]byte, int(blockCount)*blockDirSize) if _, err := f.ReadAt(buf, int64(dirOffset)); err != nil { diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index ff15523a..e60e521d 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -2,6 +2,7 @@ package segment import ( + "encoding/binary" "os" "path/filepath" "testing" @@ -166,3 +167,45 @@ func TestStoreRejectsCorruptCommittedSegment(t *testing.T) { t.Fatal("Open succeeded with a corrupt committed segment") } } + +func TestOpenRejectsSegmentWithCorruptSectionOffsets(t *testing.T) { + base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() + rows := []Span{{Namespace: "default", TraceID: "trace-a", SpanID: "1", ServiceName: "api", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}} + corrupt := func(t *testing.T, mutate func(header []byte)) { + t.Helper() + dir := t.TempDir() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := store.AppendID("seg-a", rows); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "seg-a.fseg") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + mutate(data[:headerSize]) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + if _, err := Open(dir); err == nil { + t.Fatal("Open succeeded with corrupt section offsets") + } + } + t.Run("rollup offset before index offset", func(t *testing.T) { + corrupt(t, func(header []byte) { + indexOffset := binary.LittleEndian.Uint64(header[48:56]) + binary.LittleEndian.PutUint64(header[56:64], indexOffset-1) + }) + }) + t.Run("rollup offset beyond file size", func(t *testing.T) { + corrupt(t, func(header []byte) { + binary.LittleEndian.PutUint64(header[56:64], 1<<40) + }) + }) +} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 19f09276..9f77a5c0 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -42,14 +42,9 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if len(selected) < minCompactionInputs { return 0, nil } - marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinNanos: math.MaxInt64, Generation: selected[0].Generation + 1} + marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinNanos: math.MaxInt64, Generation: selected[0].Generation + 1, Sources: compactionSources(selected)} for _, batch := range selected { marker.Inputs = append(marker.Inputs, batch.ID) - if len(batch.Sources) == 0 { - marker.Sources = append(marker.Sources, batch.ID) - } else { - marker.Sources = append(marker.Sources, batch.Sources...) - } if batch.MinNanos > 0 { marker.MinNanos = min(marker.MinNanos, batch.MinNanos) } @@ -122,6 +117,23 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches return len(selected), nil } +// compactionSources builds the replay ledger for one compaction output. Only +// raw ingest batches folded in this pass need protection: their WAL files are +// removed durably when this compaction completes, and any writer still +// retrying one of them consults this ledger. Ledgers inherited from earlier +// outputs are dropped rather than folded forward — those WALs were already +// removed when their own compaction completed — which bounds the manifest to +// one generation of batch IDs instead of the whole retention window. +func compactionSources(selected []batchMetadata) []string { + sources := make([]string, 0, len(selected)) + for _, batch := range selected { + if len(batch.Sources) == 0 { + sources = append(sources, batch.ID) + } + } + return sources +} + type compactionKey struct { day int64 generation uint32 @@ -262,7 +274,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { } } kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation, Sources: append([]string(nil), marker.Sources...)}) - next := repositoryManifest{Version: 1, Batches: kept} + next := repositoryManifest{Version: 1, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} if err := writeRepositoryManifest(r.root, next); err != nil { return err } diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index c36d27af..f130f6a3 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -35,19 +35,26 @@ type batchMetadata struct { MinNanos int64 `json:"min_nanos"` MaxNanos int64 `json:"max_nanos"` Generation uint32 `json:"generation"` - // Sources retains the original ingest batch IDs folded into a compacted - // output. It is a retention-bounded replay ledger: a stale WAL can never - // resurrect rows already present in this output. + // Sources retains the raw ingest batch IDs folded into a compacted output + // by its own compaction pass. It is a one-generation replay ledger: a stale + // WAL can never resurrect rows already present in this output, and earlier + // generations need no entries because their WALs were removed durably when + // their own compaction completed. Sources []string `json:"sources,omitempty"` } type repositoryManifest struct { - Version uint32 `json:"version"` - Batches []batchMetadata `json:"batches"` + Version uint32 `json:"version"` + HotCutoffNanos int64 `json:"hot_cutoff_nanos"` + Batches []batchMetadata `json:"batches"` } type Repository struct { - mu sync.RWMutex + mu sync.RWMutex + // hotMu makes the persisted prune watermark and the hot-segment snapshot one + // atomic read boundary. A query can never observe an old watermark after the + // corresponding segments have been retired. + hotMu sync.RWMutex commitMu sync.Mutex compactionMu sync.Mutex root string @@ -117,12 +124,61 @@ func (r *Repository) Close() error { // PruneHot removes acceleration segments older than cutoff. Parquet remains // authoritative for longer retention and SQL queries. func (r *Repository) PruneHot(cutoff int64) (int, error) { + r.hotMu.Lock() + defer r.hotMu.Unlock() + + // Publish the boundary before retiring segments. A crash or partial prune can + // therefore create only harmless overlap (Parquet below the boundary and hot + // segments above it), never a hole after restart. + publishCutoff := func() error { + r.commitMu.Lock() + defer r.commitMu.Unlock() + r.mu.Lock() + defer r.mu.Unlock() + if cutoff > r.manifest.HotCutoffNanos { + next := r.manifest + next.HotCutoffNanos = cutoff + if err := writeRepositoryManifest(r.root, next); err != nil { + return err + } + r.manifest = next + } + return nil + } + if err := publishCutoff(); err != nil { + return 0, fmt.Errorf("publish hot prune cutoff: %w", err) + } + spans, spanErr := r.Spans.PruneBefore(cutoff) logs, logErr := r.Logs.PruneBefore(cutoff) metricRows, metricErr := r.Metrics.PruneBefore(cutoff) return spans + logs + metricRows, errors.Join(spanErr, logErr, metricErr) } +// ScanHotLogs reads the portion of [start,end) that is guaranteed complete in +// the hot tier and returns the durable boundary below which Parquet is +// authoritative. The boundary and scan are serialized with PruneHot. +func (r *Repository) ScanHotLogs(start, end int64, visit func(telemetry.Log) bool) (int64, error) { + r.hotMu.RLock() + defer r.hotMu.RUnlock() + r.mu.RLock() + cutoff := r.manifest.HotCutoffNanos + r.mu.RUnlock() + return cutoff, r.Logs.Scan(max(start, cutoff), end, visit) +} + +// HotTrace returns the hot trace snapshot and the durable prune boundary that +// was in force for that snapshot. +func (r *Repository) HotTrace(traceID string) ([]telemetry.Span, int64, error) { + r.hotMu.RLock() + defer r.hotMu.RUnlock() + r.mu.RLock() + cutoff := r.manifest.HotCutoffNanos + r.mu.RUnlock() + spans, err := r.Spans.Trace(traceID) + return spans, cutoff, err +} + // PruneParquet removes complete ingest batches older than cutoff. A batch that // straddles the boundary is retained intact, so retention never removes newer // telemetry from another signal in the same atomic commit. @@ -159,7 +215,7 @@ func (r *Repository) PruneParquet(cutoff int64) (int, error) { if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { return 0, errors.Join(removeErr, err) } - next := repositoryManifest{Version: 1, Batches: kept} + next := repositoryManifest{Version: 1, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} if err := writeRepositoryManifest(r.root, next); err != nil { return 0, errors.Join(removeErr, err) } @@ -198,6 +254,34 @@ func (r *Repository) Commit(batch Batch) error { return r.removeWAL(batch.ID) } +// Stage durably records a batch in the WAL without publishing its projections. +// Writers call this before handing a batch to an asynchronous commit worker, so +// every queued or in-flight batch is replayable if shutdown interrupts retries. +func (r *Repository) Stage(batch Batch) error { + if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { + return errors.New("telemetry batch requires a safe ID") + } + normalizeBatch(&batch) + r.commitMu.Lock() + defer r.commitMu.Unlock() + consumed := r.batchConsumedLocked(batch.ID) + if consumed { + return r.removeWAL(batch.ID) + } + return r.writeWAL(batch) +} + +// Discard removes a durably staged batch after the writer has explicitly +// classified it as poison and accounted every row as dropped. +func (r *Repository) Discard(id string) error { + if id == "" || strings.ContainsAny(id, `/\\`) { + return errors.New("telemetry batch requires a safe ID") + } + r.commitMu.Lock() + defer r.commitMu.Unlock() + return r.removeWAL(id) +} + func (r *Repository) apply(batch Batch) error { if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { return fmt.Errorf("commit span segment: %w", err) @@ -466,6 +550,11 @@ func writeRepositoryManifest(root string, manifest repositoryManifest) error { func normalizeBatch(batch *Batch) { for i := range batch.Spans { batch.Spans[i].Namespace = telemetry.NormalizeNamespace(batch.Spans[i].Namespace) + if batch.Spans[i].StartUnixNanos == 0 { + // Mirror the Parquet start_time coalesce so hot segments and cold SQL + // key a zero-start span on the same instant. + batch.Spans[i].StartUnixNanos = batch.Spans[i].IngestedAt + } } for i := range batch.Logs { batch.Logs[i].Namespace = telemetry.NormalizeNamespace(batch.Logs[i].Namespace) diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 4777c41f..bd1d858f 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -203,6 +204,42 @@ func TestRepositoryPrunesOnlyCompleteExpiredParquetBatches(t *testing.T) { } } +func TestRepositoryPersistsHotPruneBoundary(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + batch.ID = "hot-boundary" + batch.Logs = []telemetry.Log{{EventUnixNanos: 100}, {EventUnixNanos: 300}} + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + if _, err := repository.PruneHot(250); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + var timestamps []int64 + cutoff, err := reopened.ScanHotLogs(0, 400, func(row telemetry.Log) bool { + timestamps = append(timestamps, row.EventUnixNanos) + return true + }) + if err != nil { + t.Fatal(err) + } + if cutoff != 250 || !slices.Equal(timestamps, []int64{300}) { + t.Fatalf("cutoff=%d timestamps=%v, want cutoff 250 and only hot timestamp 300", cutoff, timestamps) + } +} + func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -218,6 +255,9 @@ func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { t.Fatal(err) } } + if _, err := repository.PruneHot(50); err != nil { + t.Fatal(err) + } db, err := sql.Open("duckdb", "") if err != nil { t.Fatal(err) @@ -237,6 +277,9 @@ func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { if stats["spans"].Files != 1 { t.Fatalf("span files = %d, want 1", stats["spans"].Files) } + if repository.manifest.HotCutoffNanos != 50 { + t.Fatalf("hot cutoff after compaction = %d, want 50", repository.manifest.HotCutoffNanos) + } pattern := filepath.ToSlash(filepath.Join(dir, "parquet", "spans", "*.parquet")) var rows int if err := db.QueryRow("SELECT count(*) FROM read_parquet(?)", pattern).Scan(&rows); err != nil { @@ -488,3 +531,29 @@ func testBatchAt(timestamp int64) Batch { batch.Metrics[0].IngestedAt = timestamp return batch } + +func TestNormalizeBatchBackfillsSpanStartFromIngestedAt(t *testing.T) { + batch := Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", IngestedAt: 12345}}} + normalizeBatch(&batch) + if got := batch.Spans[0].StartUnixNanos; got != 12345 { + t.Fatalf("StartUnixNanos = %d, want ingested-at fallback 12345", got) + } +} + +func TestCompactionSourcesDropInheritedLedger(t *testing.T) { + selected := []batchMetadata{ + {ID: "raw-1"}, + {ID: "out-1", Sources: []string{"old-a", "old-b"}}, + {ID: "raw-2"}, + } + got := compactionSources(selected) + want := []string{"raw-1", "raw-2"} + if len(got) != len(want) { + t.Fatalf("compactionSources = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("compactionSources = %v, want %v", got, want) + } + } +} diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index dd7620b8..f44941a3 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "log/slog" "time" @@ -17,7 +18,9 @@ const ( ) type batchCommitter interface { + Stage(Batch) error Commit(Batch) error + Discard(string) error } type Writer struct { @@ -116,6 +119,20 @@ func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { return nil } batch := Batch{ID: uuid.NewString(), Spans: append([]telemetry.Span(nil), w.bufSpans...), Logs: append([]telemetry.Log(nil), w.bufLogs...), Metrics: append([]telemetry.Metric(nil), w.bufMetrics...)} + // Establish durability before the asynchronous handoff. From this point on, + // cancellation may stop retries or leave batches queued, but every row is + // replayable from WAL on the next start. When the WAL itself is unwritable + // no durability exists to protect; drop this batch with visible accounting + // and keep the process serving rather than shutting everything down. + if err := w.repository.Stage(batch); err != nil { + metrics.FlushErrors.WithLabelValues("stage").Inc() + recordDroppedBatch(batch) + slog.Error("telemetry batch could not be staged durably; dropping", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) + w.bufSpans = w.bufSpans[:0] + w.bufLogs = w.bufLogs[:0] + w.bufMetrics = w.bufMetrics[:0] + return nil + } select { case out <- batch: w.bufSpans = w.bufSpans[:0] @@ -180,6 +197,10 @@ func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- e } } if !committed { + if err := w.repository.Discard(batch.ID); err != nil { + done <- fmt.Errorf("discard poison telemetry batch %s: %w", batch.ID, err) + return + } recordDroppedBatch(batch) slog.Error("telemetry batch permanently failed; dropping after bounded retries", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) } diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index c4f510b7..1d57765a 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -3,6 +3,7 @@ package store import ( "context" "errors" + "fmt" "sync" "testing" "time" @@ -14,9 +15,19 @@ type recoveringCommitter struct { mu sync.Mutex failures int calls int + staged []Batch batches []Batch } +func (c *recoveringCommitter) Stage(batch Batch) error { + c.mu.Lock() + defer c.mu.Unlock() + c.staged = append(c.staged, batch) + return nil +} + +func (c *recoveringCommitter) Discard(string) error { return nil } + func (c *recoveringCommitter) Commit(batch Batch) error { c.mu.Lock() defer c.mu.Unlock() @@ -48,11 +59,86 @@ func TestWriterRetainsBatchAcrossCommitFailures(t *testing.T) { if committer.calls != 7 { t.Fatalf("Commit calls = %d, want 7", committer.calls) } + if len(committer.staged) != 1 { + t.Fatalf("staged batches = %d, want 1", len(committer.staged)) + } if len(committer.batches) != 1 || len(committer.batches[0].Spans) != 1 { t.Fatalf("committed batches = %#v, want original batch exactly once", committer.batches) } } +type durableFailCommitter struct { + repository *Repository + attempted chan struct{} + staged chan struct{} + once sync.Once +} + +func (c *durableFailCommitter) Stage(batch Batch) error { + if err := c.repository.Stage(batch); err != nil { + return err + } + c.staged <- struct{}{} + return nil +} + +func (c *durableFailCommitter) Discard(id string) error { return c.repository.Discard(id) } + +func (c *durableFailCommitter) Commit(Batch) error { + c.once.Do(func() { close(c.attempted) }) + return errors.New("storage stalled") +} + +func TestWriterShutdownReplaysEveryStagedBatch(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + spans := make(chan telemetry.Span, 5) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + for i := range 5 { + spans <- telemetry.Span{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: int64(100 + i)} + } + committer := &durableFailCommitter{repository: repository, attempted: make(chan struct{}), staged: make(chan struct{}, 5)} + w := &Writer{ + repository: committer, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return time.Hour }, shutdownGrace: 25 * time.Millisecond, + } + ctx, cancel := context.WithCancel(context.Background()) + finished := make(chan error, 1) + go func() { finished <- w.Run(ctx) }() + for range 5 { + select { + case <-committer.staged: + case <-time.After(time.Second): + t.Fatal("queued batch was not staged") + } + } + select { + case <-committer.attempted: + case <-time.After(time.Second): + t.Fatal("commit attempt did not start") + } + cancel() + if err := <-finished; !errors.Is(err, context.Canceled) { + t.Fatalf("Run error = %v, want context canceled", err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + recovered, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if got := recovered.Spans.RowCount(); got != 5 { + t.Fatalf("replayed spans = %d, want 5", got) + } +} + func TestWriterDropsPoisonBatchAfterBoundedRetries(t *testing.T) { spans := make(chan telemetry.Span, 1) logs := make(chan telemetry.Log) @@ -115,3 +201,50 @@ func TestWriterCancellationInterruptsCommitBackoff(t *testing.T) { t.Fatal("writer shutdown remained blocked in retry backoff") } } + +type stageFailCommitter struct { + mu sync.Mutex + stages int + commits int +} + +func (c *stageFailCommitter) Stage(Batch) error { + c.mu.Lock() + defer c.mu.Unlock() + c.stages++ + return errors.New("wal device unavailable") +} + +func (c *stageFailCommitter) Discard(string) error { return nil } + +func (c *stageFailCommitter) Commit(Batch) error { + c.mu.Lock() + defer c.mu.Unlock() + c.commits++ + return nil +} + +func TestWriterSurvivesStageFailure(t *testing.T) { + spans := make(chan telemetry.Span, 1) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} + close(spans) + close(logs) + close(metricRows) + committer := &stageFailCommitter{} + w := &Writer{ + repository: committer, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return 0 }, + } + if err := w.Run(context.Background()); err != nil { + t.Fatalf("Run error = %v, want nil: an unstageable batch must be dropped with accounting, not kill the writer", err) + } + if committer.stages == 0 { + t.Fatal("Stage was never attempted") + } + if committer.commits != 0 { + t.Fatalf("Commit calls = %d, want 0 for an unstaged batch", committer.commits) + } +} From 05a77f9260374b74c36f22bfaeabe9bd55ea6ca8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 20:58:21 -0700 Subject: [PATCH 07/31] fix(storage): keep telemetry replayable and bound cold reads Never delete a batch's WAL to resolve a commit failure: exhausted retries now defer the batch to replay on the next start, and replay quarantines a batch that cannot apply so a poison entry cannot crash-loop the server. A batch that cannot be staged is carried to the next flush instead of dropped, bounded by three batches of rows. Bound the cold log path in DuckDB again with a limited sample query and a grouped histogram query, so a wide window no longer streams the whole retained tier through the driver. Give the Parquet gate a bounded publisher grace: readers keep entering while maintenance queues, but only until the grace expires, so retention and compaction can no longer be starved by overlapping query traffic. Validate segment headers with subtraction-based bounds in both the span and signal stores, so a wrapping offset cannot pass the check and size an allocation, and count canonical OTLP error status in segment rollups. Claude-Session: https://claude.ai/code/session_01Cxec3QnsbwcU1dqaFCtnTf --- internal/observability/logs.go | 59 +++++- internal/observability/service_test.go | 62 ++++++- internal/query/duck_test.go | 8 +- internal/query/parquet_gate.go | 84 +++++++-- internal/query/parquet_gate_test.go | 94 ++++++++++ internal/telemetry/segment/signal_store.go | 27 ++- .../telemetry/segment/signal_store_test.go | 16 ++ internal/telemetry/segment/span_store.go | 51 +++++- internal/telemetry/segment/span_store_test.go | 56 ++++++ internal/telemetry/store/repository.go | 24 ++- internal/telemetry/store/writer.go | 88 ++++++--- internal/telemetry/store/writer_test.go | 173 +++++++++++++++++- 12 files changed, 652 insertions(+), 90 deletions(-) create mode 100644 internal/query/parquet_gate_test.go diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 2da63f35..5c50428f 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -59,16 +59,32 @@ func retainEarliest(entries *earliestLogHeap, entry LogEntry, limit int) { } } -var coldLogsQuery = ` -SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, - coalesce(trace_id, ''), coalesce(span_id, '') -FROM logs +var coldLogFilters = ` WHERE time >= ? AND time < ? AND (? = '' OR namespace = ?) AND (? = '' OR service = ?) AND (? = '' OR lower(severity) = lower(?)) - AND (? = '' OR contains(lower(` + redactLogBodySQL("body") + `), lower(?))) -ORDER BY time ASC` + AND (? = '' OR contains(lower(` + redactLogBodySQL("body") + `), lower(?)))` + +// The cold tier answers the two questions the API actually asks: the newest +// `limit` entries, and per-bucket counts. Both stay bounded in DuckDB — the +// entry sample by LIMIT, the histogram by GROUP BY — so a wide window costs a +// page of rows instead of the whole retained window streamed through the +// driver. +var coldLogEntriesQuery = ` +SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, + coalesce(trace_id, ''), coalesce(span_id, '') +FROM logs` + coldLogFilters + ` +ORDER BY time DESC +LIMIT ?` + +var coldLogBucketsQuery = ` +SELECT time_bucket(INTERVAL '5 minutes', time) AS point_time, + coalesce(nullif(upper(severity), ''), 'UNSPECIFIED') AS bucket_severity, + CAST(count(*) AS BIGINT) +FROM logs` + coldLogFilters + ` +GROUP BY point_time, bucket_severity +ORDER BY point_time ASC, bucket_severity ASC` func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, search string, limit int) (Result[Logs], error) { scope, err := s.normalizeScope(scope) @@ -124,9 +140,9 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear hotStart := max(hotCutoff, startNanos) usedCold := coldEnd > startNanos if usedCold { - rows, queryErr := s.db.QueryContext(ctx, coldLogsQuery, - scope.Start, time.Unix(0, coldEnd).UTC(), - scope.Namespace, scope.Namespace, service, service, severity, severity, search, search) + coldStop := time.Unix(0, coldEnd).UTC() + filters := []any{scope.Start, coldStop, scope.Namespace, scope.Namespace, service, service, severity, severity, search, search} + rows, queryErr := s.db.QueryContext(ctx, coldLogEntriesQuery, append(append([]any{}, filters...), limit)...) if queryErr != nil { return Result[Logs]{}, fmt.Errorf("query cold logs: %w", queryErr) } @@ -136,13 +152,36 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear rows.Close() return Result[Logs]{}, fmt.Errorf("scan cold log: %w", err) } - accumulate(entry) + retainNewest(&entries, entry, limit) } if err := rows.Err(); err != nil { rows.Close() return Result[Logs]{}, fmt.Errorf("iterate cold logs: %w", err) } rows.Close() + + bucketRows, bucketErr := s.db.QueryContext(ctx, coldLogBucketsQuery, filters...) + if bucketErr != nil { + return Result[Logs]{}, fmt.Errorf("query cold log histogram: %w", bucketErr) + } + for bucketRows.Next() { + var ( + bucketTime time.Time + bucketSeverity string + count int64 + ) + if err := bucketRows.Scan(&bucketTime, &bucketSeverity, &count); err != nil { + bucketRows.Close() + return Result[Logs]{}, fmt.Errorf("scan cold log bucket: %w", err) + } + matched += int(count) + buckets[bucketKey{time: bucketTime.UTC().UnixNano(), severity: bucketSeverity}] += count + } + if err := bucketRows.Err(); err != nil { + bucketRows.Close() + return Result[Logs]{}, fmt.Errorf("iterate cold log histogram: %w", err) + } + bucketRows.Close() } if err := ctx.Err(); err != nil { return Result[Logs]{}, err diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 1b7938b4..d69a03ba 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "regexp" + "strings" "testing" "time" @@ -306,10 +307,14 @@ func TestLogsFallsBackToParquetOutsideHotRetention(t *testing.T) { if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { t.Fatal(err) } - mock.ExpectQuery(regexp.QuoteMeta(coldLogsQuery)). - WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", ""). + mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(time.Minute), "ERROR", "checkout", "token=[REDACTED]", "trace-cold", "")) + mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). + AddRow(start, "ERROR", int64(1))) result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "checkout", "error", "", 10) if err != nil { t.Fatal(err) @@ -342,12 +347,16 @@ func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { t.Fatal(err) } - mock.ExpectQuery(regexp.QuoteMeta(coldLogsQuery)). - WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", ""). + mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). + WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(100*time.Millisecond), "INFO", "", "late-old", "", ""). AddRow(start.Add(150*time.Millisecond), "INFO", "", "newer-old", "", ""). AddRow(start.Add(200*time.Millisecond), "INFO", "", "late-boundary", "", "")) + mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). + WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). + AddRow(start, "INFO", int64(3))) result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "", "", 10) if err != nil { t.Fatal(err) @@ -451,3 +460,48 @@ func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { } var _ DB = queryrows.SQLAdapter{} + +func TestLogsBoundsColdQueryWithLimitAndAggregatedBuckets(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-cold", Logs: []telemetry.Log{{ + Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", + ServiceName: "checkout", Body: "hello", TraceID: "trace-cold", + }}}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { + t.Fatal(err) + } + // The sample query must carry the row limit so a wide window cannot stream + // the whole cold tier through the driver. + mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). + WithArgs(start, end, "prod", "prod", "", "", "", "", "", "", 2). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(3*time.Minute), "ERROR", "checkout", "newest", "trace-c", ""). + AddRow(start.Add(2*time.Minute), "INFO", "checkout", "older", "trace-b", "")) + // Histogram counts come back aggregated, so a million matching rows cost + // one row per bucket rather than a million transfers. + mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). + WithArgs(start, end, "prod", "prod", "", "", "", "", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). + AddRow(start, "ERROR", int64(900000)). + AddRow(start.Add(5*time.Minute), "INFO", int64(100000))) + result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "", "", 2) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Entries) != 2 || result.Data.Entries[0].Body != "newest" { + t.Fatalf("entries = %#v, want the two newest sampled rows", result.Data.Entries) + } + if len(result.Data.Buckets) != 2 { + t.Fatalf("buckets = %#v, want one row per aggregated bucket", result.Data.Buckets) + } + if !strings.Contains(result.Summary, "1000000") { + t.Fatalf("summary = %q, want the aggregated match count", result.Summary) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 41990bca..83e01b7a 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -228,13 +228,7 @@ func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { close(writerDone) }() deadline := time.Now().Add(time.Second) - for { - d.parquetMu.mu.Lock() - waiting := d.parquetMu.waitingWriters - d.parquetMu.mu.Unlock() - if waiting > 0 { - break - } + for d.parquetMu.WaitingWriters() == 0 { if time.Now().After(deadline) { d.parquetMu.RUnlock() t.Fatal("maintenance writer did not begin waiting") diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 3dd13a2c..e40caf62 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -1,29 +1,68 @@ package query -import "sync" +import ( + "sync" + "time" +) + +// defaultWriterGrace bounds how long readers may keep entering ahead of a +// waiting publisher. Long enough that ordinary dashboard traffic never queues +// behind maintenance, short enough that retention and compaction always run. +const defaultWriterGrace = 5 * time.Second // parquetReadGate protects Parquet file publication without sync.RWMutex's -// writer preference. Maintenance may wait behind a long reader, but merely -// queuing that maintenance must never block unrelated API reads or readiness -// probes. Once readers drain, the writer publishes while new readers wait. +// unconditional writer preference. Queuing maintenance must not stall +// unrelated API reads or readiness probes, so readers continue to be admitted +// while a publisher waits — but only until that publisher's grace period +// expires, after which new readers queue so retention and compaction cannot be +// starved by overlapping query traffic. type parquetReadGate struct { - once sync.Once - mu sync.Mutex - changed *sync.Cond - readers int - writer bool - waitingWriters int + once sync.Once + mu sync.Mutex + changed *sync.Cond + readers int + writer bool + waiting []time.Time + writerGrace time.Duration + now func() time.Time } func (g *parquetReadGate) init() { g.once.Do(func() { g.changed = sync.NewCond(&g.mu) }) } +func (g *parquetReadGate) clock() time.Time { + if g.now != nil { + return g.now() + } + return time.Now() +} + +func (g *parquetReadGate) grace() time.Duration { + if g.writerGrace != 0 { + return g.writerGrace + } + return defaultWriterGrace +} + +// admitsReaderLocked reports whether a new reader may enter. A reader is +// refused once a publisher is active, or once the longest-waiting publisher +// has been queued for longer than the grace period. +func (g *parquetReadGate) admitsReaderLocked() bool { + if g.writer { + return false + } + if len(g.waiting) == 0 { + return true + } + return g.clock().Sub(g.waiting[0]) < g.grace() +} + func (g *parquetReadGate) TryRLock() bool { g.init() g.mu.Lock() defer g.mu.Unlock() - if g.writer { + if !g.admitsReaderLocked() { return false } g.readers++ @@ -33,7 +72,7 @@ func (g *parquetReadGate) TryRLock() bool { func (g *parquetReadGate) RLock() { g.init() g.mu.Lock() - for g.writer { + for !g.admitsReaderLocked() { g.changed.Wait() } g.readers++ @@ -57,11 +96,20 @@ func (g *parquetReadGate) RUnlock() { func (g *parquetReadGate) Lock() { g.init() g.mu.Lock() - g.waitingWriters++ + queued := g.clock() + g.waiting = append(g.waiting, queued) + // Wake any readers parked on an earlier publisher so they re-evaluate this + // publisher's grace, and so the grace clock starts for readers immediately. + g.changed.Broadcast() for g.writer || g.readers > 0 { g.changed.Wait() } - g.waitingWriters-- + for i, at := range g.waiting { + if at.Equal(queued) { + g.waiting = append(g.waiting[:i], g.waiting[i+1:]...) + break + } + } g.writer = true g.mu.Unlock() } @@ -77,3 +125,11 @@ func (g *parquetReadGate) Unlock() { g.changed.Broadcast() g.mu.Unlock() } + +// WaitingWriters reports how many publishers are queued behind active readers. +func (g *parquetReadGate) WaitingWriters() int { + g.init() + g.mu.Lock() + defer g.mu.Unlock() + return len(g.waiting) +} diff --git a/internal/query/parquet_gate_test.go b/internal/query/parquet_gate_test.go new file mode 100644 index 00000000..cda531a2 --- /dev/null +++ b/internal/query/parquet_gate_test.go @@ -0,0 +1,94 @@ +package query + +import ( + "sync" + "testing" + "time" +) + +// fakeClock lets the gate's grace period be crossed deterministically. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func waitForQueuedWriter(t *testing.T, gate *parquetReadGate) { + t.Helper() + deadline := time.Now().Add(time.Second) + for gate.WaitingWriters() == 0 { + if time.Now().After(deadline) { + t.Fatal("publisher never queued") + } + time.Sleep(time.Millisecond) + } +} + +func TestParquetGateAdmitsReadersWhileWriterGraceRuns(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} + gate := &parquetReadGate{now: clock.Now} + if !gate.TryRLock() { + t.Fatal("first reader was not admitted") + } + go gate.Lock() + waitForQueuedWriter(t, gate) + clock.Advance(defaultWriterGrace / 2) + if !gate.TryRLock() { + t.Fatal("reader refused while the publisher was still inside its grace period") + } + gate.RUnlock() + gate.RUnlock() +} + +func TestParquetGateQueuesReadersOnceWriterGraceExpires(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} + gate := &parquetReadGate{now: clock.Now} + if !gate.TryRLock() { + t.Fatal("first reader was not admitted") + } + go gate.Lock() + waitForQueuedWriter(t, gate) + clock.Advance(defaultWriterGrace + time.Second) + if gate.TryRLock() { + gate.RUnlock() + gate.RUnlock() + t.Fatal("reader admitted past an expired publisher grace; maintenance can starve indefinitely") + } + gate.RUnlock() +} + +func TestParquetGatePublishesAfterOverlappingReadersDrain(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} + gate := &parquetReadGate{now: clock.Now} + gate.RLock() + published := make(chan struct{}) + go func() { + gate.Lock() + close(published) + gate.Unlock() + }() + waitForQueuedWriter(t, gate) + // Overlapping traffic keeps arriving, but only within the grace period. + if !gate.TryRLock() { + t.Fatal("reader refused inside the grace period") + } + clock.Advance(defaultWriterGrace + time.Second) + gate.RUnlock() + gate.RUnlock() + select { + case <-published: + case <-time.After(2 * time.Second): + t.Fatal("publisher never ran after readers drained") + } +} diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index 15ef6432..ff374bc0 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -405,6 +405,18 @@ func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { return len(removed), errors.Join(removeErr, syncDir(s.dir)) } +// validateSignalDirectory bounds the block directory against the file size, +// so a torn header cannot size an allocation the file could never hold. +func validateSignalDirectory(size, dirOffset uint64, blockCount uint32) error { + if dirOffset < signalHeaderSize || dirOffset > size { + return errors.New("corrupt directory offset") + } + if uint64(blockCount) > (size-dirOffset)/signalBlockSize { + return errors.New("directory does not fit in the segment") + } + return nil +} + func openSignalSegment(path string) (signalSegment, error) { f, err := os.Open(path) if err != nil { @@ -424,13 +436,18 @@ func openSignalSegment(path string) (signalSegment, error) { min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40])), fingerprint: binary.LittleEndian.Uint64(header[48:56]), } - count := int(binary.LittleEndian.Uint32(header[16:20])) - dirOffset := int64(binary.LittleEndian.Uint64(header[40:48])) - if count < 0 || dirOffset < signalHeaderSize { - return signalSegment{}, errors.New("invalid signal directory") + blockCount := binary.LittleEndian.Uint32(header[16:20]) + dirOffset := binary.LittleEndian.Uint64(header[40:48]) + info, err := f.Stat() + if err != nil { + return signalSegment{}, err + } + if err := validateSignalDirectory(uint64(info.Size()), dirOffset, blockCount); err != nil { + return signalSegment{}, fmt.Errorf("signal segment %s: %w", filepath.Base(path), err) } + count := int(blockCount) directory := make([]byte, count*signalBlockSize) - if _, err := f.ReadAt(directory, dirOffset); err != nil { + if _, err := f.ReadAt(directory, int64(dirOffset)); err != nil { return signalSegment{}, err } for i := range count { diff --git a/internal/telemetry/segment/signal_store_test.go b/internal/telemetry/segment/signal_store_test.go index 979b1e12..aa441a42 100644 --- a/internal/telemetry/segment/signal_store_test.go +++ b/internal/telemetry/segment/signal_store_test.go @@ -66,3 +66,19 @@ func stringID(i int) string { const digits = "0123456789abcdef" return "batch-" + string([]byte{digits[(i>>4)&15], digits[i&15]}) } + +func TestValidateSignalDirectoryRejectsOutOfBoundsCount(t *testing.T) { + const size = 4096 + if err := validateSignalDirectory(size, signalHeaderSize, 0xFFFFFFFF); err == nil { + t.Fatal("validateSignalDirectory accepted a block count larger than the file") + } + if err := validateSignalDirectory(size, ^uint64(0)-1024, 64); err == nil { + t.Fatal("validateSignalDirectory accepted a wrapping directory offset") + } + if err := validateSignalDirectory(size, 0, 1); err == nil { + t.Fatal("validateSignalDirectory accepted a directory inside the header") + } + if err := validateSignalDirectory(size, signalHeaderSize, 8); err != nil { + t.Fatalf("validateSignalDirectory rejected a sound header: %v", err) + } +} diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index 95a0b7ee..b0a5a731 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -423,7 +423,7 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { rollups[key] = r } r.calls++ - if row.StatusCode == "ERROR" { + if isErrorStatus(row.StatusCode) { r.errors++ } r.duration += row.DurationMS @@ -748,7 +748,7 @@ func (s *Store) ScanService(namespace, service string, start, end int64) (Aggreg defer s.mu.RUnlock() var out Aggregate wanted := []int{colNamespace, colServiceName, colStartUnixNanos, colDurationMS, colStatusCode} - namespaceNeedle, serviceNeedle, errorNeedle := []byte(namespace), []byte(service), []byte("ERROR") + namespaceNeedle, serviceNeedle := []byte(namespace), []byte(service) for i := range s.segments { seg := s.segments[i] if seg.max < start || seg.min >= end { @@ -810,7 +810,7 @@ func (s *Store) ScanService(namespace, service string, start, end int64) (Aggreg } out.Calls++ out.DurationMS += float64At(columns[colDurationMS], row) - if bytes.Equal(statusValue, errorNeedle) { + if isErrorStatus(string(statusValue)) { out.Errors++ } } @@ -854,6 +854,43 @@ func (s *Store) readColumns(f *os.File, block blockDir, wanted []int) (map[int][ return columns, err } +// isErrorStatus matches both status spellings the lake carries: OTLP ingest +// stores Status.Code.String() ("STATUS_CODE_ERROR"), while other producers and +// older rows use the bare code. The DuckDB rollups compare against the same +// pair. +func isErrorStatus(status string) bool { + return strings.EqualFold(status, "ERROR") || strings.EqualFold(status, "STATUS_CODE_ERROR") +} + +// validateSegmentSections bounds every header offset against the file before +// any of them is used to size an allocation. All comparisons are written as +// subtractions against size so a corrupt offset near the top of the address +// space cannot wrap past the check. +func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, blockCount uint32) error { + if err := validateDirectory(size, dirOffset, blockCount, blockDirSize); err != nil { + return err + } + if indexOffset < dirOffset+uint64(blockCount)*blockDirSize || indexOffset > size { + return errors.New("corrupt trace index offset") + } + if rollupOffset < indexOffset || rollupOffset > size { + return errors.New("corrupt rollup offset") + } + return nil +} + +// validateDirectory reports whether count fixed-size entries fit in the file +// when placed at offset. +func validateDirectory(size, offset uint64, count uint32, entrySize uint64) error { + if offset < headerSize || offset > size { + return errors.New("corrupt directory offset") + } + if uint64(count) > (size-offset)/entrySize { + return errors.New("directory does not fit in the segment") + } + return nil +} + func openSegment(path string) (segment, error) { f, err := os.Open(path) if err != nil { @@ -880,12 +917,8 @@ func openSegment(path string) (segment, error) { if err != nil { return segment{}, err } - // Header offsets come from disk; a torn or bit-rotted segment must fail - // with an error naming the file, never drive a negative or huge make(). - size := uint64(info.Size()) - dirEnd := dirOffset + uint64(blockCount)*blockDirSize - if dirOffset < headerSize || dirEnd > indexOffset || indexOffset > rollupOffset || rollupOffset > size { - return segment{}, fmt.Errorf("segment %s has corrupt section offsets", filepath.Base(path)) + if err := validateSegmentSections(uint64(info.Size()), dirOffset, indexOffset, rollupOffset, blockCount); err != nil { + return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) } seg.blocks = make([]blockDir, blockCount) buf := make([]byte, int(blockCount)*blockDirSize) diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index e60e521d..4e54c423 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -209,3 +209,59 @@ func TestOpenRejectsSegmentWithCorruptSectionOffsets(t *testing.T) { }) }) } + +func TestEndpointsCountCanonicalOTelErrorStatus(t *testing.T) { + dir := t.TempDir() + base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() + rows := []Span{ + {Namespace: "default", TraceID: "t1", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/x", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, + {Namespace: "default", TraceID: "t2", SpanID: "2", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/x", StartUnixNanos: base + 1, EndUnixNanos: base + 2, DurationMS: 1, StatusCode: "STATUS_CODE_OK"}, + } + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err := store.AppendID("seg-status", rows); err != nil { + t.Fatal(err) + } + endpoints := store.Endpoints("default", "api", base, base+int64(time.Minute), 10) + if len(endpoints) != 1 { + t.Fatalf("endpoints = %#v, want one route", endpoints) + } + if endpoints[0].Errors != 1 { + t.Fatalf("Errors = %d, want 1: OTLP ingest stores Status.Code.String() as STATUS_CODE_ERROR", endpoints[0].Errors) + } + agg, err := store.ScanService("default", "api", base, base+int64(time.Minute)) + if err != nil { + t.Fatal(err) + } + if agg.Errors != 1 { + t.Fatalf("aggregate Errors = %d, want 1", agg.Errors) + } +} + +func TestValidateSegmentSectionsRejectsOutOfBoundsDirectory(t *testing.T) { + const size = 4096 + tests := []struct { + name string + dirOffset, indexOffset, rollupOffset uint64 + blockCount uint32 + }{ + {"wrapping directory end", ^uint64(0) - uint64(0xFFFFFFFF)*blockDirSize + 1, 512, 1024, 0xFFFFFFFF}, + {"block count past index", headerSize, 512, 1024, 0xFFFFFFFF}, + {"directory before header", 0, 512, 1024, 1}, + {"sections out of order", headerSize, 2048, 1024, 1}, + {"rollups past end of file", headerSize, 512, size + 1, 1}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := validateSegmentSections(size, test.dirOffset, test.indexOffset, test.rollupOffset, test.blockCount); err == nil { + t.Fatal("validateSegmentSections accepted a corrupt header") + } + }) + } + if err := validateSegmentSections(size, headerSize, 512, 1024, 8); err != nil { + t.Fatalf("validateSegmentSections rejected a sound header: %v", err) + } +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index f130f6a3..7878e031 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -271,17 +271,6 @@ func (r *Repository) Stage(batch Batch) error { return r.writeWAL(batch) } -// Discard removes a durably staged batch after the writer has explicitly -// classified it as poison and accounted every row as dropped. -func (r *Repository) Discard(id string) error { - if id == "" || strings.ContainsAny(id, `/\\`) { - return errors.New("telemetry batch requires a safe ID") - } - r.commitMu.Lock() - defer r.commitMu.Unlock() - return r.removeWAL(id) -} - func (r *Repository) apply(batch Batch) error { if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { return fmt.Errorf("commit span segment: %w", err) @@ -386,11 +375,20 @@ func (r *Repository) recover() error { } continue } + // A batch that cannot be applied would otherwise abort every boot. Move + // it aside so the instance starts; if quarantine itself fails the + // environment is broken, and failing loudly is then the right answer. if err := r.apply(batch); err != nil { - return fmt.Errorf("replay %s: %w", name, err) + if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { + return errors.Join(fmt.Errorf("replay %s: %w", name, err), quarantineErr) + } + continue } if err := r.recordBatch(batch); err != nil { - return fmt.Errorf("record replayed %s: %w", name, err) + if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { + return errors.Join(fmt.Errorf("record replayed %s: %w", name, err), quarantineErr) + } + continue } if err := os.Remove(filepath.Join(r.walDir, name)); err != nil { return err diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index f44941a3..6ac07f91 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -2,7 +2,6 @@ package store import ( "context" - "fmt" "log/slog" "time" @@ -13,6 +12,7 @@ import ( const ( flushQueueDepth = 4 + carryBatches = 3 commitRetryLimit = 8 writerShutdownGrace = 5 * time.Second ) @@ -20,7 +20,6 @@ const ( type batchCommitter interface { Stage(Batch) error Commit(Batch) error - Discard(string) error } type Writer struct { @@ -56,7 +55,7 @@ func (w *Writer) Run(ctx context.Context) error { spans, logs, metricRows := w.spans, w.logs, w.metricRows finish := func() error { w.drain(&spans, &logs, &metricRows) - if err := w.flush(flushes, workerDone); err != nil { + if err := w.flushBuffered(flushes, workerDone, true); err != nil { return err } close(flushes) @@ -115,22 +114,29 @@ func (w *Writer) Run(ctx context.Context) error { } func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { + return w.flushBuffered(out, workerDone, false) +} + +// flushBuffered publishes the buffered rows. Until a batch is staged in the +// WAL no durable copy exists, so a failed staging attempt keeps the rows +// buffered for the next tick rather than discarding them — bounded by +// carryBatches so a long storage outage cannot grow the buffer without limit. +// The final flush has no next tick, so there the rows are accounted as dropped. +func (w *Writer) flushBuffered(out chan<- Batch, workerDone <-chan error, final bool) error { if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) == 0 { return nil } batch := Batch{ID: uuid.NewString(), Spans: append([]telemetry.Span(nil), w.bufSpans...), Logs: append([]telemetry.Log(nil), w.bufLogs...), Metrics: append([]telemetry.Metric(nil), w.bufMetrics...)} - // Establish durability before the asynchronous handoff. From this point on, - // cancellation may stop retries or leave batches queued, but every row is - // replayable from WAL on the next start. When the WAL itself is unwritable - // no durability exists to protect; drop this batch with visible accounting - // and keep the process serving rather than shutting everything down. if err := w.repository.Stage(batch); err != nil { metrics.FlushErrors.WithLabelValues("stage").Inc() - recordDroppedBatch(batch) - slog.Error("telemetry batch could not be staged durably; dropping", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) - w.bufSpans = w.bufSpans[:0] - w.bufLogs = w.bufLogs[:0] - w.bufMetrics = w.bufMetrics[:0] + if final { + recordDroppedBatch(batch) + slog.Error("telemetry batch could not be staged durably during shutdown; dropping", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) + w.resetBuffers() + return nil + } + slog.Warn("telemetry batch could not be staged durably; retrying on the next flush", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) + w.trimCarry() return nil } select { @@ -197,20 +203,58 @@ func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- e } } if !committed { - if err := w.repository.Discard(batch.ID); err != nil { - done <- fmt.Errorf("discard poison telemetry batch %s: %w", batch.ID, err) - return - } - recordDroppedBatch(batch) - slog.Error("telemetry batch permanently failed; dropping after bounded retries", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) + // The batch stays in the WAL. Its projections may be partly published, + // and only replay can finish the transaction and register the batch in + // the manifest, so deleting the WAL here would both lose the rows and + // strand any parquet file the failed attempt already wrote. + metrics.FlushErrors.WithLabelValues("deferred").Inc() + slog.Error("telemetry batch commit failed after bounded retries; deferring to WAL replay on next start", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) } } } +func (w *Writer) resetBuffers() { + w.bufSpans = w.bufSpans[:0] + w.bufLogs = w.bufLogs[:0] + w.bufMetrics = w.bufMetrics[:0] +} + +// trimCarry bounds the rows held for a later staging attempt. Once the carry +// exceeds carryBatches worth of rows the oldest are dropped with accounting, +// so an unwritable WAL degrades to visible loss instead of unbounded memory. +func (w *Writer) trimCarry() { + limit := w.batchSize * carryBatches + if limit <= 0 { + return + } + spans, logs, metricRows := 0, 0, 0 + if overflow := len(w.bufSpans) - limit; overflow > 0 { + spans = overflow + w.bufSpans = append(w.bufSpans[:0], w.bufSpans[overflow:]...) + } + if overflow := len(w.bufLogs) - limit; overflow > 0 { + logs = overflow + w.bufLogs = append(w.bufLogs[:0], w.bufLogs[overflow:]...) + } + if overflow := len(w.bufMetrics) - limit; overflow > 0 { + metricRows = overflow + w.bufMetrics = append(w.bufMetrics[:0], w.bufMetrics[overflow:]...) + } + if spans+logs+metricRows == 0 { + return + } + recordDropped(spans, logs, metricRows) + slog.Error("telemetry carry buffer is full; dropping the oldest rows", "spans", spans, "logs", logs, "metrics", metricRows) +} + func recordDroppedBatch(batch Batch) { - metrics.RowsDropped.WithLabelValues("spans").Add(float64(len(batch.Spans))) - metrics.RowsDropped.WithLabelValues("logs").Add(float64(len(batch.Logs))) - metrics.RowsDropped.WithLabelValues("metrics").Add(float64(len(batch.Metrics))) + recordDropped(len(batch.Spans), len(batch.Logs), len(batch.Metrics)) +} + +func recordDropped(spans, logs, metricRows int) { + metrics.RowsDropped.WithLabelValues("spans").Add(float64(spans)) + metrics.RowsDropped.WithLabelValues("logs").Add(float64(logs)) + metrics.RowsDropped.WithLabelValues("metrics").Add(float64(metricRows)) } func defaultCommitRetryDelay(attempt int) time.Duration { diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index 1d57765a..0be349ae 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" + "strings" "sync" "testing" "time" @@ -26,8 +29,6 @@ func (c *recoveringCommitter) Stage(batch Batch) error { return nil } -func (c *recoveringCommitter) Discard(string) error { return nil } - func (c *recoveringCommitter) Commit(batch Batch) error { c.mu.Lock() defer c.mu.Unlock() @@ -82,8 +83,6 @@ func (c *durableFailCommitter) Stage(batch Batch) error { return nil } -func (c *durableFailCommitter) Discard(id string) error { return c.repository.Discard(id) } - func (c *durableFailCommitter) Commit(Batch) error { c.once.Do(func() { close(c.attempted) }) return errors.New("storage stalled") @@ -215,8 +214,6 @@ func (c *stageFailCommitter) Stage(Batch) error { return errors.New("wal device unavailable") } -func (c *stageFailCommitter) Discard(string) error { return nil } - func (c *stageFailCommitter) Commit(Batch) error { c.mu.Lock() defer c.mu.Unlock() @@ -248,3 +245,167 @@ func TestWriterSurvivesStageFailure(t *testing.T) { t.Fatalf("Commit calls = %d, want 0 for an unstaged batch", committer.commits) } } + +type ioFailCommitter struct { + repository *Repository + mu sync.Mutex + attempts int +} + +func (c *ioFailCommitter) Stage(batch Batch) error { return c.repository.Stage(batch) } + +func (c *ioFailCommitter) Commit(Batch) error { + c.mu.Lock() + defer c.mu.Unlock() + c.attempts++ + return errors.New("storage unavailable") +} + +func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + spans := make(chan telemetry.Span, 1) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100} + close(spans) + close(logs) + close(metricRows) + committer := &ioFailCommitter{repository: repository} + w := &Writer{ + repository: committer, interval: time.Hour, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return 0 }, + } + if err := w.Run(context.Background()); err != nil { + t.Fatalf("Run error = %v", err) + } + entries, err := os.ReadDir(filepath.Join(dir, "wal")) + if err != nil { + t.Fatal(err) + } + kept := 0 + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".wal") { + kept++ + } + } + if kept != 1 { + t.Fatalf("retained WAL files = %d, want 1: an I/O failure must leave the batch replayable, not delete its only durable copy", kept) + } +} + +func TestRecoverQuarantinesBatchThatCannotApply(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + if err := repository.Stage(batch); err != nil { + t.Fatal(err) + } + // Make the hot span directory unusable so replay's apply always fails. + spanDir := filepath.Join(dir, "hot", "spans") + if err := os.RemoveAll(spanDir); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(spanDir, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Open(dir); err == nil { + t.Fatal("Open succeeded with an unusable hot span directory") + } + if err := os.Remove(spanDir); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("Open error = %v: a batch that cannot apply must be quarantined, not crash-loop every boot", err) + } + defer reopened.Close() +} + +type transientStageCommitter struct { + repository *Repository + mu sync.Mutex + failures int + stages int + committed []Batch +} + +func (c *transientStageCommitter) Stage(batch Batch) error { + c.mu.Lock() + c.stages++ + fail := c.stages <= c.failures + c.mu.Unlock() + if fail { + return errors.New("wal device busy") + } + return c.repository.Stage(batch) +} + +func (c *transientStageCommitter) Commit(batch Batch) error { + if err := c.repository.Commit(batch); err != nil { + return err + } + c.mu.Lock() + defer c.mu.Unlock() + c.committed = append(c.committed, batch) + return nil +} + +func TestWriterCarriesRowsForwardAcrossTransientStageFailure(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + spans := make(chan telemetry.Span, 2) + logs := make(chan telemetry.Log) + metricRows := make(chan telemetry.Metric) + spans <- telemetry.Span{TraceID: "trace", SpanID: "a", StartUnixNanos: 100, IngestedAt: 100} + committer := &transientStageCommitter{repository: repository, failures: 1} + w := &Writer{ + repository: committer, interval: 5 * time.Millisecond, batchSize: 1, + spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), + retryDelay: func(int) time.Duration { return 0 }, + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + finished := make(chan error, 1) + go func() { finished <- w.Run(ctx) }() + deadline := time.Now().Add(time.Second) + for { + committer.mu.Lock() + done := len(committer.committed) + committer.mu.Unlock() + if done > 0 { + break + } + if time.Now().After(deadline) { + cancel() + <-finished + t.Fatal("rows were not retried after a transient Stage failure; they were dropped instead of carried forward") + } + time.Sleep(time.Millisecond) + } + close(spans) + close(logs) + close(metricRows) + cancel() + <-finished + committer.mu.Lock() + defer committer.mu.Unlock() + total := 0 + for _, batch := range committer.committed { + total += len(batch.Spans) + } + if total != 1 { + t.Fatalf("committed spans = %d, want the single carried-forward row", total) + } +} From 6b97e89a64f750ff4ad7adfe8226e953189667b9 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 22:17:16 -0700 Subject: [PATCH 08/31] fix(storage): narrow WAL quarantine and validate segment blocks Quarantine only a payload the projections can never accept. Batch IDs are validated before the WAL promises to replay them, so a poison entry is refused at staging and moved aside on replay, while an apply or manifest failure is treated as environmental: the WAL is retained and startup fails loudly, so a later healthy boot can still finish a batch whose projection prefix was already published. Validate every block extent in both segment stores against the payload region rather than only the directory header, and grow the rollup slice from the rollups actually decoded instead of sizing it from an untrusted count. Claude-Session: https://claude.ai/code/session_01Cxec3QnsbwcU1dqaFCtnTf --- internal/telemetry/segment/signal_store.go | 35 ++++++ .../telemetry/segment/signal_store_test.go | 64 +++++++++++ internal/telemetry/segment/span_store.go | 33 +++++- internal/telemetry/segment/span_store_test.go | 63 +++++++++++ internal/telemetry/store/repository.go | 40 ++++--- internal/telemetry/store/repository_test.go | 101 ++++++++++++++++++ internal/telemetry/store/writer_test.go | 31 ------ 7 files changed, 323 insertions(+), 44 deletions(-) diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index ff374bc0..40f9ed63 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -27,6 +27,11 @@ const ( var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) +// ValidID reports whether id can name a segment file. Callers that persist a +// batch before publishing it use this to reject an ID no projection could ever +// accept, rather than discovering it once part of the batch is already written. +func ValidID(id string) bool { return segmentIDPattern.MatchString(id) } + type signalBlock struct { offset uint64 length uint32 @@ -405,6 +410,33 @@ func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { return len(removed), errors.Join(removeErr, syncDir(s.dir)) } +// validateSignalBlocks bounds every decoded block entry. Scan allocates from +// block.length and decodes block.rows, so both must be known to fit the file +// before the entries are stored. Blocks live between the header and the +// directory, and their row counts must add up to the header's total. +func validateSignalBlocks(size, dirOffset uint64, blocks []signalBlock, rows uint32) error { + var counted uint64 + for _, block := range blocks { + if block.offset < signalHeaderSize || block.offset > dirOffset { + return errors.New("block offset outside the segment payload") + } + if block.length == 0 || uint64(block.length) > dirOffset-block.offset { + return errors.New("block extends past the block directory") + } + if block.rows == 0 || block.rows > signalBlockRows { + return errors.New("block row count is out of range") + } + counted += uint64(block.rows) + } + if counted != uint64(rows) { + return errors.New("block row counts disagree with the segment header") + } + if dirOffset > size { + return errors.New("block directory starts past the end of the segment") + } + return nil +} + // validateSignalDirectory bounds the block directory against the file size, // so a torn header cannot size an allocation the file could never hold. func validateSignalDirectory(size, dirOffset uint64, blockCount uint32) error { @@ -457,6 +489,9 @@ func openSignalSegment(path string) (signalSegment, error) { min: int64(binary.LittleEndian.Uint64(entry[16:24])), max: int64(binary.LittleEndian.Uint64(entry[24:32])), }) } + if err := validateSignalBlocks(uint64(info.Size()), dirOffset, seg.blocks, seg.rows); err != nil { + return signalSegment{}, fmt.Errorf("signal segment %s: %w", filepath.Base(path), err) + } return seg, nil } diff --git a/internal/telemetry/segment/signal_store_test.go b/internal/telemetry/segment/signal_store_test.go index aa441a42..99cd6210 100644 --- a/internal/telemetry/segment/signal_store_test.go +++ b/internal/telemetry/segment/signal_store_test.go @@ -1,7 +1,10 @@ package segment import ( + "encoding/binary" "os" + "path/filepath" + "strings" "sync/atomic" "testing" @@ -82,3 +85,64 @@ func TestValidateSignalDirectoryRejectsOutOfBoundsCount(t *testing.T) { t.Fatalf("validateSignalDirectory rejected a sound header: %v", err) } } + +func TestValidateSignalBlocksRejectsOutOfBoundsExtents(t *testing.T) { + const size, dirOffset = 4096, uint64(2048) + sound := []signalBlock{ + {offset: signalHeaderSize, length: 512, rows: 10}, + {offset: signalHeaderSize + 512, length: 512, rows: 10}, + } + if err := validateSignalBlocks(size, dirOffset, sound, 20); err != nil { + t.Fatalf("validateSignalBlocks rejected sound blocks: %v", err) + } + tests := []struct { + name string + blocks []signalBlock + rows uint32 + }{ + {"length past the directory", []signalBlock{{offset: signalHeaderSize, length: 4096, rows: 10}}, 10}, + {"offset inside the header", []signalBlock{{offset: 0, length: 16, rows: 10}}, 10}, + {"extent wraps", []signalBlock{{offset: ^uint64(0) - 8, length: 64, rows: 10}}, 10}, + {"rows exceed the block cap", []signalBlock{{offset: signalHeaderSize, length: 16, rows: signalBlockRows + 1}}, signalBlockRows + 1}, + {"rows disagree with the header", []signalBlock{{offset: signalHeaderSize, length: 16, rows: 10}}, 11}, + {"empty block", []signalBlock{{offset: signalHeaderSize, length: 0, rows: 0}}, 0}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := validateSignalBlocks(size, dirOffset, test.blocks, test.rows); err == nil { + t.Fatal("validateSignalBlocks accepted a corrupt block directory") + } + }) + } +} + +func TestOpenSignalStoreRejectsCorruptBlockEntry(t *testing.T) { + dir := t.TempDir() + store, err := OpenSignalStore[telemetry.Log](dir, "EventUnixNanos") + if err != nil { + t.Fatal(err) + } + if err := store.Append("seg-block", []telemetry.Log{{EventUnixNanos: 100, Body: "hello"}}); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "seg-block.fseg") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + dirOffset := binary.LittleEndian.Uint64(data[40:48]) + binary.LittleEndian.PutUint32(data[int(dirOffset)+8:], 1<<30) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + _, err = OpenSignalStore[telemetry.Log](dir, "EventUnixNanos") + if err == nil { + t.Fatal("OpenSignalStore accepted a segment whose block extends past its directory") + } + if !strings.Contains(err.Error(), "block extends past the block directory") { + t.Fatalf("OpenSignalStore error = %v, want the block-extent guard to reject it", err) + } +} diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index b0a5a731..a91935ad 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -879,6 +879,32 @@ func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, return nil } +// validateSegmentBlocks bounds every decoded block entry against the payload +// region, so a torn directory cannot drive a read or an allocation the file +// could never satisfy. +func validateSegmentBlocks(size, dirOffset uint64, blocks []blockDir, rows uint32) error { + var counted uint64 + for _, block := range blocks { + if block.offset < headerSize || block.offset > dirOffset { + return errors.New("block offset outside the segment payload") + } + if block.length == 0 || uint64(block.length) > dirOffset-block.offset { + return errors.New("block extends past the block directory") + } + if block.rows == 0 { + return errors.New("block holds no rows") + } + counted += uint64(block.rows) + } + if counted != uint64(rows) { + return errors.New("block row counts disagree with the segment header") + } + if dirOffset > size { + return errors.New("block directory starts past the end of the segment") + } + return nil +} + // validateDirectory reports whether count fixed-size entries fit in the file // when placed at offset. func validateDirectory(size, offset uint64, count uint32, entrySize uint64) error { @@ -929,6 +955,9 @@ func openSegment(path string) (segment, error) { b := buf[i*blockDirSize:] seg.blocks[i] = blockDir{offset: binary.LittleEndian.Uint64(b[0:8]), length: binary.LittleEndian.Uint32(b[8:12]), rows: binary.LittleEndian.Uint32(b[12:16]), min: int64(binary.LittleEndian.Uint64(b[16:24])), max: int64(binary.LittleEndian.Uint64(b[24:32]))} } + if err := validateSegmentBlocks(uint64(info.Size()), dirOffset, seg.blocks, seg.rows); err != nil { + return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) + } indexCompressed := make([]byte, int(rollupOffset-indexOffset)) if _, err := f.ReadAt(indexCompressed, int64(indexOffset)); err != nil { return segment{}, err @@ -962,7 +991,9 @@ func openSegment(path string) (segment, error) { return segment{}, fmt.Errorf("decode rollups: %w", err) } reader := bufio.NewReader(bytes.NewReader(rollupBytes)) - seg.rollups = make([]rollup, 0, rollupCount) + // rollupCount comes from the same untrusted header, so the slice grows with + // the rollups actually decoded rather than being sized from it up front. + seg.rollups = nil for range rollupCount { r, err := readRollup(reader) if err != nil { diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index 4e54c423..d2a02053 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "os" "path/filepath" + "strings" "testing" "time" ) @@ -265,3 +266,65 @@ func TestValidateSegmentSectionsRejectsOutOfBoundsDirectory(t *testing.T) { t.Fatalf("validateSegmentSections rejected a sound header: %v", err) } } + +func TestValidateSegmentBlocksRejectsOutOfBoundsExtents(t *testing.T) { + const size, dirOffset = 4096, uint64(2048) + sound := []blockDir{ + {offset: headerSize, length: 512, rows: 10}, + {offset: headerSize + 512, length: 512, rows: 10}, + } + if err := validateSegmentBlocks(size, dirOffset, sound, 20); err != nil { + t.Fatalf("validateSegmentBlocks rejected sound blocks: %v", err) + } + tests := []struct { + name string + blocks []blockDir + rows uint32 + }{ + {"length past the directory", []blockDir{{offset: headerSize, length: 4096, rows: 10}}, 10}, + {"offset inside the header", []blockDir{{offset: 0, length: 16, rows: 10}}, 10}, + {"extent wraps", []blockDir{{offset: ^uint64(0) - 8, length: 64, rows: 10}}, 10}, + {"rows disagree with the header", []blockDir{{offset: headerSize, length: 16, rows: 10}}, 11}, + {"empty block", []blockDir{{offset: headerSize, length: 0, rows: 0}}, 0}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := validateSegmentBlocks(size, dirOffset, test.blocks, test.rows); err == nil { + t.Fatal("validateSegmentBlocks accepted a corrupt block directory") + } + }) + } +} + +func TestOpenRejectsSegmentWithCorruptBlockEntry(t *testing.T) { + dir := t.TempDir() + base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := store.AppendID("seg-block", []Span{{Namespace: "default", TraceID: "t", SpanID: "1", ServiceName: "api", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}}); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "seg-block.fseg") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + dirOffset := binary.LittleEndian.Uint64(data[40:48]) + // Claim the first block runs far past the directory it precedes. + binary.LittleEndian.PutUint32(data[int(dirOffset)+8:], 1<<30) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + _, err = Open(dir) + if err == nil { + t.Fatal("Open accepted a segment whose block extends past its directory") + } + if !strings.Contains(err.Error(), "block extends past the block directory") { + t.Fatalf("Open error = %v, want the block-extent guard to reject it", err) + } +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 7878e031..945c9aa4 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -226,8 +226,8 @@ func (r *Repository) PruneParquet(cutoff int64) (int, error) { // Commit durably records a batch and publishes its three signal projections // exactly once. A crash at any point leaves the WAL for replay on next boot. func (r *Repository) Commit(batch Batch) error { - if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { - return errors.New("telemetry batch requires a safe ID") + if err := validateBatch(batch); err != nil { + return err } normalizeBatch(&batch) // Commits are serialized, but their segment and Parquet fsyncs do not hold @@ -258,8 +258,8 @@ func (r *Repository) Commit(batch Batch) error { // Writers call this before handing a batch to an asynchronous commit worker, so // every queued or in-flight batch is replayable if shutdown interrupts retries. func (r *Repository) Stage(batch Batch) error { - if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { - return errors.New("telemetry batch requires a safe ID") + if err := validateBatch(batch); err != nil { + return err } normalizeBatch(&batch) r.commitMu.Lock() @@ -271,6 +271,19 @@ func (r *Repository) Stage(batch Batch) error { return r.writeWAL(batch) } +// validateBatch rejects a batch no projection could ever publish. The segment +// stores name their files after the batch ID, so an ID they would refuse must +// be caught before the WAL promises to replay it forever. +func validateBatch(batch Batch) error { + if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { + return errors.New("telemetry batch requires a safe ID") + } + if !segment.ValidID(batch.ID) { + return fmt.Errorf("telemetry batch ID %q cannot name a segment", batch.ID) + } + return nil +} + func (r *Repository) apply(batch Batch) error { if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { return fmt.Errorf("commit span segment: %w", err) @@ -375,20 +388,23 @@ func (r *Repository) recover() error { } continue } - // A batch that cannot be applied would otherwise abort every boot. Move - // it aside so the instance starts; if quarantine itself fails the - // environment is broken, and failing loudly is then the right answer. - if err := r.apply(batch); err != nil { + // Poison is decided before anything is published: a payload the + // projections can never accept is moved aside so it cannot abort every + // boot. An apply or manifest failure after that point is environmental, + // so the WAL is retained and startup fails loudly — a later healthy boot + // must still be able to finish the batch, including one whose projection + // prefix this attempt already published. + if err := validateBatch(batch); err != nil { if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { return errors.Join(fmt.Errorf("replay %s: %w", name, err), quarantineErr) } continue } + if err := r.apply(batch); err != nil { + return fmt.Errorf("replay %s: %w", name, err) + } if err := r.recordBatch(batch); err != nil { - if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { - return errors.Join(fmt.Errorf("record replayed %s: %w", name, err), quarantineErr) - } - continue + return fmt.Errorf("record replayed %s: %w", name, err) } if err := os.Remove(filepath.Join(r.walDir, name)); err != nil { return err diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index bd1d858f..315a9978 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -557,3 +557,104 @@ func TestCompactionSourcesDropInheritedLedger(t *testing.T) { } } } + +func TestRecoverQuarantinesBatchThatCanNeverApply(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + // A batch ID the segment stores can never accept: it decodes cleanly, so + // only an apply attempt can reject it, and it will do so on every boot. + poison := testBatch() + poison.ID = "poison batch" + if err := repository.writeWAL(poison); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("Open error = %v, want the unappliable batch quarantined instead of a boot loop", err) + } + defer reopened.Close() + entries, err := os.ReadDir(filepath.Join(dir, "wal")) + if err != nil { + t.Fatal(err) + } + corrupt, live := 0, 0 + for _, entry := range entries { + switch { + case strings.HasSuffix(entry.Name(), ".corrupt"): + corrupt++ + case strings.HasSuffix(entry.Name(), ".wal"): + live++ + } + } + if corrupt != 1 || live != 0 { + t.Fatalf("quarantined = %d, live = %d, want the poison WAL renamed aside", corrupt, live) + } +} + +func TestRecoverRetainsWALWhenApplyFailsFromEnvironment(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root bypasses the directory permissions this test relies on") + } + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + if err := repository.Stage(batch); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + // Hot segments still accept the batch, so replay publishes a projection + // prefix and then fails on Parquet: an environmental failure that a later + // healthy boot must be able to finish. + parquetSpans := filepath.Join(dir, "parquet", "spans") + if err := os.Chmod(parquetSpans, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(parquetSpans, 0o755) }) + if _, err := Open(dir); err == nil { + t.Fatal("Open succeeded although replay could not publish the batch") + } + entries, err := os.ReadDir(filepath.Join(dir, "wal")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".corrupt") { + t.Fatalf("environmental failure quarantined %s; a healthy restart can no longer finish the batch", entry.Name()) + } + } + if err := os.Chmod(parquetSpans, 0o755); err != nil { + t.Fatal(err) + } + healthy, err := Open(dir) + if err != nil { + t.Fatalf("Open error = %v, want the retained WAL to replay once the environment recovered", err) + } + defer healthy.Close() + if got := healthy.Spans.RowCount(); got != uint64(len(batch.Spans)) { + t.Fatalf("replayed spans = %d, want %d", got, len(batch.Spans)) + } +} + +func TestStageRejectsBatchTheSegmentStoresCannotAccept(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + batch := testBatch() + batch.ID = "poison batch" + if err := repository.Stage(batch); err == nil { + t.Fatal("Stage accepted a batch ID no projection can ever publish") + } +} diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index 0be349ae..423cdaf0 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -298,37 +298,6 @@ func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { } } -func TestRecoverQuarantinesBatchThatCannotApply(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - batch := testBatch() - if err := repository.Stage(batch); err != nil { - t.Fatal(err) - } - // Make the hot span directory unusable so replay's apply always fails. - spanDir := filepath.Join(dir, "hot", "spans") - if err := os.RemoveAll(spanDir); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(spanDir, []byte("not a directory"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := Open(dir); err == nil { - t.Fatal("Open succeeded with an unusable hot span directory") - } - if err := os.Remove(spanDir); err != nil { - t.Fatal(err) - } - reopened, err := Open(dir) - if err != nil { - t.Fatalf("Open error = %v: a batch that cannot apply must be quarantined, not crash-loop every boot", err) - } - defer reopened.Close() -} - type transientStageCommitter struct { repository *Repository mu sync.Mutex From 1f05228b6f5f53f33f8c0dd6f6d5aa80590c2152 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 22:42:04 -0700 Subject: [PATCH 09/31] fix(storage): bound segment decoders and drop the rollup key limit Every zstd reader over on-disk telemetry now carries a product-sized memory ceiling, so a corrupt or crafted frame is refused instead of allocating against the library's 64 GiB default. Rollup key parts are varint-framed like every other string in the format, removing the 65535-byte publish failure that a valid batch could hit and then repeat on every restart, and span blocks bound their row count the way signal blocks already did. Both segment formats carry a new magic and version, so an existing hot tier is rejected rather than misread. Claude-Session: https://claude.ai/code/session_01Cxec3QnsbwcU1dqaFCtnTf --- internal/telemetry/segment/signal_store.go | 6 +-- internal/telemetry/segment/span_store.go | 45 ++++++++++------ internal/telemetry/segment/span_store_test.go | 53 +++++++++++++++++++ internal/telemetry/store/repository.go | 13 ++++- internal/telemetry/store/repository_test.go | 18 +++++++ 5 files changed, 116 insertions(+), 19 deletions(-) diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index 40f9ed63..eea27823 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -18,8 +18,8 @@ import ( ) const ( - signalMagic = "FANSIG03" - signalVersion = uint32(3) + signalMagic = "FANSIG04" + signalVersion = uint32(4) signalHeaderSize = 64 signalBlockSize = 32 signalBlockRows = 2048 @@ -95,7 +95,7 @@ func OpenSignalStore[T any](dir, timeField string) (*SignalStore[T], error) { } s := &SignalStore[T]{dir: dir, timeField: field.Index[0], codec: codec, encoder: enc, openFile: func(path string) (signalFile, error) { return os.Open(path) }} s.decoders.New = func() any { - dec, decErr := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + dec, decErr := newSegmentDecoder() if decErr != nil { panic(decErr) } diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index a91935ad..510bb03d 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -26,9 +26,21 @@ import ( "github.com/zeebo/xxh3" ) +// segmentDecoderMaxMemory bounds what one decompressed segment section may +// claim. A block holds at most rowsPerBlock rows and a section is decoded whole, +// so this is far above any sound file while refusing a corrupt or crafted frame +// long before zstd's 64 GiB default would. +const segmentDecoderMaxMemory = 512 << 20 + +// newSegmentDecoder builds a decoder bounded to segmentDecoderMaxMemory. Every +// segment read goes through it, so no on-disk frame can size an allocation. +func newSegmentDecoder() (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(segmentDecoderMaxMemory)) +} + const ( - segmentMagic = "FANSEG02" - segmentVersion = uint32(2) + segmentMagic = "FANSEG03" + segmentVersion = uint32(3) headerSize = 64 blockDirSize = 32 traceEntrySize = 16 @@ -122,7 +134,7 @@ func Open(dir string) (*Store, error) { } s := &Store{dir: dir, encoder: enc} s.decoders.New = func() any { - dec, decErr := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + dec, decErr := newSegmentDecoder() if decErr != nil { panic(decErr) } @@ -891,8 +903,8 @@ func validateSegmentBlocks(size, dirOffset uint64, blocks []blockDir, rows uint3 if block.length == 0 || uint64(block.length) > dirOffset-block.offset { return errors.New("block extends past the block directory") } - if block.rows == 0 { - return errors.New("block holds no rows") + if block.rows == 0 || block.rows > rowsPerBlock { + return errors.New("block row count is out of range") } counted += uint64(block.rows) } @@ -962,7 +974,7 @@ func openSegment(path string) (segment, error) { if _, err := f.ReadAt(indexCompressed, int64(indexOffset)); err != nil { return segment{}, err } - dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + dec, err := newSegmentDecoder() if err != nil { return segment{}, err } @@ -1033,13 +1045,13 @@ func writeRollup(w io.Writer, r rollup) error { if _, err := w.Write(fixed[:]); err != nil { return fmt.Errorf("write rollup: %w", err) } + // Key parts are varint-framed like every other string in the format: a + // pathological route must never make a batch unpublishable, because the WAL + // would then abort every restart with no way to make progress. for _, value := range []string{r.key.namespace, r.key.service, r.key.method, r.key.route} { - if len(value) > math.MaxUint16 { - return errors.New("rollup key exceeds 65535 bytes") - } - var length [2]byte - binary.LittleEndian.PutUint16(length[:], uint16(len(value))) - if _, err := w.Write(length[:]); err != nil { + var length [binary.MaxVarintLen64]byte + n := binary.PutUvarint(length[:], uint64(len(value))) + if _, err := w.Write(length[:n]); err != nil { return err } if _, err := io.WriteString(w, value); err != nil { @@ -1059,11 +1071,14 @@ func readRollup(r *bufio.Reader) (rollup, error) { out.bins[i] = binary.LittleEndian.Uint32(fixed[32+i*4:]) } for _, target := range []*string{&out.key.namespace, &out.key.service, &out.key.method, &out.key.route} { - var length [2]byte - if _, err := io.ReadFull(r, length[:]); err != nil { + length, err := binary.ReadUvarint(r) + if err != nil { return rollup{}, err } - value := make([]byte, binary.LittleEndian.Uint16(length[:])) + if length > segmentDecoderMaxMemory { + return rollup{}, errors.New("rollup key length is out of range") + } + value := make([]byte, length) if _, err := io.ReadFull(r, value); err != nil { return rollup{}, err } diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index d2a02053..fc2784c7 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -3,6 +3,7 @@ package segment import ( "encoding/binary" + "github.com/klauspost/compress/zstd" "os" "path/filepath" "strings" @@ -328,3 +329,55 @@ func TestOpenRejectsSegmentWithCorruptBlockEntry(t *testing.T) { t.Fatalf("Open error = %v, want the block-extent guard to reject it", err) } } + +func TestStoreAcceptsSpanWithOversizedRollupKey(t *testing.T) { + dir := t.TempDir() + base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() + // A pathological route must not be a deterministic publish failure: the WAL + // would then abort every restart with no way to make progress. + route := "/" + strings.Repeat("x", 70000) + rows := []Span{{Namespace: "default", TraceID: "t", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: route, StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}} + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := store.AppendID("seg-big-key", rows); err != nil { + t.Fatalf("AppendID error = %v, want an oversized rollup key to be publishable", err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("Open error = %v, want the segment to round-trip", err) + } + defer reopened.Close() + endpoints := reopened.Endpoints("default", "api", base, base+int64(time.Minute), 10) + if len(endpoints) != 1 || endpoints[0].Route != route { + t.Fatalf("endpoints = %d entries, want the full route preserved", len(endpoints)) + } +} + +func TestValidateSegmentBlocksRejectsRowsPastBlockCap(t *testing.T) { + blocks := []blockDir{{offset: headerSize, length: 16, rows: rowsPerBlock + 1}} + if err := validateSegmentBlocks(4096, 2048, blocks, rowsPerBlock+1); err == nil { + t.Fatal("validateSegmentBlocks accepted a block claiming more rows than a block can hold") + } +} + +func TestSegmentDecoderRejectsOversizedFrame(t *testing.T) { + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + if err != nil { + t.Fatal(err) + } + defer encoder.Close() + frame := encoder.EncodeAll(make([]byte, segmentDecoderMaxMemory+(1<<20)), nil) + decoder, err := newSegmentDecoder() + if err != nil { + t.Fatal(err) + } + defer decoder.Close() + if _, err := decoder.DecodeAll(frame, nil); err == nil { + t.Fatal("segment decoder accepted a frame declaring more memory than the product ever needs") + } +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 945c9aa4..b4de6548 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -346,6 +346,17 @@ func (r *Repository) writeWAL(batch Batch) error { return syncDirectory(r.walDir) } +// walDecoderMaxMemory bounds one decompressed WAL batch. Flush batches are +// bounded by the writer's batch size, so this sits far above any batch the +// writer produces while refusing a corrupt or crafted frame long before zstd's +// 64 GiB default would. +const walDecoderMaxMemory = 1 << 30 + +// newWALDecoder builds the bounded decoder every WAL read goes through. +func newWALDecoder() (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(walDecoderMaxMemory)) +} + func (r *Repository) recover() error { entries, err := os.ReadDir(r.walDir) if err != nil { @@ -358,7 +369,7 @@ func (r *Repository) recover() error { } } sort.Strings(names) - dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) + dec, err := newWALDecoder() if err != nil { return err } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 315a9978..a3ba1c9e 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "github.com/klauspost/compress/zstd" "os" "path/filepath" "slices" @@ -658,3 +659,20 @@ func TestStageRejectsBatchTheSegmentStoresCannotAccept(t *testing.T) { t.Fatal("Stage accepted a batch ID no projection can ever publish") } } + +func TestWALDecoderRejectsOversizedFrame(t *testing.T) { + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) + if err != nil { + t.Fatal(err) + } + defer encoder.Close() + frame := encoder.EncodeAll(make([]byte, walDecoderMaxMemory+(1<<20)), nil) + decoder, err := newWALDecoder() + if err != nil { + t.Fatal(err) + } + defer decoder.Close() + if _, err := decoder.DecodeAll(frame, nil); err == nil { + t.Fatal("WAL decoder accepted a frame declaring more memory than any batch needs") + } +} From e6b4b586c758e62fc4b7677f68f6aff0d8fd8a09 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 26 Aug 2026 23:14:25 -0700 Subject: [PATCH 10/31] fix(storage): enforce durable ingest guarantees Stage request batches before OTLP success and stop on permanent commit failures. Compact committed hot files across all signals and rebuild corrupt acceleration data from authoritative Parquet. Align write and read memory budgets and validate derived rollups before WAL staging. --- cmd/fanout/main.go | 10 +- internal/ingest/http_test.go | 8 +- internal/ingest/server.go | 70 +++----- internal/ingest/server_test.go | 2 +- internal/ingest/submitter_test.go | 43 +++++ internal/query/duck.go | 3 +- internal/telemetry/segment/signal_store.go | 141 ++++++++++++++- internal/telemetry/segment/span_store.go | 120 ++++++++++++- internal/telemetry/segment/span_store_test.go | 26 ++- internal/telemetry/store/repository.go | 164 +++++++++++++++--- internal/telemetry/store/repository_test.go | 83 ++++++++- internal/telemetry/store/writer.go | 112 +++++++++++- internal/telemetry/store/writer_test.go | 88 +++++++++- 13 files changed, 764 insertions(+), 106 deletions(-) create mode 100644 internal/ingest/submitter_test.go diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 5623446c..9aaa921e 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -41,7 +41,6 @@ import ( "github.com/labstack/fanout/internal/query" "github.com/labstack/fanout/internal/settings" "github.com/labstack/fanout/internal/store" - "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/labstack/fanout/internal/ui" ) @@ -96,11 +95,6 @@ func main() { os.Exit(1) } - // Channels for OTLP decoding → the single authoritative telemetry writer. - chSpans := make(chan telemetry.Span, 10000) - chLogs := make(chan telemetry.Log, 10000) - chMetrics := make(chan telemetry.Metric, 10000) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -126,7 +120,7 @@ func main() { } defer q.Close() - writer := telemetrystore.NewWriter(repository, cfg.FlushInterval, cfg.FlushBatchSize, chSpans, chLogs, chMetrics) + writer := telemetrystore.NewWriter(repository, cfg.FlushInterval, cfg.FlushBatchSize, nil, nil, nil) writerResult := make(chan error, 1) go func() { err := writer.Run(ctx) @@ -200,7 +194,7 @@ func main() { os.Exit(1) } grpcSrv := grpc.NewServer(grpcOpts...) - ing := ingest.NewServer(cfg, chSpans, chLogs, chMetrics) + ing := ingest.NewServer(cfg, writer) ingest.RegisterOTLP(grpcSrv, ing) otlpHTTPLis, err := net.Listen("tcp", cfg.OTLPHTTPAddr) if err != nil { diff --git a/internal/ingest/http_test.go b/internal/ingest/http_test.go index 3f3ebb64..dd24d440 100644 --- a/internal/ingest/http_test.go +++ b/internal/ingest/http_test.go @@ -53,7 +53,7 @@ func newHTTPIngestFixture(t *testing.T, configured bool) *httpIngestFixture { spans := make(chan telemetry.Span, 8) logs := make(chan telemetry.Log, 8) metrics := make(chan telemetry.Metric, 8) - srv := NewServer(config.Config{DefaultNamespace: "default"}, spans, logs, metrics) + srv := NewServer(config.Config{DefaultNamespace: "default"}, newTestSubmitter(spans, logs, metrics)) return &httpIngestFixture{ handler: NewHTTPHandler(srv, store), token: token, @@ -268,7 +268,7 @@ func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("traces", func(t *testing.T) { request := testTraceRequest() grpcRows := make(chan telemetry.Span, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, grpcRows, make(chan telemetry.Log, 1), make(chan telemetry.Metric, 1)) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, newTestSubmitter(grpcRows, make(chan telemetry.Log, 1), make(chan telemetry.Metric, 1))) if _, err := grpcSrv.exportTraces(context.Background(), request); err != nil { t.Fatal(err) } @@ -285,7 +285,7 @@ func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("logs", func(t *testing.T) { request := testLogsRequest() grpcRows := make(chan telemetry.Log, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan telemetry.Span, 1), grpcRows, make(chan telemetry.Metric, 1)) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, newTestSubmitter(make(chan telemetry.Span, 1), grpcRows, make(chan telemetry.Metric, 1))) if _, err := grpcSrv.exportLogs(context.Background(), request); err != nil { t.Fatal(err) } @@ -302,7 +302,7 @@ func TestHTTPAndGRPCPathsProduceEquivalentRows(t *testing.T) { t.Run("metrics", func(t *testing.T) { request := testMetricsRequest() grpcRows := make(chan telemetry.Metric, 1) - grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, make(chan telemetry.Span, 1), make(chan telemetry.Log, 1), grpcRows) + grpcSrv := NewServer(config.Config{DefaultNamespace: "default"}, newTestSubmitter(make(chan telemetry.Span, 1), make(chan telemetry.Log, 1), grpcRows)) if _, err := grpcSrv.exportMetrics(context.Background(), request); err != nil { t.Fatal(err) } diff --git a/internal/ingest/server.go b/internal/ingest/server.go index 03933ef6..3dbdf4b8 100644 --- a/internal/ingest/server.go +++ b/internal/ingest/server.go @@ -25,13 +25,16 @@ import ( "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) +type batchSubmitter interface { + Submit(context.Context, telemetrystore.Batch) error +} + type Server struct { - cfg config.Config - outSpans chan<- telemetry.Span - outLogs chan<- telemetry.Log - outMetrics chan<- telemetry.Metric + cfg config.Config + submitter batchSubmitter } type traceService struct { @@ -49,8 +52,8 @@ type metricsService struct { srv *Server } -func NewServer(cfg config.Config, spans chan<- telemetry.Span, logs chan<- telemetry.Log, metrics chan<- telemetry.Metric) *Server { - return &Server{cfg: cfg, outSpans: spans, outLogs: logs, outMetrics: metrics} +func NewServer(cfg config.Config, submitter batchSubmitter) *Server { + return &Server{cfg: cfg, submitter: submitter} } func RegisterOTLP(s grpc.ServiceRegistrar, srv *Server) { @@ -68,6 +71,7 @@ func (ts *traceService) Export(ctx context.Context, req *collectortrace.ExportTr func (s *Server) exportTraces(ctx context.Context, req *collectortrace.ExportTraceServiceRequest) (*collectortrace.ExportTraceServiceResponse, error) { cfg := s.cfg now := time.Now().UnixNano() + batch := telemetrystore.Batch{} for _, rs := range req.ResourceSpans { resourceJSON := resourceAttrsJSON(rs.Resource) svc := getServiceName(rs.Resource) @@ -115,15 +119,13 @@ func (s *Server) exportTraces(ctx context.Context, req *collectortrace.ExportTra excType, excMsg := extractException(sp.Events) row.ExceptionType = excType row.ExceptionMessage = excMsg - // Partial ingest is acceptable: OTLP clients retry the full batch on error. - select { - case s.outSpans <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Spans = append(batch.Spans, row) } } } + if err := s.submitter.Submit(ctx, batch); err != nil { + return nil, err + } return &collectortrace.ExportTraceServiceResponse{}, nil } @@ -136,6 +138,7 @@ func (ls *logsService) Export(ctx context.Context, req *collectorlogs.ExportLogs func (s *Server) exportLogs(ctx context.Context, req *collectorlogs.ExportLogsServiceRequest) (*collectorlogs.ExportLogsServiceResponse, error) { cfg := s.cfg now := time.Now().UnixNano() + batch := telemetrystore.Batch{} for _, rl := range req.ResourceLogs { resourceJSON := resourceAttrsJSON(rl.Resource) svc := getServiceName(rl.Resource) @@ -166,14 +169,13 @@ func (s *Server) exportLogs(ctx context.Context, req *collectorlogs.ExportLogsSe ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outLogs <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Logs = append(batch.Logs, row) } } } + if err := s.submitter.Submit(ctx, batch); err != nil { + return nil, err + } return &collectorlogs.ExportLogsServiceResponse{}, nil } @@ -186,6 +188,7 @@ func (ms *metricsService) Export(ctx context.Context, req *collectormetrics.Expo func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.ExportMetricsServiceRequest) (*collectormetrics.ExportMetricsServiceResponse, error) { cfg := s.cfg now := time.Now().UnixNano() + batch := telemetrystore.Batch{} for _, rm := range req.ResourceMetrics { resourceJSON := resourceAttrsJSON(rm.Resource) svc := getServiceName(rm.Resource) @@ -215,11 +218,7 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outMetrics <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Metrics = append(batch.Metrics, row) } case *metricspb.Metric_Sum: kind := "sum" @@ -243,11 +242,7 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outMetrics <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Metrics = append(batch.Metrics, row) } case *metricspb.Metric_Histogram: for _, dp := range d.Histogram.DataPoints { @@ -274,11 +269,7 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outMetrics <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Metrics = append(batch.Metrics, row) } case *metricspb.Metric_ExponentialHistogram: for _, dp := range d.ExponentialHistogram.DataPoints { @@ -305,11 +296,7 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outMetrics <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Metrics = append(batch.Metrics, row) } case *metricspb.Metric_Summary: for _, dp := range d.Summary.DataPoints { @@ -331,16 +318,15 @@ func (s *Server) exportMetrics(ctx context.Context, req *collectormetrics.Export ScopeVersion: scopeVer, IngestedAt: now, } - select { - case s.outMetrics <- row: - case <-ctx.Done(): - return nil, ctx.Err() - } + batch.Metrics = append(batch.Metrics, row) } } } } } + if err := s.submitter.Submit(ctx, batch); err != nil { + return nil, err + } return &collectormetrics.ExportMetricsServiceResponse{}, nil } diff --git a/internal/ingest/server_test.go b/internal/ingest/server_test.go index a8389942..2894cd2f 100644 --- a/internal/ingest/server_test.go +++ b/internal/ingest/server_test.go @@ -511,7 +511,7 @@ func TestTraceExportContextCancellation(t *testing.T) { logs := make(chan telemetry.Log, 1) metrics := make(chan telemetry.Metric, 1) - srv := NewServer(config.Config{}, spans, logs, metrics) + srv := NewServer(config.Config{}, newTestSubmitter(spans, logs, metrics)) ts := &traceService{srv: srv} // Cancel the context before calling Export diff --git a/internal/ingest/submitter_test.go b/internal/ingest/submitter_test.go new file mode 100644 index 00000000..6e6eaeb2 --- /dev/null +++ b/internal/ingest/submitter_test.go @@ -0,0 +1,43 @@ +package ingest + +import ( + "context" + + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" +) + +type channelSubmitter struct { + spans chan<- telemetry.Span + logs chan<- telemetry.Log + metrics chan<- telemetry.Metric +} + +func newTestSubmitter(spans chan<- telemetry.Span, logs chan<- telemetry.Log, metrics chan<- telemetry.Metric) *channelSubmitter { + return &channelSubmitter{spans: spans, logs: logs, metrics: metrics} +} + +func (s *channelSubmitter) Submit(ctx context.Context, batch telemetrystore.Batch) error { + for _, row := range batch.Spans { + select { + case s.spans <- row: + case <-ctx.Done(): + return ctx.Err() + } + } + for _, row := range batch.Logs { + select { + case s.logs <- row: + case <-ctx.Done(): + return ctx.Err() + } + } + for _, row := range batch.Metrics { + select { + case s.metrics <- row: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} diff --git a/internal/query/duck.go b/internal/query/duck.go index fa6b8cf0..ad65ee9f 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -436,6 +436,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { var pruneErr error if d.repository != nil { _, pruneErr = d.repository.PruneHot(cutoff) + _, hotCompactErr := d.repository.CompactHot(64) var parquetErr error if d.cfg.RetentionDays > 0 { d.parquetMu.Lock() @@ -451,7 +452,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { compactResult = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) - pruneErr = errors.Join(pruneErr, parquetErr, compactErr) + pruneErr = errors.Join(pruneErr, hotCompactErr, parquetErr, compactErr) } var cacheErr error unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go index eea27823..327907a6 100644 --- a/internal/telemetry/segment/signal_store.go +++ b/internal/telemetry/segment/signal_store.go @@ -13,6 +13,7 @@ import ( "reflect" "regexp" "sync" + "time" "github.com/klauspost/compress/zstd" ) @@ -23,6 +24,7 @@ const ( signalHeaderSize = 64 signalBlockSize = 32 signalBlockRows = 2048 + signalMaxBlocks = 1 << 20 ) var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) @@ -371,6 +373,140 @@ func (s *SignalStore[T]) SegmentCount() int { return len(s.segments) } +// CompactCommitted combines raw ingest segments known to be fully committed. +// Compressed column blocks are copied verbatim, so compaction is independent of +// row width and does not inflate the process heap. +func (s *SignalStore[T]) CompactCommitted(committed map[string]struct{}, maxInputs int) (int, error) { + if maxInputs < 2 { + return 0, nil + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.mu.RLock() + selected := make([]signalSegment, 0, maxInputs) + rest := make([]signalSegment, 0, len(s.segments)) + for _, seg := range s.segments { + if len(selected) < maxInputs { + if _, ok := committed[seg.id]; ok { + selected = append(selected, seg) + continue + } + } + rest = append(rest, seg) + } + s.mu.RUnlock() + if len(selected) < 2 { + return 0, nil + } + id := fmt.Sprintf("compact-%d", time.Now().UnixNano()) + name := id + ".fseg" + tmp, final := filepath.Join(s.dir, name+".tmp"), filepath.Join(s.dir, name) + replacement, err := s.writeCompactedSegment(tmp, id, selected) + if err != nil { + return 0, err + } + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return 0, fmt.Errorf("publish compacted signal segment: %w", err) + } + if err := syncDir(s.dir); err != nil { + return 0, err + } + next := signalManifest{Version: signalVersion, Files: make([]string, 0, len(rest)+1)} + next.Files = append(next.Files, name) + for _, seg := range rest { + next.Files = append(next.Files, filepath.Base(seg.path)) + } + if err := writeSignalManifest(s.dir, next); err != nil { + return 0, err + } + replacement.path = final + s.mu.Lock() + s.manifest = next + s.segments = append([]signalSegment{replacement}, rest...) + s.mu.Unlock() + var removeErr error + for _, seg := range selected { + if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { + removeErr = errors.Join(removeErr, err) + } + } + return len(selected), errors.Join(removeErr, syncDir(s.dir)) +} + +func (s *SignalStore[T]) writeCompactedSegment(path, id string, inputs []signalSegment) (signalSegment, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return signalSegment{}, fmt.Errorf("create compacted signal segment: %w", err) + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(make([]byte, signalHeaderSize)); err != nil { + return signalSegment{}, err + } + replacement := signalSegment{id: id, min: math.MaxInt64, max: math.MinInt64, fieldCount: uint32(len(s.codec.fields)), fingerprint: s.codec.fingerprint} + offset := uint64(signalHeaderSize) + for _, input := range inputs { + source, err := os.Open(input.path) + if err != nil { + return signalSegment{}, err + } + for _, block := range input.blocks { + if _, err := io.CopyN(f, io.NewSectionReader(source, int64(block.offset), int64(block.length)), int64(block.length)); err != nil { + _ = source.Close() + return signalSegment{}, fmt.Errorf("copy compressed signal block: %w", err) + } + replacement.blocks = append(replacement.blocks, signalBlock{offset: offset, length: block.length, rows: block.rows, min: block.min, max: block.max}) + offset += uint64(block.length) + } + if err := source.Close(); err != nil { + return signalSegment{}, err + } + replacement.rows += input.rows + replacement.min = min(replacement.min, input.min) + replacement.max = max(replacement.max, input.max) + } + dirOffset := offset + directory := make([]byte, len(replacement.blocks)*signalBlockSize) + for i, block := range replacement.blocks { + entry := directory[i*signalBlockSize:] + binary.LittleEndian.PutUint64(entry[0:8], block.offset) + binary.LittleEndian.PutUint32(entry[8:12], block.length) + binary.LittleEndian.PutUint32(entry[12:16], block.rows) + binary.LittleEndian.PutUint64(entry[16:24], uint64(block.min)) + binary.LittleEndian.PutUint64(entry[24:32], uint64(block.max)) + } + if _, err := f.Write(directory); err != nil { + return signalSegment{}, err + } + var header [signalHeaderSize]byte + copy(header[0:8], signalMagic) + binary.LittleEndian.PutUint32(header[8:12], signalVersion) + binary.LittleEndian.PutUint32(header[12:16], replacement.rows) + binary.LittleEndian.PutUint32(header[16:20], uint32(len(replacement.blocks))) + binary.LittleEndian.PutUint32(header[20:24], replacement.fieldCount) + binary.LittleEndian.PutUint64(header[24:32], uint64(replacement.min)) + binary.LittleEndian.PutUint64(header[32:40], uint64(replacement.max)) + binary.LittleEndian.PutUint64(header[40:48], dirOffset) + binary.LittleEndian.PutUint64(header[48:56], replacement.fingerprint) + if _, err := f.WriteAt(header[:], 0); err != nil { + return signalSegment{}, err + } + if err := f.Sync(); err != nil { + return signalSegment{}, err + } + if err := f.Close(); err != nil { + return signalSegment{}, err + } + ok = true + return replacement, nil +} + func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { s.writeMu.Lock() defer s.writeMu.Unlock() @@ -420,7 +556,7 @@ func validateSignalBlocks(size, dirOffset uint64, blocks []signalBlock, rows uin if block.offset < signalHeaderSize || block.offset > dirOffset { return errors.New("block offset outside the segment payload") } - if block.length == 0 || uint64(block.length) > dirOffset-block.offset { + if block.length == 0 || uint64(block.length) > dirOffset-block.offset || uint64(block.length) > segmentMaxCompressedBytes { return errors.New("block extends past the block directory") } if block.rows == 0 || block.rows > signalBlockRows { @@ -440,6 +576,9 @@ func validateSignalBlocks(size, dirOffset uint64, blocks []signalBlock, rows uin // validateSignalDirectory bounds the block directory against the file size, // so a torn header cannot size an allocation the file could never hold. func validateSignalDirectory(size, dirOffset uint64, blockCount uint32) error { + if blockCount > signalMaxBlocks { + return errors.New("signal block count is out of range") + } if dirOffset < signalHeaderSize || dirOffset > size { return errors.New("corrupt directory offset") } diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index 510bb03d..e7eb0c55 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -30,12 +30,21 @@ import ( // claim. A block holds at most rowsPerBlock rows and a section is decoded whole, // so this is far above any sound file while refusing a corrupt or crafted frame // long before zstd's 64 GiB default would. -const segmentDecoderMaxMemory = 512 << 20 +const ( + segmentDecoderMaxMemory = 128 << 20 + maxRollupKeyBytes = 32 << 20 + segmentMaxCompressedBytes = segmentDecoderMaxMemory + (1 << 20) + segmentMaxBlocks = 1 << 20 +) // newSegmentDecoder builds a decoder bounded to segmentDecoderMaxMemory. Every // segment read goes through it, so no on-disk frame can size an allocation. func newSegmentDecoder() (*zstd.Decoder, error) { - return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(segmentDecoderMaxMemory)) + return newSegmentDecoderWithLimit(segmentDecoderMaxMemory) +} + +func newSegmentDecoderWithLimit(limit uint64) (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(limit)) } const ( @@ -51,6 +60,42 @@ const ( type Span = telemetry.Span +// ValidateSpanRows rejects rows whose derived rollup representation could not +// be reopened within the production decoder budget. It must run before WAL +// staging so deterministic format errors never become boot-blocking WALs. +func ValidateSpanRows(rows []Span) error { + return validateSpanRowsWithLimits(rows, maxRollupKeyBytes, segmentDecoderMaxMemory) +} + +func validateSpanRowsWithLimits(rows []Span, maxKeyBytes, maxSectionBytes uint64) error { + keys := make(map[rollupKey]struct{}, len(rows)) + var sectionBytes uint64 + for _, row := range rows { + key := rollupKey{ + bucket: row.StartUnixNanos - row.StartUnixNanos%int64(rollupWindow), + namespace: row.Namespace, service: row.ServiceName, method: row.HTTPMethod, route: row.HTTPRoute, + } + if _, exists := keys[key]; exists { + continue + } + keys[key] = struct{}{} + keyBytes := uint64(len(key.namespace)) + uint64(len(key.service)) + uint64(len(key.method)) + uint64(len(key.route)) + if keyBytes > maxKeyBytes { + return fmt.Errorf("span rollup key uses %d bytes; maximum is %d", keyBytes, maxKeyBytes) + } + size := uint64(160) + var scratch [binary.MaxVarintLen64]byte + for _, value := range []string{key.namespace, key.service, key.method, key.route} { + size += uint64(binary.PutUvarint(scratch[:], uint64(len(value)))) + uint64(len(value)) + } + if size > maxSectionBytes-sectionBytes { + return fmt.Errorf("span rollups exceed %d-byte decoder budget", maxSectionBytes) + } + sectionBytes += size + } + return nil +} + type Endpoint struct { Service string Method string @@ -306,8 +351,46 @@ func (s *Store) CompactOldest(count int) error { old := append([]segment(nil), s.segments[:count]...) rest := append([]segment(nil), s.segments[count:]...) current := s.manifest - id := current.NextID s.mu.RUnlock() + return s.compactSegments(old, rest, current) +} + +// CompactCommitted compacts only raw ingest segments whose batch IDs are in +// the authoritative repository manifest. A partially applied WAL is therefore +// never folded into a replacement that could defeat replay idempotence. +func (s *Store) CompactCommitted(committed map[string]struct{}, maxInputs int) (int, error) { + if maxInputs < 2 { + return 0, nil + } + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.mu.RLock() + current := s.manifest + selected := make([]segment, 0, maxInputs) + rest := make([]segment, 0, len(s.segments)) + for _, seg := range s.segments { + id := strings.TrimSuffix(filepath.Base(seg.path), ".fseg") + if len(selected) < maxInputs { + if _, ok := committed[id]; ok { + selected = append(selected, seg) + continue + } + } + rest = append(rest, seg) + } + s.mu.RUnlock() + if len(selected) < 2 { + return 0, nil + } + if err := s.compactSegments(selected, rest, current); err != nil { + return 0, err + } + return len(selected), nil +} + +// compactSegments publishes a replacement while writeMu is held. +func (s *Store) compactSegments(old, rest []segment, current manifest) error { + id := current.NextID name := fmt.Sprintf("%020d.fseg", id) tmp, final := filepath.Join(s.dir, name+".tmp"), filepath.Join(s.dir, name) @@ -391,6 +474,9 @@ func (s *Store) PruneBefore(cutoff int64) (int, error) { } func (s *Store) writeSegment(path string, rows []Span) (segment, error) { + if err := ValidateSpanRows(rows); err != nil { + return segment{}, err + } f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) if err != nil { return segment{}, fmt.Errorf("create segment: %w", err) @@ -495,6 +581,9 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { } indexOffset, _ := f.Seek(0, io.SeekCurrent) indexPlain := make([]byte, len(index)*traceEntrySize) + if len(indexPlain) > segmentDecoderMaxMemory { + return segment{}, errors.New("trace index exceeds decoder memory limit") + } for i, entry := range index { buf := indexPlain[i*traceEntrySize:] binary.LittleEndian.PutUint64(buf[0:8], entry.hash) @@ -511,6 +600,9 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { return segment{}, err } } + if rollupPlain.Len() > segmentDecoderMaxMemory { + return segment{}, errors.New("rollup section exceeds decoder memory limit") + } if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { return segment{}, fmt.Errorf("write rollups: %w", err) } @@ -624,6 +716,9 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e } indexOffset, _ := f.Seek(0, io.SeekCurrent) indexPlain := make([]byte, len(replacement.traceIndex)*traceEntrySize) + if len(indexPlain) > segmentDecoderMaxMemory { + return segment{}, errors.New("compacted trace index exceeds decoder memory limit") + } for i, entry := range replacement.traceIndex { buf := indexPlain[i*traceEntrySize:] binary.LittleEndian.PutUint64(buf[0:8], entry.hash) @@ -640,6 +735,9 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e return segment{}, err } } + if rollupPlain.Len() > segmentDecoderMaxMemory { + return segment{}, errors.New("compacted rollup section exceeds decoder memory limit") + } if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { return segment{}, err } @@ -879,6 +977,9 @@ func isErrorStatus(status string) bool { // subtractions against size so a corrupt offset near the top of the address // space cannot wrap past the check. func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, blockCount uint32) error { + if blockCount > segmentMaxBlocks { + return errors.New("segment block count is out of range") + } if err := validateDirectory(size, dirOffset, blockCount, blockDirSize); err != nil { return err } @@ -888,6 +989,9 @@ func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, if rollupOffset < indexOffset || rollupOffset > size { return errors.New("corrupt rollup offset") } + if rollupOffset-indexOffset > segmentMaxCompressedBytes || size-rollupOffset > segmentMaxCompressedBytes { + return errors.New("compressed segment section exceeds memory limit") + } return nil } @@ -900,7 +1004,7 @@ func validateSegmentBlocks(size, dirOffset uint64, blocks []blockDir, rows uint3 if block.offset < headerSize || block.offset > dirOffset { return errors.New("block offset outside the segment payload") } - if block.length == 0 || uint64(block.length) > dirOffset-block.offset { + if block.length == 0 || uint64(block.length) > dirOffset-block.offset || uint64(block.length) > segmentMaxCompressedBytes { return errors.New("block extends past the block directory") } if block.rows == 0 || block.rows > rowsPerBlock { @@ -1062,6 +1166,10 @@ func writeRollup(w io.Writer, r rollup) error { } func readRollup(r *bufio.Reader) (rollup, error) { + return readRollupWithLimit(r, maxRollupKeyBytes) +} + +func readRollupWithLimit(r *bufio.Reader, maxKeyBytes uint64) (rollup, error) { var fixed [160]byte if _, err := io.ReadFull(r, fixed[:]); err != nil { return rollup{}, err @@ -1070,14 +1178,16 @@ func readRollup(r *bufio.Reader) (rollup, error) { for i := range out.bins { out.bins[i] = binary.LittleEndian.Uint32(fixed[32+i*4:]) } + remaining := maxKeyBytes for _, target := range []*string{&out.key.namespace, &out.key.service, &out.key.method, &out.key.route} { length, err := binary.ReadUvarint(r) if err != nil { return rollup{}, err } - if length > segmentDecoderMaxMemory { + if length > remaining { return rollup{}, errors.New("rollup key length is out of range") } + remaining -= length value := make([]byte, length) if _, err := io.ReadFull(r, value); err != nil { return rollup{}, err diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index fc2784c7..2d65932a 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -2,6 +2,8 @@ package segment import ( + "bufio" + "bytes" "encoding/binary" "github.com/klauspost/compress/zstd" "os" @@ -366,13 +368,14 @@ func TestValidateSegmentBlocksRejectsRowsPastBlockCap(t *testing.T) { } func TestSegmentDecoderRejectsOversizedFrame(t *testing.T) { + const testLimit = 64 << 10 encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) if err != nil { t.Fatal(err) } defer encoder.Close() - frame := encoder.EncodeAll(make([]byte, segmentDecoderMaxMemory+(1<<20)), nil) - decoder, err := newSegmentDecoder() + frame := encoder.EncodeAll(make([]byte, testLimit+(1<<10)), nil) + decoder, err := newSegmentDecoderWithLimit(testLimit) if err != nil { t.Fatal(err) } @@ -381,3 +384,22 @@ func TestSegmentDecoderRejectsOversizedFrame(t *testing.T) { t.Fatal("segment decoder accepted a frame declaring more memory than the product ever needs") } } + +func TestReadRollupRejectsLengthBeforeAllocating(t *testing.T) { + payload := make([]byte, 160, 170) + payload = binary.AppendUvarint(payload, 9) + if _, err := readRollupWithLimit(bufio.NewReader(bytes.NewReader(payload)), 8); err == nil { + t.Fatal("readRollup accepted a disk-controlled allocation above its budget") + } +} + +func TestValidateSpanRowsRejectsUnreopenableRollups(t *testing.T) { + rows := []Span{{HTTPRoute: "123456789"}} + if err := validateSpanRowsWithLimits(rows, 8, 1024); err == nil { + t.Fatal("validator accepted a rollup key larger than the reader budget") + } + rows = []Span{{HTTPRoute: "a"}, {HTTPRoute: "b"}} + if err := validateSpanRowsWithLimits(rows, 1024, 200); err == nil { + t.Fatal("validator accepted a rollup section larger than the decoder budget") + } +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index b4de6548..ebd15d3d 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -30,6 +30,11 @@ type Batch struct { Metrics []telemetry.Metric } +const ( + maxBatchRows = 50_000 + walDecoderMaxMemory = 128 << 20 +) + type batchMetadata struct { ID string `json:"id"` MinNanos int64 `json:"min_nanos"` @@ -79,20 +84,22 @@ func Open(root string) (*Repository, error) { return nil, err } } - spans, err := segment.Open(filepath.Join(root, "hot", "spans")) - if err != nil { - return nil, err - } - logs, err := segment.OpenSignalStore[telemetry.Log](filepath.Join(root, "hot", "logs"), "EventUnixNanos") + spans, logs, metricsStore, err := openHotStores(root) + hotRebuilt := false if err != nil { - _ = spans.Close() - return nil, err - } - metricsStore, err := segment.OpenSignalStore[telemetry.Metric](filepath.Join(root, "hot", "metrics"), "EventUnixNanos") - if err != nil { - _ = logs.Close() - _ = spans.Close() - return nil, err + quarantine := filepath.Join(root, fmt.Sprintf("hot.corrupt-%d", time.Now().UnixNano())) + if renameErr := os.Rename(filepath.Join(root, "hot"), quarantine); renameErr != nil { + return nil, errors.Join(fmt.Errorf("open hot telemetry tier: %w", err), fmt.Errorf("quarantine corrupt hot tier: %w", renameErr)) + } + if syncErr := syncDirectory(root); syncErr != nil { + return nil, fmt.Errorf("sync quarantined hot tier: %w", syncErr) + } + spans, logs, metricsStore, err = openHotStores(root) + if err != nil { + return nil, fmt.Errorf("rebuild hot telemetry tier: %w", err) + } + hotRebuilt = true + slog.Warn("corrupt hot telemetry tier quarantined and rebuilt from authoritative Parquet", "path", quarantine) } parquet, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { @@ -106,6 +113,13 @@ func Open(root string) (*Repository, error) { _ = r.Close() return nil, fmt.Errorf("load telemetry manifest: %w", err) } + if hotRebuilt { + r.manifest.HotCutoffNanos = max(r.manifest.HotCutoffNanos, time.Now().UnixNano()) + if err := writeRepositoryManifest(r.root, r.manifest); err != nil { + _ = r.Close() + return nil, fmt.Errorf("publish rebuilt hot-tier cutoff: %w", err) + } + } if err := r.recoverCompaction(); err != nil { _ = r.Close() return nil, fmt.Errorf("recover parquet compaction: %w", err) @@ -117,6 +131,31 @@ func Open(root string) (*Repository, error) { return r, nil } +func openHotStores(root string) (*segment.Store, *segment.SignalStore[telemetry.Log], *segment.SignalStore[telemetry.Metric], error) { + hot := filepath.Join(root, "hot") + for _, signal := range []string{"spans", "logs", "metrics"} { + if err := os.MkdirAll(filepath.Join(hot, signal), 0o755); err != nil { + return nil, nil, nil, err + } + } + spans, err := segment.Open(filepath.Join(hot, "spans")) + if err != nil { + return nil, nil, nil, err + } + logs, err := segment.OpenSignalStore[telemetry.Log](filepath.Join(hot, "logs"), "EventUnixNanos") + if err != nil { + _ = spans.Close() + return nil, nil, nil, err + } + metricsStore, err := segment.OpenSignalStore[telemetry.Metric](filepath.Join(hot, "metrics"), "EventUnixNanos") + if err != nil { + _ = logs.Close() + _ = spans.Close() + return nil, nil, nil, err + } + return spans, logs, metricsStore, nil +} + func (r *Repository) Close() error { return errors.Join(r.Spans.Close(), r.Logs.Close(), r.Metrics.Close()) } @@ -155,6 +194,56 @@ func (r *Repository) PruneHot(cutoff int64) (int, error) { return spans + logs + metricRows, errors.Join(spanErr, logErr, metricErr) } +// CompactHot drains committed raw segments into larger immutable files for all +// three signals. It intentionally excludes any segment not present in the +// repository manifest, because that file may belong to a partially applied WAL +// transaction that still needs exact-ID replay. +func (r *Repository) CompactHot(maxInputs int) (int, error) { + if maxInputs < 2 { + return 0, nil + } + r.hotMu.Lock() + defer r.hotMu.Unlock() + r.commitMu.Lock() + defer r.commitMu.Unlock() + r.mu.RLock() + committed := make(map[string]struct{}, len(r.manifest.Batches)) + for _, batch := range r.manifest.Batches { + committed[batch.ID] = struct{}{} + for _, source := range batch.Sources { + committed[source] = struct{}{} + } + } + r.mu.RUnlock() + total := 0 + var compactErr error + for { + n, err := r.Spans.CompactCommitted(committed, maxInputs) + total += n + compactErr = errors.Join(compactErr, err) + if err != nil || n < 2 { + break + } + } + for { + n, err := r.Logs.CompactCommitted(committed, maxInputs) + total += n + compactErr = errors.Join(compactErr, err) + if err != nil || n < 2 { + break + } + } + for { + n, err := r.Metrics.CompactCommitted(committed, maxInputs) + total += n + compactErr = errors.Join(compactErr, err) + if err != nil || n < 2 { + break + } + } + return total, compactErr +} + // ScanHotLogs reads the portion of [start,end) that is guaranteed complete in // the hot tier and returns the durable boundary below which Parquet is // authoritative. The boundary and scan are serialized with PruneHot. @@ -226,10 +315,10 @@ func (r *Repository) PruneParquet(cutoff int64) (int, error) { // Commit durably records a batch and publishes its three signal projections // exactly once. A crash at any point leaves the WAL for replay on next boot. func (r *Repository) Commit(batch Batch) error { + normalizeBatch(&batch) if err := validateBatch(batch); err != nil { return err } - normalizeBatch(&batch) // Commits are serialized, but their segment and Parquet fsyncs do not hold // the repository metadata lock. Each projection has its own atomic publish // protocol; the WAL keeps a partially applied transaction replayable. @@ -258,10 +347,10 @@ func (r *Repository) Commit(batch Batch) error { // Writers call this before handing a batch to an asynchronous commit worker, so // every queued or in-flight batch is replayable if shutdown interrupts retries. func (r *Repository) Stage(batch Batch) error { + normalizeBatch(&batch) if err := validateBatch(batch); err != nil { return err } - normalizeBatch(&batch) r.commitMu.Lock() defer r.commitMu.Unlock() consumed := r.batchConsumedLocked(batch.ID) @@ -281,6 +370,12 @@ func validateBatch(batch Batch) error { if !segment.ValidID(batch.ID) { return fmt.Errorf("telemetry batch ID %q cannot name a segment", batch.ID) } + if rows := len(batch.Spans) + len(batch.Logs) + len(batch.Metrics); rows > maxBatchRows { + return fmt.Errorf("telemetry batch has %d rows; maximum is %d", rows, maxBatchRows) + } + if err := segment.ValidateSpanRows(batch.Spans); err != nil { + return fmt.Errorf("telemetry batch cannot be represented by the hot span tier: %w", err) + } return nil } @@ -313,16 +408,10 @@ func (r *Repository) writeWAL(batch Batch) error { } else if !errors.Is(err, os.ErrNotExist) { return err } - var plain bytes.Buffer - if err := gob.NewEncoder(&plain).Encode(batch); err != nil { - return err - } - enc, err := zstd.NewWriter(nil, zstd.WithEncoderCRC(true), zstd.WithEncoderConcurrency(1)) + data, err := encodeWALBatch(batch, walDecoderMaxMemory) if err != nil { return err } - data := enc.EncodeAll(plain.Bytes(), nil) - enc.Close() tmp := final + ".tmp" _ = os.Remove(tmp) f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) @@ -346,15 +435,30 @@ func (r *Repository) writeWAL(batch Batch) error { return syncDirectory(r.walDir) } -// walDecoderMaxMemory bounds one decompressed WAL batch. Flush batches are -// bounded by the writer's batch size, so this sits far above any batch the -// writer produces while refusing a corrupt or crafted frame long before zstd's -// 64 GiB default would. -const walDecoderMaxMemory = 1 << 30 +func encodeWALBatch(batch Batch, maxDecodedBytes int) ([]byte, error) { + var plain bytes.Buffer + if err := gob.NewEncoder(&plain).Encode(batch); err != nil { + return nil, err + } + if plain.Len() > maxDecodedBytes { + return nil, fmt.Errorf("telemetry batch encodes to %d bytes; maximum is %d", plain.Len(), maxDecodedBytes) + } + enc, err := zstd.NewWriter(nil, zstd.WithEncoderCRC(true), zstd.WithEncoderConcurrency(1)) + if err != nil { + return nil, err + } + data := enc.EncodeAll(plain.Bytes(), nil) + enc.Close() + return data, nil +} // newWALDecoder builds the bounded decoder every WAL read goes through. func newWALDecoder() (*zstd.Decoder, error) { - return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(walDecoderMaxMemory)) + return newWALDecoderWithLimit(walDecoderMaxMemory) +} + +func newWALDecoderWithLimit(limit uint64) (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(limit)) } func (r *Repository) recover() error { @@ -405,6 +509,7 @@ func (r *Repository) recover() error { // so the WAL is retained and startup fails loudly — a later healthy boot // must still be able to finish the batch, including one whose projection // prefix this attempt already published. + normalizeBatch(&batch) if err := validateBatch(batch); err != nil { if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { return errors.Join(fmt.Errorf("replay %s: %w", name, err), quarantineErr) @@ -489,6 +594,9 @@ func (r *Repository) batchConsumed(id string) bool { func (r *Repository) batchConsumedLocked(id string) bool { for _, batch := range r.manifest.Batches { + if batch.ID == id { + return true + } for _, source := range batch.Sources { if source == id { return true diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index a3ba1c9e..017db66e 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -661,13 +661,14 @@ func TestStageRejectsBatchTheSegmentStoresCannotAccept(t *testing.T) { } func TestWALDecoderRejectsOversizedFrame(t *testing.T) { + const testLimit = 64 << 10 encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) if err != nil { t.Fatal(err) } defer encoder.Close() - frame := encoder.EncodeAll(make([]byte, walDecoderMaxMemory+(1<<20)), nil) - decoder, err := newWALDecoder() + frame := encoder.EncodeAll(make([]byte, testLimit+(1<<10)), nil) + decoder, err := newWALDecoderWithLimit(testLimit) if err != nil { t.Fatal(err) } @@ -676,3 +677,81 @@ func TestWALDecoderRejectsOversizedFrame(t *testing.T) { t.Fatal("WAL decoder accepted a frame declaring more memory than any batch needs") } } + +func TestWALWriterRejectsBatchLargerThanDecoderBudget(t *testing.T) { + batch := Batch{ID: "large", Logs: []telemetry.Log{{Body: strings.Repeat("x", 2048)}}} + if _, err := encodeWALBatch(batch, 1024); err == nil { + t.Fatal("WAL writer produced a frame its paired decoder budget could not reopen") + } +} + +func TestCompactHotCompactsEverySignalAndPreservesRows(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range 6 { + now := int64(100 + i) + batch := Batch{ + ID: fmt.Sprintf("batch-%d", i), + Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: now, IngestedAt: now}}, + Logs: []telemetry.Log{{Body: fmt.Sprintf("log-%d", i), EventUnixNanos: now, IngestedAt: now}}, + Metrics: []telemetry.Metric{{Name: "requests", EventUnixNanos: now, IngestedAt: now}}, + } + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + if _, err := repository.CompactHot(3); err != nil { + t.Fatal(err) + } + if got := repository.Spans.SegmentCount(); got != 2 { + t.Fatalf("span segments = %d, want 2 compacted files", got) + } + if got := repository.Logs.SegmentCount(); got != 2 { + t.Fatalf("log segments = %d, want 2 compacted files", got) + } + if got := repository.Metrics.SegmentCount(); got != 2 { + t.Fatalf("metric segments = %d, want 2 compacted files", got) + } + if repository.Spans.RowCount() != 6 || repository.Logs.RowCount() != 6 || repository.Metrics.RowCount() != 6 { + t.Fatal("hot compaction changed row counts") + } +} + +func TestOpenQuarantinesCorruptDisposableHotTier(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := Batch{ID: "committed", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}} + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "hot", "spans", "committed.fseg"), []byte("corrupt"), 0o644); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir) + if err != nil { + t.Fatalf("Open failed on disposable hot corruption: %v", err) + } + defer reopened.Close() + if got := reopened.Spans.RowCount(); got != 0 { + t.Fatalf("rebuilt hot rows = %d, want empty acceleration tier", got) + } + if reopened.manifest.HotCutoffNanos == 0 { + t.Fatal("rebuilt hot tier did not move the authoritative boundary to Parquet") + } + matches, err := filepath.Glob(filepath.Join(dir, "hot.corrupt-*")) + if err != nil || len(matches) != 1 { + t.Fatalf("hot quarantine paths = %v, err = %v", matches, err) + } + if _, err := os.Stat(filepath.Join(dir, "parquet", "spans", "committed.parquet")); err != nil { + t.Fatalf("authoritative Parquet was not preserved: %v", err) + } +} diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 6ac07f91..6b02c077 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -2,6 +2,8 @@ package store import ( "context" + "errors" + "fmt" "log/slog" "time" @@ -11,10 +13,11 @@ import ( ) const ( - flushQueueDepth = 4 - carryBatches = 3 - commitRetryLimit = 8 - writerShutdownGrace = 5 * time.Second + flushQueueDepth = 4 + carryBatches = 3 + commitRetryLimit = 8 + writerShutdownGrace = 5 * time.Second + submissionQueueDepth = 256 ) type batchCommitter interface { @@ -35,14 +38,45 @@ type Writer struct { retryDelay func(int) time.Duration shutdownGrace time.Duration done chan struct{} + submissions chan submission +} + +type submission struct { + batch Batch + ack chan error } func NewWriter(repository *Repository, interval time.Duration, batchSize int, spans <-chan telemetry.Span, logs <-chan telemetry.Log, metricRows <-chan telemetry.Metric) *Writer { - return &Writer{repository: repository, interval: interval, batchSize: batchSize, spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{})} + return &Writer{repository: repository, interval: interval, batchSize: batchSize, spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth)} } func (w *Writer) Wait() { <-w.done } +// Submit accepts one decoded OTLP request. It returns only after the complete +// request is fsynced to the WAL, so a successful OTLP response is durable even +// though publication to the query projections continues asynchronously. +func (w *Writer) Submit(ctx context.Context, batch Batch) error { + if len(batch.Spans)+len(batch.Logs)+len(batch.Metrics) == 0 { + return nil + } + request := submission{batch: batch, ack: make(chan error, 1)} + select { + case w.submissions <- request: + case <-w.done: + return errors.New("telemetry writer is stopped") + case <-ctx.Done(): + return ctx.Err() + } + select { + case err := <-request.ack: + return err + case <-w.done: + return errors.New("telemetry writer stopped before durable acknowledgement") + case <-ctx.Done(): + return ctx.Err() + } +} + func (w *Writer) Run(ctx context.Context) error { defer close(w.done) flushes := make(chan Batch, flushQueueDepth) @@ -53,6 +87,7 @@ func (w *Writer) Run(ctx context.Context) error { ticker := time.NewTicker(w.interval) defer ticker.Stop() spans, logs, metricRows := w.spans, w.logs, w.metricRows + legacyInputs := spans != nil || logs != nil || metricRows != nil finish := func() error { w.drain(&spans, &logs, &metricRows) if err := w.flushBuffered(flushes, workerDone, true); err != nil { @@ -72,6 +107,10 @@ func (w *Writer) Run(ctx context.Context) error { } for { select { + case request := <-w.submissions: + if err := w.stageSubmission(request, flushes, workerDone); err != nil { + return err + } case row, ok := <-spans: if !ok { spans = nil @@ -107,12 +146,69 @@ func (w *Writer) Run(ctx context.Context) error { return err } } - if spans == nil && logs == nil && metricRows == nil { + if legacyInputs && spans == nil && logs == nil && metricRows == nil { return finish() } } } +func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDone <-chan error) error { + requests := []submission{request} + draining := true + for draining && len(requests) < submissionQueueDepth { + select { + case next := <-w.submissions: + requests = append(requests, next) + default: + draining = false + } + } + limit := min(w.batchSize, maxBatchRows) + if limit <= 0 { + limit = maxBatchRows + } + for len(requests) > 0 { + batch := Batch{ID: uuid.NewString()} + group := make([]submission, 0, len(requests)) + rows := 0 + for len(requests) > 0 { + next := requests[0] + nextRows := len(next.batch.Spans) + len(next.batch.Logs) + len(next.batch.Metrics) + if len(group) > 0 && rows+nextRows > limit { + break + } + requests = requests[1:] + group = append(group, next) + rows += nextRows + batch.Spans = append(batch.Spans, next.batch.Spans...) + batch.Logs = append(batch.Logs, next.batch.Logs...) + batch.Metrics = append(batch.Metrics, next.batch.Metrics...) + if rows >= limit { + break + } + } + if err := w.repository.Stage(batch); err != nil { + metrics.FlushErrors.WithLabelValues("stage").Inc() + for _, item := range group { + item.ack <- err + } + continue + } + metrics.RecordIngest("spans", len(batch.Spans)) + metrics.RecordIngest("logs", len(batch.Logs)) + metrics.RecordIngest("metrics", len(batch.Metrics)) + for _, item := range group { + item.ack <- nil + } + select { + case out <- batch: + case err := <-workerDone: + return err + } + } + return nil +} + func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { return w.flushBuffered(out, workerDone, false) } @@ -208,7 +304,9 @@ func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- e // the manifest, so deleting the WAL here would both lose the rows and // strand any parquet file the failed attempt already wrote. metrics.FlushErrors.WithLabelValues("deferred").Inc() - slog.Error("telemetry batch commit failed after bounded retries; deferring to WAL replay on next start", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) + slog.Error("telemetry batch commit failed after bounded retries; stopping ingest with WAL retained for replay", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) + done <- fmt.Errorf("commit telemetry batch %s after %d attempts: %w", batch.ID, commitRetryLimit, lastErr) + return } } } diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index 423cdaf0..6d384ee7 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -22,6 +22,84 @@ type recoveringCommitter struct { batches []Batch } +type blockingStageCommitter struct { + entered chan struct{} + release chan struct{} +} + +func (c *blockingStageCommitter) Stage(Batch) error { + close(c.entered) + <-c.release + return nil +} + +func (c *blockingStageCommitter) Commit(Batch) error { return nil } + +func TestWriterAcknowledgesSubmissionOnlyAfterDurableStage(t *testing.T) { + committer := &blockingStageCommitter{entered: make(chan struct{}), release: make(chan struct{})} + w := &Writer{repository: committer, interval: time.Hour, batchSize: 50_000, done: make(chan struct{}), submissions: make(chan submission, 1)} + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + submitDone := make(chan error, 1) + go func() { + submitDone <- w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}) + }() + select { + case <-committer.entered: + case <-time.After(time.Second): + t.Fatal("Stage was not called") + } + select { + case err := <-submitDone: + t.Fatalf("Submit returned before Stage completed: %v", err) + default: + } + close(committer.release) + if err := <-submitDone; err != nil { + t.Fatalf("Submit error = %v", err) + } + cancel() + if err := <-runDone; err != nil { + t.Fatalf("Run error = %v", err) + } +} + +func TestWriterGroupCommitsConcurrentSubmissions(t *testing.T) { + committer := &recoveringCommitter{} + w := &Writer{repository: committer, interval: time.Hour, batchSize: 50_000, done: make(chan struct{}), submissions: make(chan submission, 4)} + results := make(chan error, 2) + for i := range 2 { + go func() { + results <- w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i)}}}) + }() + } + deadline := time.Now().Add(time.Second) + for len(w.submissions) != 2 { + if time.Now().After(deadline) { + t.Fatal("submissions were not queued") + } + time.Sleep(time.Millisecond) + } + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + for range 2 { + if err := <-results; err != nil { + t.Fatal(err) + } + } + cancel() + if err := <-runDone; err != nil { + t.Fatal(err) + } + committer.mu.Lock() + defer committer.mu.Unlock() + if len(committer.staged) != 1 || len(committer.staged[0].Spans) != 2 { + t.Fatalf("staged batches = %#v, want one two-row group commit", committer.staged) + } +} + func (c *recoveringCommitter) Stage(batch Batch) error { c.mu.Lock() defer c.mu.Unlock() @@ -138,7 +216,7 @@ func TestWriterShutdownReplaysEveryStagedBatch(t *testing.T) { } } -func TestWriterDropsPoisonBatchAfterBoundedRetries(t *testing.T) { +func TestWriterSurfacesPermanentCommitFailureAfterBoundedRetries(t *testing.T) { spans := make(chan telemetry.Span, 1) logs := make(chan telemetry.Log) metricRows := make(chan telemetry.Metric) @@ -152,8 +230,8 @@ func TestWriterDropsPoisonBatchAfterBoundedRetries(t *testing.T) { spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), retryDelay: func(int) time.Duration { return 0 }, } - if err := w.Run(context.Background()); err != nil { - t.Fatalf("Run error = %v", err) + if err := w.Run(context.Background()); err == nil { + t.Fatal("Run returned nil after permanent commit failure") } if committer.calls != commitRetryLimit { t.Fatalf("Commit calls = %d, want %d", committer.calls, commitRetryLimit) @@ -280,8 +358,8 @@ func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), retryDelay: func(int) time.Duration { return 0 }, } - if err := w.Run(context.Background()); err != nil { - t.Fatalf("Run error = %v", err) + if err := w.Run(context.Background()); err == nil { + t.Fatal("Run returned nil after permanent commit failure") } entries, err := os.ReadDir(filepath.Join(dir, "wal")) if err != nil { From a221291c2bcb1883a529eec63a980062a0be2739 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 00:43:05 -0700 Subject: [PATCH 11/31] feat(storage)!: finalize telemetry engine Use WAL-backed request acknowledgements, atomic Parquet publication, a span-only hot index, and DuckDB for bounded analytical reads. BREAKING CHANGE: existing DuckLake telemetry and deprecated ingest flush settings are not migrated. --- .gitignore | 1 - bench/storage/chdb/go.mod | 38 + bench/storage/chdb/go.sum | 88 ++ .../storage/chdb}/main.go | 2 +- {cmd/storage-poc => bench/storage}/main.go | 8 +- cmd/bench/main.go | 2 +- cmd/bench/metrics_report_test.go | 6 +- cmd/fanout/main.go | 4 +- docs/diagrams/architecture.d2 | 12 +- docs/diagrams/architecture.svg | 190 ++-- docs/diagrams/persistence.d2 | 20 +- docs/diagrams/persistence.svg | 192 ++-- docs/operations.md | 8 +- docs/storage-architecture-options.md | 12 +- docs/{storage-poc.md => storage-benchmark.md} | 53 +- experiments/storage-poc-chdb/go.mod | 23 - experiments/storage-poc-chdb/go.sum | 24 - fanout.example.yaml | 3 +- internal/api/health.go | 2 +- internal/api/health_test.go | 2 +- internal/config/config.go | 12 +- internal/config/config_test.go | 34 +- internal/metrics/metrics_test.go | 8 +- internal/observability/logs.go | 197 +---- .../performance_benchmark_test.go | 2 +- .../observability/performance_rollup_test.go | 2 +- internal/observability/service_test.go | 96 +- internal/observability/trace.go | 61 +- internal/query/attr_json_test.go | 2 +- internal/query/duck.go | 15 +- internal/query/edge_backlog_test.go | 6 +- internal/query/ensure_limit_test.go | 21 - internal/query/parquet_gate.go | 21 +- internal/query/parquet_gate_test.go | 52 ++ internal/query/rollup_test.go | 12 +- internal/query/schema.go | 10 +- internal/query/schema_test.go | 2 +- internal/query/sql.go | 11 +- internal/query/sql_test.go | 8 +- internal/query/views.go | 24 +- internal/telemetry/parquet.go | 109 ++- internal/telemetry/parquet_test.go | 56 ++ internal/telemetry/segment/signal_store.go | 826 ------------------ .../telemetry/segment/signal_store_test.go | 148 ---- internal/telemetry/segment/span_store.go | 9 +- internal/telemetry/store/compaction.go | 59 +- internal/telemetry/store/publication_test.go | 59 ++ internal/telemetry/store/repository.go | 381 ++++---- internal/telemetry/store/repository_test.go | 226 ++++- internal/telemetry/store/writer.go | 258 ++---- internal/telemetry/store/writer_test.go | 359 ++++---- .../docs/explanation/storage-model.mdx | 93 +- .../docs/guides/back-up-and-restore.mdx | 15 +- .../content/docs/guides/tune-retention.mdx | 23 +- .../content/docs/reference/data-layout.mdx | 21 +- .../docs/reference/settings/ingest.mdx | 3 +- site/src/content/docs/start/first-boot.mdx | 2 +- 57 files changed, 1642 insertions(+), 2291 deletions(-) create mode 100644 bench/storage/chdb/go.mod create mode 100644 bench/storage/chdb/go.sum rename {experiments/storage-poc-chdb => bench/storage/chdb}/main.go (99%) rename {cmd/storage-poc => bench/storage}/main.go (98%) rename docs/{storage-poc.md => storage-benchmark.md} (58%) delete mode 100644 experiments/storage-poc-chdb/go.mod delete mode 100644 experiments/storage-poc-chdb/go.sum create mode 100644 internal/telemetry/parquet_test.go delete mode 100644 internal/telemetry/segment/signal_store.go delete mode 100644 internal/telemetry/segment/signal_store_test.go create mode 100644 internal/telemetry/store/publication_test.go diff --git a/.gitignore b/.gitignore index 43165f98..17d80c14 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,5 @@ cover.out .playwright-mcp/ # Ad-hoc output from `go build` without -o (named after the package). -/bench /fanout /fanout-docgen diff --git a/bench/storage/chdb/go.mod b/bench/storage/chdb/go.mod new file mode 100644 index 00000000..ad066034 --- /dev/null +++ b/bench/storage/chdb/go.mod @@ -0,0 +1,38 @@ +module github.com/labstack/fanout/bench/storage/chdb + +go 1.27.0 + +require ( + github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 + github.com/chdb-io/chdb-go/v2 v2.1.0 + github.com/labstack/fanout v0.0.0 +) + +require ( + github.com/andybalholm/brotli v1.2.2 // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/apache/thrift v0.24.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 // indirect + github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 // indirect + github.com/ebitengine/purego v0.8.2 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/pierrec/lz4/v4 v4.1.29 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace github.com/labstack/fanout => ../../.. diff --git a/bench/storage/chdb/go.sum b/bench/storage/chdb/go.sum new file mode 100644 index 00000000..4ec371e6 --- /dev/null +++ b/bench/storage/chdb/go.sum @@ -0,0 +1,88 @@ +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 h1:QiXQ1vZWcQCbRpciirWG/+F3KRXYnDLiFOVYkAJxCls= +github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1/go.mod h1:jB7U0oct7fDV+SbrDzh3oorQFAQ8YOM5QFXf273ZEEs= +github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 h1:qvkIS/fozvgJfd30MEPM1nVDSM1JMXQgqZfqp1Oh7aQ= +github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1/go.mod h1:tebe6DiYx113PoHD0WjWCWVY74QDmaPcCdWA+aNpWPM= +github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 h1:FBSXH0ChVm7fOEHUWpf3bNd2BKnp1cv9LhEn1eLcUdE= +github.com/chdb-io/chdb-go/lib/embedded v0.260700.1/go.mod h1:N9Dra/RfDuELfnT2TSMVBF+NzyDKL6AFiDqHwCWF7T0= +github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 h1:w1q1whc7LlBpJtzoMkokgnSsdRrFLAG2kgRkb1tQ7jM= +github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1/go.mod h1:LK3ORN5rYtQDUYKswMZKf09RT1qIdwlCGk/3/vB7aHE= +github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 h1:GoviqHKnVOJIfnfgiSYWiEqxciThy2XGmSHUdq1qvmI= +github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1/go.mod h1:vkAVOjzg+j6TwFWobfxvmU+pV8fD7dS9ibBdZc3Wf8Y= +github.com/chdb-io/chdb-go/v2 v2.1.0 h1:Nf/StmYfE90mePp0EzdWqCbqUv/TJUbVOrnea/x3PN0= +github.com/chdb-io/chdb-go/v2 v2.1.0/go.mod h1:tyiHoF8pWUfrD7ylseofFEnELnA+jocf/yElE3AHBaQ= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= +github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/experiments/storage-poc-chdb/main.go b/bench/storage/chdb/main.go similarity index 99% rename from experiments/storage-poc-chdb/main.go rename to bench/storage/chdb/main.go index 8648c1ea..ed448c51 100644 --- a/experiments/storage-poc-chdb/main.go +++ b/bench/storage/chdb/main.go @@ -222,7 +222,7 @@ func formatDuration(value time.Duration) string { return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) } -func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-poc-chdb:", err); os.Exit(1) } +func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-bench-chdb:", err); os.Exit(1) } const spansDDL = `CREATE TABLE fanout.spans ( namespace String,trace_id String,span_id String,parent_span_id String,service_name LowCardinality(String),name LowCardinality(String),kind LowCardinality(String), diff --git a/cmd/storage-poc/main.go b/bench/storage/main.go similarity index 98% rename from cmd/storage-poc/main.go rename to bench/storage/main.go index eb1d9321..81fb4b90 100644 --- a/cmd/storage-poc/main.go +++ b/bench/storage/main.go @@ -1,4 +1,4 @@ -// Command storage-poc compares a workload-specific immutable segment format +// Command storage-bench compares Fanout's storage path with alternative formats // with native DuckDB and Parquet on Fanout-shaped spans. It is an experiment, // not a supported Fanout command. package main @@ -54,7 +54,7 @@ func main() { root := *keep if root == "" { var err error - root, err = os.MkdirTemp("", "fanout-storage-poc-") + root, err = os.MkdirTemp("", "fanout-storage-bench-") if err != nil { fatal(err) } @@ -67,7 +67,7 @@ func main() { targetTrace := storagebench.TraceID(uint64(*rows / 2 / 5)) start := base end := base + storagebench.DayNanos - fmt.Printf("Fanout storage POC: %d spans, %d-row commits, %s/%s, %d CPUs\n", *rows, *batch, runtime.GOOS, runtime.GOARCH, runtime.NumCPU()) + fmt.Printf("Fanout storage benchmark: %d spans, %d-row commits, %s/%s, %d CPUs\n", *rows, *batch, runtime.GOOS, runtime.GOARCH, runtime.NumCPU()) var results []result if *engine == "all" || *engine == "repository" { @@ -522,4 +522,4 @@ func formatDuration(value time.Duration) string { return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) } -func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-poc:", err); os.Exit(1) } +func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-bench:", err); os.Exit(1) } diff --git a/cmd/bench/main.go b/cmd/bench/main.go index e167a81a..1869f408 100644 --- a/cmd/bench/main.go +++ b/cmd/bench/main.go @@ -81,7 +81,7 @@ type config struct { maxQueryP95 float64 // backfillHours, when >0, spreads each event's timestamp uniformly over the // last N hours (instead of "now"). Used to PRE-SEED a multi-hour dataset so - // the lake spans several hour partitions — required to exercise within-day + // Parquet spans several hour partitions — required to exercise within-day // (hour-partition) pruning, which a same-hour run can't. backfillHours float64 // seed makes the synthetic workload reproducible: same seed, same services, diff --git a/cmd/bench/metrics_report_test.go b/cmd/bench/metrics_report_test.go index a87a4746..3074dd94 100644 --- a/cmd/bench/metrics_report_test.go +++ b/cmd/bench/metrics_report_test.go @@ -141,9 +141,9 @@ fanout_rollup_component_total{rollup="service",result="noop"} 2 assertFloat(t, "ingest rows start", report.IngestRowsStart, 15) assertFloat(t, "ingest rows end", report.IngestRowsEnd, 45) assertFloat(t, "ingest rows", report.IngestRowsDelta, 30) - assertFloat(t, "lake partitions delta", report.ParquetFilesDelta, 2) - assertFloat(t, "lake size delta", report.ParquetSizeBytesDelta, 80) - assertFloat(t, "lake growth rate", report.ParquetGrowthBytesPerSec, 10) + assertFloat(t, "Parquet files delta", report.ParquetFilesDelta, 2) + assertFloat(t, "Parquet size delta", report.ParquetSizeBytesDelta, 80) + assertFloat(t, "Parquet growth rate", report.ParquetGrowthBytesPerSec, 10) assertFloat(t, "average rollup", report.AvgRollupMs, 750) assertFloat(t, "average flush", report.AvgFlushMs, 200) assertFloat(t, "average query", report.AvgQueryMs, 500) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 9aaa921e..7bae6b5e 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -120,12 +120,12 @@ func main() { } defer q.Close() - writer := telemetrystore.NewWriter(repository, cfg.FlushInterval, cfg.FlushBatchSize, nil, nil, nil) + writer := telemetrystore.NewWriter(repository, cfg.IngestBatchSize) writerResult := make(chan error, 1) go func() { err := writer.Run(ctx) // Publish the result before notifying the process-wide error channel. The - // shutdown path waits on writerResult after Writer.Wait, so a final-flush + // shutdown path waits on writerResult after Writer.Wait, so a final-commit // failure cannot be lost in the close(done) -> goroutine-send scheduling gap. writerResult <- err if err != nil { diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 index 7932131b..01acf2a4 100644 --- a/docs/diagrams/architecture.d2 +++ b/docs/diagrams/architecture.d2 @@ -26,12 +26,11 @@ fanout: "fanout — one Go process" { agent: "Agent runtime\nmodel + tool loop" mcp: "MCP server\n5 typed + 4 dashboard tools" obs: "Typed observability contract" - lake: "Lake writer\nbatched flush" + commit: "Telemetry commit worker\nWAL to Parquet" query: "Query kernel\nDuckDB + rollups" - gate: "Write gate\none catalog write in flight" {style.stroke-width: 2} alert: "Alert engine\nrule evaluation + webhooks" - ingest -> lake + ingest -> commit http -> ui: serves embedded assets http -> agent: AG-UI stream http -> mcp @@ -40,13 +39,12 @@ fanout: "fanout — one Go process" { http -> obs: typed HTTP API obs -> query alert -> query: evaluates rollups - lake -> gate - query -> gate: "rollups, merge, maintenance" + query -> commit: "merge and maintenance" } store: Storage { style.fill: transparent - telemetry: "DuckLake + Parquet\nstorage.data_dir/telemetry" {shape: cylinder} + telemetry: "WAL + manifest + Parquet\nstorage.data_dir/telemetry" {shape: cylinder} qstate: "Query catalog\nstorage.data_dir/query" {shape: cylinder} control: "Control SQLite\nstorage.data_dir/control" {shape: cylinder} } @@ -57,7 +55,7 @@ clients.collector -> fanout.ingest: "OTLP/gRPC or OTLP/HTTP" clients.browser -> fanout.http: HTTPS clients.ext -> fanout.http: "/mcp — OAuth" -fanout.gate -> store.telemetry +fanout.commit -> store.telemetry fanout.query -> store.qstate fanout.http -> store.control: "users, sessions, settings, dashboards, threads" fanout.alert -> store.control: "rules and fired alerts" diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 4e266d4f..cd4d4d86 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,23 +1,23 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractLake writerbatched flushQuery kernelDuckDB + rollupsWrite gateone catalog write in flightAlert enginerule evaluation + webhooksDuckLake + Parquetstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsrollups, merge, maintenanceOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - - - - - - - - - - - - - + .d2-2259769862 .fill-N1{fill:#0A0F25;} + .d2-2259769862 .fill-N2{fill:#676C7E;} + .d2-2259769862 .fill-N3{fill:#9499AB;} + .d2-2259769862 .fill-N4{fill:#CFD2DD;} + .d2-2259769862 .fill-N5{fill:#DEE1EB;} + .d2-2259769862 .fill-N6{fill:#EEF1F8;} + .d2-2259769862 .fill-N7{fill:#FFFFFF;} + .d2-2259769862 .fill-B1{fill:#000536;} + .d2-2259769862 .fill-B2{fill:#0F66B7;} + .d2-2259769862 .fill-B3{fill:#4393DD;} + .d2-2259769862 .fill-B4{fill:#87BFF3;} + .d2-2259769862 .fill-B5{fill:#BCDDFB;} + .d2-2259769862 .fill-B6{fill:#E5F3FF;} + .d2-2259769862 .fill-AA2{fill:#7639C5;} + .d2-2259769862 .fill-AA4{fill:#C1A2F3;} + .d2-2259769862 .fill-AA5{fill:#DACEFB;} + .d2-2259769862 .fill-AB4{fill:#EA99C6;} + .d2-2259769862 .fill-AB5{fill:#FFDEF1;} + .d2-2259769862 .stroke-N1{stroke:#0A0F25;} + .d2-2259769862 .stroke-N2{stroke:#676C7E;} + .d2-2259769862 .stroke-N3{stroke:#9499AB;} + .d2-2259769862 .stroke-N4{stroke:#CFD2DD;} + .d2-2259769862 .stroke-N5{stroke:#DEE1EB;} + .d2-2259769862 .stroke-N6{stroke:#EEF1F8;} + .d2-2259769862 .stroke-N7{stroke:#FFFFFF;} + .d2-2259769862 .stroke-B1{stroke:#000536;} + .d2-2259769862 .stroke-B2{stroke:#0F66B7;} + .d2-2259769862 .stroke-B3{stroke:#4393DD;} + .d2-2259769862 .stroke-B4{stroke:#87BFF3;} + .d2-2259769862 .stroke-B5{stroke:#BCDDFB;} + .d2-2259769862 .stroke-B6{stroke:#E5F3FF;} + .d2-2259769862 .stroke-AA2{stroke:#7639C5;} + .d2-2259769862 .stroke-AA4{stroke:#C1A2F3;} + .d2-2259769862 .stroke-AA5{stroke:#DACEFB;} + .d2-2259769862 .stroke-AB4{stroke:#EA99C6;} + .d2-2259769862 .stroke-AB5{stroke:#FFDEF1;} + .d2-2259769862 .background-color-N1{background-color:#0A0F25;} + .d2-2259769862 .background-color-N2{background-color:#676C7E;} + .d2-2259769862 .background-color-N3{background-color:#9499AB;} + .d2-2259769862 .background-color-N4{background-color:#CFD2DD;} + .d2-2259769862 .background-color-N5{background-color:#DEE1EB;} + .d2-2259769862 .background-color-N6{background-color:#EEF1F8;} + .d2-2259769862 .background-color-N7{background-color:#FFFFFF;} + .d2-2259769862 .background-color-B1{background-color:#000536;} + .d2-2259769862 .background-color-B2{background-color:#0F66B7;} + .d2-2259769862 .background-color-B3{background-color:#4393DD;} + .d2-2259769862 .background-color-B4{background-color:#87BFF3;} + .d2-2259769862 .background-color-B5{background-color:#BCDDFB;} + .d2-2259769862 .background-color-B6{background-color:#E5F3FF;} + .d2-2259769862 .background-color-AA2{background-color:#7639C5;} + .d2-2259769862 .background-color-AA4{background-color:#C1A2F3;} + .d2-2259769862 .background-color-AA5{background-color:#DACEFB;} + .d2-2259769862 .background-color-AB4{background-color:#EA99C6;} + .d2-2259769862 .background-color-AB5{background-color:#FFDEF1;} + .d2-2259769862 .color-N1{color:#0A0F25;} + .d2-2259769862 .color-N2{color:#676C7E;} + .d2-2259769862 .color-N3{color:#9499AB;} + .d2-2259769862 .color-N4{color:#CFD2DD;} + .d2-2259769862 .color-N5{color:#DEE1EB;} + .d2-2259769862 .color-N6{color:#EEF1F8;} + .d2-2259769862 .color-N7{color:#FFFFFF;} + .d2-2259769862 .color-B1{color:#000536;} + .d2-2259769862 .color-B2{color:#0F66B7;} + .d2-2259769862 .color-B3{color:#4393DD;} + .d2-2259769862 .color-B4{color:#87BFF3;} + .d2-2259769862 .color-B5{color:#BCDDFB;} + .d2-2259769862 .color-B6{color:#E5F3FF;} + .d2-2259769862 .color-AA2{color:#7639C5;} + .d2-2259769862 .color-AA4{color:#C1A2F3;} + .d2-2259769862 .color-AA5{color:#DACEFB;} + .d2-2259769862 .color-AB4{color:#EA99C6;} + .d2-2259769862 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2259769862);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2259769862);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2259769862);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2259769862);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2259769862);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2259769862);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2259769862);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2259769862);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2259769862);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2259769862);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2259769862);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2259769862);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workerWAL to ParquetQuery kernelDuckDB + rollupsAlert enginerule evaluation + webhooksWAL + manifest + Parquetstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsmerge and maintenanceOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + + + + + + + + + + + + + diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 index 2345f6cb..fb2bf477 100644 --- a/docs/diagrams/persistence.d2 +++ b/docs/diagrams/persistence.d2 @@ -12,12 +12,12 @@ vars: { writers: Writers { style.fill: transparent - ingest: "Ingest flush\nspans, logs, metrics" {shape: rectangle} + ingest: "OTLP request\nspans, logs, metrics" {shape: rectangle} rollup: "Rollups\nservice, endpoint, edge" {shape: rectangle} maint: "Merge and maintenance" {shape: rectangle} } -gate: "Write gate — internal/lake/writegate\none catalog write in flight at a time" { +wal: "Durable WAL + commit worker\nrequest acknowledged after fsync" { shape: rectangle style: { stroke-width: 2 @@ -25,14 +25,14 @@ gate: "Write gate — internal/lake/writegate\none catalog write in flight at a } } -telemetry: "DuckLake catalog + Parquet\nstorage.data_dir/telemetry" { +telemetry: "Manifest + Parquet + hot span index\nstorage.data_dir/telemetry" { shape: cylinder - tooltip: Partitioned telemetry. Written only through the gate. + tooltip: Authoritative telemetry and its crash-recovery state. } querystate: "DuckDB query state\nstorage.data_dir/query" { shape: cylinder - tooltip: Catalog attachment and temp spill. Not the telemetry itself. + tooltip: Rebuildable rollups and temp spill. Not the telemetry itself. } control: "Control SQLite\nstorage.data_dir/control/fanout.sqlite" { @@ -45,9 +45,9 @@ control_tables: "users, user_identities, verifications, sessions, auth_audit_eve style.font-size: 13 } -writers.ingest -> gate -writers.rollup -> gate -writers.maint -> gate -gate -> telemetry: "wait and hold measured per operation" -telemetry <- querystate: attached +writers.ingest -> wal +wal -> telemetry: "asynchronous publication" +writers.rollup -> querystate +writers.maint -> telemetry +telemetry <- querystate: "DuckDB scans Parquet" control -> control_tables: {style.stroke-dash: 3} diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 9ccf978c..80ff6de4 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,27 +1,27 @@ -WritersWrite gate — internal/lake/writegateone catalog write in flight at a timeDuckLake catalog + Parquetstorage.data_dir/telemetryPartitioned telemetry. Written only through the gate.DuckDB query statestorage.data_dir/queryCatalog attachment and temp spill. Not the telemetry itself.Control SQLitestorage.data_dir/control/fanout.sqliteApplication state. Never on the telemetry write path.users, user_identities, verifications, sessions, auth_audit_eventsoauth_clients, oauth_tokens, oauth_authorization_codesdashboards, dashboard_widgets, dashboard_stateagui_threads, agui_runs, alert_rules, alerts, settingsIngest flushspans, logs, metricsRollupsservice, endpoint, edgeMerge and maintenance wait and hold measured per operation attached Partitioned telemetry. Written only through the gate. - + .d2-2154956904 .fill-N1{fill:#0A0F25;} + .d2-2154956904 .fill-N2{fill:#676C7E;} + .d2-2154956904 .fill-N3{fill:#9499AB;} + .d2-2154956904 .fill-N4{fill:#CFD2DD;} + .d2-2154956904 .fill-N5{fill:#DEE1EB;} + .d2-2154956904 .fill-N6{fill:#EEF1F8;} + .d2-2154956904 .fill-N7{fill:#FFFFFF;} + .d2-2154956904 .fill-B1{fill:#000536;} + .d2-2154956904 .fill-B2{fill:#0F66B7;} + .d2-2154956904 .fill-B3{fill:#4393DD;} + .d2-2154956904 .fill-B4{fill:#87BFF3;} + .d2-2154956904 .fill-B5{fill:#BCDDFB;} + .d2-2154956904 .fill-B6{fill:#E5F3FF;} + .d2-2154956904 .fill-AA2{fill:#7639C5;} + .d2-2154956904 .fill-AA4{fill:#C1A2F3;} + .d2-2154956904 .fill-AA5{fill:#DACEFB;} + .d2-2154956904 .fill-AB4{fill:#EA99C6;} + .d2-2154956904 .fill-AB5{fill:#FFDEF1;} + .d2-2154956904 .stroke-N1{stroke:#0A0F25;} + .d2-2154956904 .stroke-N2{stroke:#676C7E;} + .d2-2154956904 .stroke-N3{stroke:#9499AB;} + .d2-2154956904 .stroke-N4{stroke:#CFD2DD;} + .d2-2154956904 .stroke-N5{stroke:#DEE1EB;} + .d2-2154956904 .stroke-N6{stroke:#EEF1F8;} + .d2-2154956904 .stroke-N7{stroke:#FFFFFF;} + .d2-2154956904 .stroke-B1{stroke:#000536;} + .d2-2154956904 .stroke-B2{stroke:#0F66B7;} + .d2-2154956904 .stroke-B3{stroke:#4393DD;} + .d2-2154956904 .stroke-B4{stroke:#87BFF3;} + .d2-2154956904 .stroke-B5{stroke:#BCDDFB;} + .d2-2154956904 .stroke-B6{stroke:#E5F3FF;} + .d2-2154956904 .stroke-AA2{stroke:#7639C5;} + .d2-2154956904 .stroke-AA4{stroke:#C1A2F3;} + .d2-2154956904 .stroke-AA5{stroke:#DACEFB;} + .d2-2154956904 .stroke-AB4{stroke:#EA99C6;} + .d2-2154956904 .stroke-AB5{stroke:#FFDEF1;} + .d2-2154956904 .background-color-N1{background-color:#0A0F25;} + .d2-2154956904 .background-color-N2{background-color:#676C7E;} + .d2-2154956904 .background-color-N3{background-color:#9499AB;} + .d2-2154956904 .background-color-N4{background-color:#CFD2DD;} + .d2-2154956904 .background-color-N5{background-color:#DEE1EB;} + .d2-2154956904 .background-color-N6{background-color:#EEF1F8;} + .d2-2154956904 .background-color-N7{background-color:#FFFFFF;} + .d2-2154956904 .background-color-B1{background-color:#000536;} + .d2-2154956904 .background-color-B2{background-color:#0F66B7;} + .d2-2154956904 .background-color-B3{background-color:#4393DD;} + .d2-2154956904 .background-color-B4{background-color:#87BFF3;} + .d2-2154956904 .background-color-B5{background-color:#BCDDFB;} + .d2-2154956904 .background-color-B6{background-color:#E5F3FF;} + .d2-2154956904 .background-color-AA2{background-color:#7639C5;} + .d2-2154956904 .background-color-AA4{background-color:#C1A2F3;} + .d2-2154956904 .background-color-AA5{background-color:#DACEFB;} + .d2-2154956904 .background-color-AB4{background-color:#EA99C6;} + .d2-2154956904 .background-color-AB5{background-color:#FFDEF1;} + .d2-2154956904 .color-N1{color:#0A0F25;} + .d2-2154956904 .color-N2{color:#676C7E;} + .d2-2154956904 .color-N3{color:#9499AB;} + .d2-2154956904 .color-N4{color:#CFD2DD;} + .d2-2154956904 .color-N5{color:#DEE1EB;} + .d2-2154956904 .color-N6{color:#EEF1F8;} + .d2-2154956904 .color-N7{color:#FFFFFF;} + .d2-2154956904 .color-B1{color:#000536;} + .d2-2154956904 .color-B2{color:#0F66B7;} + .d2-2154956904 .color-B3{color:#4393DD;} + .d2-2154956904 .color-B4{color:#87BFF3;} + .d2-2154956904 .color-B5{color:#BCDDFB;} + .d2-2154956904 .color-B6{color:#E5F3FF;} + .d2-2154956904 .color-AA2{color:#7639C5;} + .d2-2154956904 .color-AA4{color:#C1A2F3;} + .d2-2154956904 .color-AA5{color:#DACEFB;} + .d2-2154956904 .color-AB4{color:#EA99C6;} + .d2-2154956904 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2154956904);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2154956904);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2154956904);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2154956904);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2154956904);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2154956904);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2154956904);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2154956904);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2154956904);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2154956904);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2154956904);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2154956904);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>WritersDurable WAL + commit workerrequest acknowledged after fsyncManifest + Parquet + hot span indexstorage.data_dir/telemetryAuthoritative telemetry and its crash-recovery state.DuckDB query statestorage.data_dir/queryRebuildable rollups and temp spill. Not the telemetry itself.Control SQLitestorage.data_dir/control/fanout.sqliteApplication state. Never on the telemetry write path.users, user_identities, verifications, sessions, auth_audit_eventsoauth_clients, oauth_tokens, oauth_authorization_codesdashboards, dashboard_widgets, dashboard_stateagui_threads, agui_runs, alert_rules, alerts, settingsOTLP requestspans, logs, metricsRollupsservice, endpoint, edgeMerge and maintenance asynchronous publication DuckDB scans Parquet Authoritative telemetry and its crash-recovery state. + - + -Catalog attachment and temp spill. Not the telemetry itself. - +Rebuildable rollups and temp spill. Not the telemetry itself. + - + Application state. Never on the telemetry write path. - + - + - - - - + + + + diff --git a/docs/operations.md b/docs/operations.md index 79a6c80a..f508bd76 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -67,14 +67,14 @@ its configured cycle rather than immediately when the setting changes. ## Backup The supported portable baseline is a **cold backup** of the complete -`FANOUT_DATA_DIR`. It contains the telemetry catalog and Parquet files, query -state, and the control SQLite database; copying only one subdirectory does not -produce a recoverable installation. +`FANOUT_DATA_DIR`. It contains the telemetry WAL, commit manifest, Parquet +files, query state, and the control SQLite database; copying only one +subdirectory does not produce a recoverable installation. 1. Record the running Fanout version and configuration, excluding secrets from ordinary logs or tickets. 2. Stop Fanout cleanly and wait for the process to exit. Shutdown stops both - OTLP listeners before draining the lake writer. + OTLP listeners before draining the telemetry commit worker. 3. Snapshot or copy the complete data directory with ownership and permissions preserved. 4. Restart Fanout and confirm `/readyz`. diff --git a/docs/storage-architecture-options.md b/docs/storage-architecture-options.md index 5e859946..2eff0234 100644 --- a/docs/storage-architecture-options.md +++ b/docs/storage-architecture-options.md @@ -67,7 +67,7 @@ Important distinctions: ## Measured result -The normalized POC used one million complete Fanout-shaped spans, 50,000-row +The normalized benchmark used one million complete Fanout-shaped spans, 50,000-row commits, live endpoint rollups, complete trace reads, and another 200,000 rows under concurrent trace load at 100 queries per second. @@ -92,7 +92,7 @@ The production-repository row includes the real atomic WAL + hot-segment + Parquet commit path and was rerun on 2026-08-26. The isolated rows measure each engine separately. These are development measurements from an Apple M3 Max, not published capacity claims. The detailed methodology and reproduction commands are in -[storage-poc.md](storage-poc.md). +[storage-benchmark.md](storage-benchmark.md). ## Options at a glance @@ -134,7 +134,7 @@ not published capacity claims. The detailed methodology and reproduction command compaction correctness. - Hot and cold data use different physical formats. - Product queries spanning hot and cold data must merge two result streams. -- The current POC's broad scan is much slower than DuckDB. +- The current benchmark's broad scan is much slower than DuckDB. - Further promoted-attribute indexes and long-run compaction tuning remain workload-driven optimizations. @@ -162,7 +162,7 @@ SQL and interoperable cold storage to established components. ### Costs and risks - Ingestion was approximately five times slower than the custom hot store in - the full-shape POC. + the full-shape benchmark. - Peak RSS was much higher in the isolated comparison. - Scheduled rollup work remains outside ingestion. - Native files are not an interoperable telemetry format. @@ -283,7 +283,7 @@ requirements exist. ### Decision **Not selected.** It is a credible one-engine architecture, but the normalized -POC no longer justifies its footprint and binding complexity for Fanout. +benchmark no longer justifies its footprint and binding complexity for Fanout. ## Option F: fully custom database @@ -298,7 +298,7 @@ and transaction system. ### Why not -- The POC already demonstrates that custom **storage and fixed execution** +- The benchmark already demonstrates that custom **storage and fixed execution** provide most of the useful advantage. - Building general SQL would duplicate years of DuckDB work. - Correct recovery, concurrency, query planning, joins, spilling, and schema diff --git a/docs/storage-poc.md b/docs/storage-benchmark.md similarity index 58% rename from docs/storage-poc.md rename to docs/storage-benchmark.md index 8f089e70..94572b7a 100644 --- a/docs/storage-poc.md +++ b/docs/storage-benchmark.md @@ -1,6 +1,6 @@ -# Fanout-native storage POC +# Fanout storage benchmark -This experiment asks whether a storage path designed only for Fanout's +This benchmark asks whether a storage path designed only for Fanout's telemetry workload can outperform embedded general-purpose databases while remaining crash-safe and retaining a path to ad-hoc SQL. @@ -39,16 +39,17 @@ Collected on Darwin/arm64, Apple M3 Max, 14 logical CPUs. This is a development comparison, not a published Fanout capacity claim. Peak RSS was measured in an isolated process for each embedded engine. -| Storage / execution | Write rows/s | Maintenance | Active disk | Endpoint | Full trace | Raw service | Mixed write | Mixed trace p95 | Peak RSS | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Fanout columnar + direct | **520,879** | **135 ms** | 34.4 MiB | **0.224 ms** | **0.507 ms** | 26.98 ms | **528,668/s** | **1.33 ms** | **197 MiB** | -| DuckDB native | 98,030 | 274 ms rollup + 4 ms checkpoint | 47.5 MiB | 0.905 ms | 1.54 ms | **1.44 ms** | 85,514/s | 2.14 ms | 1,693 MiB | -| Zstd Parquet + DuckDB | 94,824 effective | 345 ms export | **21.8 MiB** | 1.35 ms | 10.53 ms | 3.88 ms | n/a | n/a | included in DuckDB process | -| chDB MergeTree | 129,724 | 4.13 s optimize | 38.1 MiB active | 2.81 ms | 7.39 ms | 6.06 ms | 118,722/s | 9.61 ms | 699 MiB | +| Storage / execution | Write rows/s | Maintenance | Active disk | Endpoint | Full trace | Raw service | Mixed write | Mixed trace p95 | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| **Production Fanout + Parquet** | 154,848 | live | 56.6 MiB | **0.171 ms** | **0.508 ms** | 27.42 ms | 156,547/s | 0.864 ms | +| Fanout columnar experiment | **511,113** | **94 ms** | 34.4 MiB | 0.210 ms | 0.551 ms | 27.99 ms | **513,927/s** | **0.816 ms** | +| DuckDB native | 93,189 | 143 ms rollup + 5 ms checkpoint | 47.5 MiB | 0.899 ms | 1.52 ms | **1.16 ms** | 80,255/s | 2.02 ms | +| Zstd Parquet + DuckDB | 90,215 effective | 354 ms export | **21.8 MiB** | 1.35 ms | 9.40 ms | 3.64 ms | n/a | n/a | +| chDB MergeTree | 78,434 | 4.64 s optimize | 38.1 MiB active | 3.47 ms | 9.47 ms | 6.88 ms | 97,271/s | 12.38 ms | The chDB directory occupied 106.7 MiB after forced merges because inactive and engine-internal files remain present; the table's active parts occupied 38.1 -MiB. Its embedded-engine initialization took 412 ms in the measured run. +MiB. Its embedded-engine initialization took 747 ms in the measured run. Iceberg is not listed as an execution engine. Its data plane is Parquet; table metadata, snapshots, manifests, deletion vectors, and planning would sit above @@ -56,32 +57,32 @@ the Parquet/DuckDB result and add capabilities plus some overhead. ## Interpretation -The custom path wins Fanout's fixed ingestion, endpoint, trace, concurrency, -maintenance, and memory objectives. DuckDB remains about 19 times faster for -the broad raw aggregation, and Parquet remains about 37% smaller than the -custom durable format. +The custom span experiment establishes the upper bound behind the earlier +roughly 500k rows/s figure. It is not the production write rate: it omits the +authoritative Parquet projection for logs and metrics and is intentionally not +a general SQL store. -This supports a hybrid architecture rather than a home-grown general SQL +The production design keeps the useful parts without taking on a home-grown database: -- Fanout owns the hot WAL/manifest, columnar segments, indexes, retention, - compaction, and ingestion-time rollups; -- known product queries use direct vectorized execution; -- cold segments use Parquet when interoperability and density matter; -- an established vectorized SQL engine handles arbitrary scans over cold data; -- SQLite remains control/configuration storage only. +- Fanout owns request-level WAL durability, a compact commit journal, the + recent-span index, retention, and compaction; +- Parquet is the single authoritative format for spans, logs, and metrics; +- DuckDB executes SQL, filtering, ordering, and broad analytical scans; +- SQLite stores transactional control and identity state only. -Before production replacement, the POC still needs logs and metrics, promoted -attribute indexes, retention under active readers, corruption checksums, -bounded-memory multi-day compaction, Linux 4-vCPU/8-GB measurements, and a -long-running kill/restart soak. +This trades some maximum write throughput for much lower implementation risk, +full telemetry coverage, standard files, and a featureful SQL engine. The +benchmark reports direct durable publication throughput; request acknowledgement +is decoupled through the WAL and should be measured separately under the target +collector concurrency and hardware before publishing a capacity claim. ## Reproduction Custom, DuckDB, and Parquet: ```sh -go run ./cmd/storage-poc \ +go run ./bench/storage \ -rows 1000000 \ -batch 50000 \ -repeats 11 \ @@ -94,6 +95,6 @@ chDB is a nested experiment module so its embedded C++ library does not enter Fanout's production dependency graph or binary: ```sh -cd experiments/storage-poc-chdb +cd bench/storage/chdb go run . -rows 1000000 -batch 50000 -repeats 11 -mixed-rows 200000 ``` diff --git a/experiments/storage-poc-chdb/go.mod b/experiments/storage-poc-chdb/go.mod deleted file mode 100644 index 640adf23..00000000 --- a/experiments/storage-poc-chdb/go.mod +++ /dev/null @@ -1,23 +0,0 @@ -module github.com/labstack/fanout/experiments/storage-poc-chdb - -go 1.27.0 - -require ( - github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 - github.com/chdb-io/chdb-go/v2 v2.1.0 - github.com/labstack/fanout v0.0.0 -) - -require ( - github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 // indirect - github.com/ebitengine/purego v0.8.2 // indirect - github.com/klauspost/compress v1.19.2 // indirect - github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect - golang.org/x/sys v0.47.0 // indirect -) - -replace github.com/labstack/fanout => ../.. diff --git a/experiments/storage-poc-chdb/go.sum b/experiments/storage-poc-chdb/go.sum deleted file mode 100644 index fddb0c32..00000000 --- a/experiments/storage-poc-chdb/go.sum +++ /dev/null @@ -1,24 +0,0 @@ -github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 h1:QiXQ1vZWcQCbRpciirWG/+F3KRXYnDLiFOVYkAJxCls= -github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1/go.mod h1:jB7U0oct7fDV+SbrDzh3oorQFAQ8YOM5QFXf273ZEEs= -github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 h1:qvkIS/fozvgJfd30MEPM1nVDSM1JMXQgqZfqp1Oh7aQ= -github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1/go.mod h1:tebe6DiYx113PoHD0WjWCWVY74QDmaPcCdWA+aNpWPM= -github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 h1:FBSXH0ChVm7fOEHUWpf3bNd2BKnp1cv9LhEn1eLcUdE= -github.com/chdb-io/chdb-go/lib/embedded v0.260700.1/go.mod h1:N9Dra/RfDuELfnT2TSMVBF+NzyDKL6AFiDqHwCWF7T0= -github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 h1:w1q1whc7LlBpJtzoMkokgnSsdRrFLAG2kgRkb1tQ7jM= -github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1/go.mod h1:LK3ORN5rYtQDUYKswMZKf09RT1qIdwlCGk/3/vB7aHE= -github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 h1:GoviqHKnVOJIfnfgiSYWiEqxciThy2XGmSHUdq1qvmI= -github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1/go.mod h1:vkAVOjzg+j6TwFWobfxvmU+pV8fD7dS9ibBdZc3Wf8Y= -github.com/chdb-io/chdb-go/v2 v2.1.0 h1:Nf/StmYfE90mePp0EzdWqCbqUv/TJUbVOrnea/x3PN0= -github.com/chdb-io/chdb-go/v2 v2.1.0/go.mod h1:tyiHoF8pWUfrD7ylseofFEnELnA+jocf/yElE3AHBaQ= -github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= -github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= -github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/fanout.example.yaml b/fanout.example.yaml index 27441770..7eebd4e4 100644 --- a/fanout.example.yaml +++ b/fanout.example.yaml @@ -21,8 +21,7 @@ ingest: otlp_http_addr: "127.0.0.1:4318" # FANOUT_OTLP_HTTP_ADDR advertised_endpoint: "" # FANOUT_INGEST_ADVERTISED_ENDPOINT default_namespace: default # FANOUT_DEFAULT_NAMESPACE - flush_interval: 15s # FANOUT_FLUSH_INTERVAL - flush_batch_size: 50000 # FANOUT_FLUSH_BATCH_SIZE + batch_size: 50000 # FANOUT_INGEST_BATCH_SIZE storage: data_dir: ./data # FANOUT_DATA_DIR diff --git a/internal/api/health.go b/internal/api/health.go index 73bbb973..a29bd054 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -190,7 +190,7 @@ func (h *HealthHandler) checkTelemetry() CheckResult { defer cancel() var one int - err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM lake.spans LIMIT 1") + err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM telemetry.spans LIMIT 1") if err != nil && err != sql.ErrNoRows { return CheckResult{ Status: "unhealthy", diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 07f17c49..5da7e158 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -125,7 +125,7 @@ func TestReadiness_HealthyTelemetryAndRollups(t *testing.T) { mock.ExpectQuery("SELECT 1"). WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) - mock.ExpectQuery("SELECT 1 FROM lake.spans LIMIT 1"). + mock.ExpectQuery("SELECT 1 FROM telemetry.spans LIMIT 1"). WillReturnRows(sqlmock.NewRows([]string{"1"})) mock.ExpectQuery("SELECT\\s+MAX\\(updated_at\\),\\s+COUNT\\(\\*\\),\\s+COALESCE\\(date_diff\\('second', MAX\\(updated_at\\), now\\(\\)\\), 0\\)"). WillReturnRows(sqlmock.NewRows([]string{"max", "count", "age_seconds"}).AddRow(time.Now().UTC(), 2, int64(30))) diff --git a/internal/config/config.go b/internal/config/config.go index 0915d35f..d1615760 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,8 @@ import ( appauth "github.com/labstack/fanout/internal/auth" ) +const maxIngestBatchSize = 50_000 + // Config is Fanout's canonical configuration schema. Public names use // FANOUT_ plus the shortest stable term that is clear in a docker run, and the // same terminology across YAML, environment variables, Go, logs, and docs. @@ -27,8 +29,7 @@ type Config struct { // derive host:port from the browser request and OTLPGRPCAddr as a best effort. IngestAdvertisedEndpoint string `koanf:"ingest.advertised_endpoint" env:"FANOUT_INGEST_ADVERTISED_ENDPOINT"` DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` - FlushInterval time.Duration `koanf:"ingest.flush_interval" env:"FANOUT_FLUSH_INTERVAL" default:"15s"` - FlushBatchSize int `koanf:"ingest.flush_batch_size" env:"FANOUT_FLUSH_BATCH_SIZE" default:"50000"` + IngestBatchSize int `koanf:"ingest.batch_size" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` @@ -182,11 +183,8 @@ func (c Config) Validate() error { if strings.TrimSpace(c.DataDir) == "" { return fmt.Errorf("storage.data_dir must not be empty") } - if c.FlushInterval < time.Second { - return fmt.Errorf("ingest.flush_interval must be at least 1s, got %s", c.FlushInterval) - } - if c.FlushBatchSize <= 0 { - return fmt.Errorf("ingest.flush_batch_size must be > 0, got %d", c.FlushBatchSize) + if c.IngestBatchSize <= 0 || c.IngestBatchSize > maxIngestBatchSize { + return fmt.Errorf("ingest.batch_size must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) } if c.RollupInterval < time.Second { return fmt.Errorf("storage.rollup_interval must be at least 1s, got %s", c.RollupInterval) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a75decae..e574b2b7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -42,11 +42,8 @@ func TestLoadReturnsDefaults(t *testing.T) { if cfg.DataDir != "./data" { t.Errorf("DataDir = %q, want %q", cfg.DataDir, "./data") } - if cfg.FlushInterval != 15*time.Second { - t.Errorf("FlushInterval = %s, want %s", cfg.FlushInterval, 15*time.Second) - } - if cfg.FlushBatchSize != 50000 { - t.Errorf("FlushBatchSize = %d, want %d", cfg.FlushBatchSize, 50000) + if cfg.IngestBatchSize != 50000 { + t.Errorf("IngestBatchSize = %d, want %d", cfg.IngestBatchSize, 50000) } if cfg.RollupInterval != time.Minute { t.Errorf("RollupInterval = %s, want %s", cfg.RollupInterval, time.Minute) @@ -280,9 +277,7 @@ func TestConfigurationSchemaUsesUnitBearingDurations(t *testing.T) { func TestLoadDurationIntervals(t *testing.T) { t.Run("YAML", func(t *testing.T) { path := filepath.Join(t.TempDir(), "fanout.yaml") - document := `ingest: - flush_interval: 30s -storage: + document := `storage: rollup_interval: 5m maintenance_interval: 1h alerts: @@ -295,7 +290,7 @@ alerts: if err != nil { t.Fatalf("Load: %v", err) } - if cfg.FlushInterval != 30*time.Second || cfg.RollupInterval != 5*time.Minute || + if cfg.RollupInterval != 5*time.Minute || cfg.MaintenanceInterval != time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { t.Fatalf("duration values were not decoded: %+v", cfg) @@ -304,7 +299,6 @@ alerts: t.Run("environment", func(t *testing.T) { cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_FLUSH_INTERVAL=30s", "FANOUT_ROLLUP_INTERVAL=5m", "FANOUT_MAINTENANCE_INTERVAL=1h", "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", @@ -312,7 +306,7 @@ alerts: if err != nil { t.Fatalf("Load: %v", err) } - if cfg.FlushInterval != 30*time.Second || cfg.RollupInterval != 5*time.Minute || + if cfg.RollupInterval != 5*time.Minute || cfg.MaintenanceInterval != time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { t.Fatalf("duration values were not decoded: %+v", cfg) @@ -398,7 +392,6 @@ func TestLoadRejectsUnknownInputs(t *testing.T) { t.Run("removed environment variables", func(t *testing.T) { for _, name := range []string{ - "FANOUT_FLUSH_SECONDS", "FANOUT_ROLLUP_EVERY_SECONDS", "FANOUT_MERGE_EVERY_SECONDS", "FANOUT_MERGE_INTERVAL", @@ -499,7 +492,7 @@ func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { for _, test := range []struct { name, document string }{ - {"invalid duration", "ingest:\n flush_interval: never\n"}, + {"invalid duration", "storage:\n rollup_interval: never\n"}, {"fractional integer", "smtp:\n port: 25.9\n"}, {"integer as boolean", "mcp:\n enabled: 2\n"}, {"YAML keyword as boolean", "alerts:\n enabled: off\n"}, @@ -517,8 +510,8 @@ func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { } t.Run("invalid merged config", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_FLUSH_INTERVAL=0s")}) - if err == nil || !strings.Contains(err.Error(), "flush") { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_INGEST_BATCH_SIZE=0")}) + if err == nil || !strings.Contains(err.Error(), "batch_size") { t.Fatalf("error = %v, want validation error", err) } }) @@ -553,7 +546,6 @@ func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { for _, test := range []struct { name, assignment, key string }{ - {"subsecond flush interval", "FANOUT_FLUSH_INTERVAL=999ms", "flush_interval"}, {"subsecond rollup interval", "FANOUT_ROLLUP_INTERVAL=999ms", "rollup_interval"}, {"subsecond maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=500ms", "maintenance_interval"}, {"subsecond alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=999ms", "evaluation_interval"}, @@ -616,8 +608,7 @@ func TestValidate(t *testing.T) { OTLPGRPCAddr: "127.0.0.1:4317", OTLPHTTPAddr: "127.0.0.1:4318", DataDir: "./data", - FlushInterval: 15 * time.Second, - FlushBatchSize: 50000, + IngestBatchSize: 50000, RollupInterval: time.Minute, RetentionDays: 30, HotRetention: 24 * time.Hour, @@ -645,10 +636,9 @@ func TestValidate(t *testing.T) { name string modify func(*Config) }{ - {"FlushInterval=0", func(c *Config) { c.FlushInterval = 0 }}, - {"FlushInterval=999ms", func(c *Config) { c.FlushInterval = 999 * time.Millisecond }}, - {"FlushBatchSize=0", func(c *Config) { c.FlushBatchSize = 0 }}, - {"FlushBatchSize=-1", func(c *Config) { c.FlushBatchSize = -1 }}, + {"IngestBatchSize=0", func(c *Config) { c.IngestBatchSize = 0 }}, + {"IngestBatchSize=-1", func(c *Config) { c.IngestBatchSize = -1 }}, + {"IngestBatchSize=50001", func(c *Config) { c.IngestBatchSize = 50001 }}, {"RollupInterval=0", func(c *Config) { c.RollupInterval = 0 }}, {"RollupInterval=999ms", func(c *Config) { c.RollupInterval = 999 * time.Millisecond }}, {"RetentionDays=-1", func(c *Config) { c.RetentionDays = -1 }}, diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 3bfc9010..2ea313d2 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -168,10 +168,10 @@ func TestRecordTelemetryOperationOutcomes(t *testing.T) { func TestBoundedMetricLabelsRejectUnknownValues(t *testing.T) { for name, call := range map[string]func(){ - "rollup component": func() { RecordRollupComponent(RollupComponent("tenant"), RollupSuccess, 0, 0) }, - "rollup result": func() { RecordRollupComponent(RollupService, RollupResult("unknown"), 0, 0) }, - "lake operation": func() { RecordTelemetryOperation(TelemetryOperation("query"), TelemetrySuccess, 0) }, - "lake result": func() { RecordTelemetryOperation(TelemetryCompaction, TelemetryResult("unknown"), 0) }, + "rollup component": func() { RecordRollupComponent(RollupComponent("tenant"), RollupSuccess, 0, 0) }, + "rollup result": func() { RecordRollupComponent(RollupService, RollupResult("unknown"), 0, 0) }, + "telemetry operation": func() { RecordTelemetryOperation(TelemetryOperation("query"), TelemetrySuccess, 0) }, + "telemetry result": func() { RecordTelemetryOperation(TelemetryCompaction, TelemetryResult("unknown"), 0) }, } { t.Run(name, func(t *testing.T) { defer func() { diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 5c50428f..3ea3c68a 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -1,88 +1,33 @@ package observability import ( - "container/heap" "context" "fmt" - "sort" "strings" - "time" - - "github.com/labstack/fanout/internal/telemetry" ) -// newestLogHeap keeps its oldest entry at the root so a full-window scan only -// retains the newest bounded result set. -type newestLogHeap []LogEntry - -func (h newestLogHeap) Len() int { return len(h) } -func (h newestLogHeap) Less(i, j int) bool { return h[i].Time.Before(h[j].Time) } -func (h newestLogHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *newestLogHeap) Push(value any) { *h = append(*h, value.(LogEntry)) } -func (h *newestLogHeap) Pop() any { - old := *h - value := old[len(old)-1] - *h = old[:len(old)-1] - return value -} - -// earliestLogHeap keeps its newest entry at the root so trace correlation can -// retain the earliest bounded result set even when batches arrive out of order. -type earliestLogHeap []LogEntry - -func (h earliestLogHeap) Len() int { return len(h) } -func (h earliestLogHeap) Less(i, j int) bool { return h[i].Time.After(h[j].Time) } -func (h earliestLogHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *earliestLogHeap) Push(value any) { *h = append(*h, value.(LogEntry)) } -func (h *earliestLogHeap) Pop() any { - old := *h - value := old[len(old)-1] - *h = old[:len(old)-1] - return value -} - -func retainNewest(entries *newestLogHeap, entry LogEntry, limit int) { - if entries.Len() < limit { - heap.Push(entries, entry) - } else if entry.Time.After((*entries)[0].Time) { - (*entries)[0] = entry - heap.Fix(entries, 0) - } -} - -func retainEarliest(entries *earliestLogHeap, entry LogEntry, limit int) { - if entries.Len() < limit { - heap.Push(entries, entry) - } else if entry.Time.Before((*entries)[0].Time) { - (*entries)[0] = entry - heap.Fix(entries, 0) - } -} - -var coldLogFilters = ` +var logFilters = ` WHERE time >= ? AND time < ? AND (? = '' OR namespace = ?) AND (? = '' OR service = ?) AND (? = '' OR lower(severity) = lower(?)) AND (? = '' OR contains(lower(` + redactLogBodySQL("body") + `), lower(?)))` -// The cold tier answers the two questions the API actually asks: the newest -// `limit` entries, and per-bucket counts. Both stay bounded in DuckDB — the -// entry sample by LIMIT, the histogram by GROUP BY — so a wide window costs a -// page of rows instead of the whole retained window streamed through the -// driver. -var coldLogEntriesQuery = ` +// DuckDB answers the two questions the API asks: the newest `limit` entries +// and per-bucket counts. LIMIT bounds the entry stream and GROUP BY bounds the +// histogram stream regardless of the retained Parquet row count. +var logEntriesQuery = ` SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, coalesce(trace_id, ''), coalesce(span_id, '') -FROM logs` + coldLogFilters + ` + FROM logs` + logFilters + ` ORDER BY time DESC LIMIT ?` -var coldLogBucketsQuery = ` +var logBucketsQuery = ` SELECT time_bucket(INTERVAL '5 minutes', time) AS point_time, coalesce(nullif(upper(severity), ''), 'UNSPECIFIED') AS bucket_severity, CAST(count(*) AS BIGINT) -FROM logs` + coldLogFilters + ` + FROM logs` + logFilters + ` GROUP BY point_time, bucket_severity ORDER BY point_time ASC, bucket_severity ASC` @@ -98,113 +43,47 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear service, severity, search = strings.TrimSpace(service), strings.TrimSpace(severity), strings.TrimSpace(search) search = strings.ToLower(search) data := Logs{Entries: []LogEntry{}, Buckets: []LogBucket{}} - type bucketKey struct { - time int64 - severity string - } - buckets := make(map[bucketKey]int64) - entries := newestLogHeap{} - matched := 0 - accumulate := func(entry LogEntry) { - matched++ - retainNewest(&entries, entry, limit) - bucketSeverity := strings.ToUpper(entry.Severity) - if bucketSeverity == "" { - bucketSeverity = "UNSPECIFIED" - } - bucketNanos := entry.Time.Truncate(5 * time.Minute).UnixNano() - buckets[bucketKey{time: bucketNanos, severity: bucketSeverity}]++ - } - startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() - hotCutoff, err := s.repository.ScanHotLogs(startNanos, endNanos, func(row telemetry.Log) bool { - select { - case <-ctx.Done(): - return false - default: - } - if (scope.Namespace != "" && row.Namespace != scope.Namespace) || (service != "" && row.ServiceName != service) || (severity != "" && !strings.EqualFold(row.Severity, severity)) { - return true - } - body := redactLogBody(row.Body) - if search != "" && !strings.Contains(strings.ToLower(body), search) { - return true - } - entryTime := time.Unix(0, row.EventUnixNanos).UTC() - accumulate(LogEntry{Time: entryTime, Severity: row.Severity, Service: row.ServiceName, Body: body, TraceID: row.TraceID, SpanID: row.SpanID}) - return true - }) + filters := []any{scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, severity, severity, search, search} + rows, err := s.db.QueryContext(ctx, logEntriesQuery, append(append([]any{}, filters...), limit)...) if err != nil { - return Result[Logs]{}, fmt.Errorf("read log segments: %w", err) + return Result[Logs]{}, fmt.Errorf("query logs: %w", err) } - coldEnd := min(max(hotCutoff, startNanos), endNanos) - hotStart := max(hotCutoff, startNanos) - usedCold := coldEnd > startNanos - if usedCold { - coldStop := time.Unix(0, coldEnd).UTC() - filters := []any{scope.Start, coldStop, scope.Namespace, scope.Namespace, service, service, severity, severity, search, search} - rows, queryErr := s.db.QueryContext(ctx, coldLogEntriesQuery, append(append([]any{}, filters...), limit)...) - if queryErr != nil { - return Result[Logs]{}, fmt.Errorf("query cold logs: %w", queryErr) - } - for rows.Next() { - var entry LogEntry - if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { - rows.Close() - return Result[Logs]{}, fmt.Errorf("scan cold log: %w", err) - } - retainNewest(&entries, entry, limit) - } - if err := rows.Err(); err != nil { + for rows.Next() { + var entry LogEntry + if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { rows.Close() - return Result[Logs]{}, fmt.Errorf("iterate cold logs: %w", err) + return Result[Logs]{}, fmt.Errorf("scan log: %w", err) } + data.Entries = append(data.Entries, entry) + } + if err := rows.Err(); err != nil { rows.Close() + return Result[Logs]{}, fmt.Errorf("iterate logs: %w", err) + } + rows.Close() - bucketRows, bucketErr := s.db.QueryContext(ctx, coldLogBucketsQuery, filters...) - if bucketErr != nil { - return Result[Logs]{}, fmt.Errorf("query cold log histogram: %w", bucketErr) - } - for bucketRows.Next() { - var ( - bucketTime time.Time - bucketSeverity string - count int64 - ) - if err := bucketRows.Scan(&bucketTime, &bucketSeverity, &count); err != nil { - bucketRows.Close() - return Result[Logs]{}, fmt.Errorf("scan cold log bucket: %w", err) - } - matched += int(count) - buckets[bucketKey{time: bucketTime.UTC().UnixNano(), severity: bucketSeverity}] += count - } - if err := bucketRows.Err(); err != nil { + matched := 0 + bucketRows, err := s.db.QueryContext(ctx, logBucketsQuery, filters...) + if err != nil { + return Result[Logs]{}, fmt.Errorf("query log histogram: %w", err) + } + for bucketRows.Next() { + var bucket LogBucket + if err := bucketRows.Scan(&bucket.Time, &bucket.Severity, &bucket.Count); err != nil { bucketRows.Close() - return Result[Logs]{}, fmt.Errorf("iterate cold log histogram: %w", err) + return Result[Logs]{}, fmt.Errorf("scan log bucket: %w", err) } - bucketRows.Close() - } - if err := ctx.Err(); err != nil { - return Result[Logs]{}, err - } - data.Entries = append(data.Entries, entries...) - sort.Slice(data.Entries, func(i, j int) bool { return data.Entries[i].Time.After(data.Entries[j].Time) }) - for key, count := range buckets { - data.Buckets = append(data.Buckets, LogBucket{Time: time.Unix(0, key.time).UTC(), Severity: key.severity, Count: count}) + bucket.Time = bucket.Time.UTC() + matched += int(bucket.Count) + data.Buckets = append(data.Buckets, bucket) } - sort.Slice(data.Buckets, func(i, j int) bool { - if data.Buckets[i].Time.Equal(data.Buckets[j].Time) { - return data.Buckets[i].Severity < data.Buckets[j].Severity - } - return data.Buckets[i].Time.Before(data.Buckets[j].Time) - }) - dataSource := "fanout_segments" - if usedCold && hotStart < endNanos { - dataSource = "fanout_segments+parquet" - } else if usedCold { - dataSource = "parquet" + if err := bucketRows.Err(); err != nil { + bucketRows.Close() + return Result[Logs]{}, fmt.Errorf("iterate log histogram: %w", err) } + bucketRows.Close() return Result[Logs]{ Schema: LogsSchema, Summary: fmt.Sprintf("%d logs matched the selected telemetry window", matched), - Data: data, Provenance: s.provenanceFor(scope, dataSource), + Data: data, Provenance: s.provenanceFor(scope, "parquet"), }, nil } diff --git a/internal/observability/performance_benchmark_test.go b/internal/observability/performance_benchmark_test.go index 898e31d2..b1ab6b26 100644 --- a/internal/observability/performance_benchmark_test.go +++ b/internal/observability/performance_benchmark_test.go @@ -26,7 +26,7 @@ func BenchmarkEndpointQueries24Hours(b *testing.B) { b.Fatal(err) } start := time.Date(2026, 7, 20, 0, 0, 30, 0, time.UTC) - if _, err := db.Exec(`INSERT INTO lake.spans ( + if _, err := db.Exec(`INSERT INTO telemetry.spans ( namespace, service, start_time, duration_ms, status, http_method, http_route, operation ) SELECT diff --git a/internal/observability/performance_rollup_test.go b/internal/observability/performance_rollup_test.go index 0123ad67..8d56837f 100644 --- a/internal/observability/performance_rollup_test.go +++ b/internal/observability/performance_rollup_test.go @@ -62,7 +62,7 @@ FROM (VALUES ` + seed.values + `) t(ms)` // Include raw rows for every minute. The query must use raw rows for the two // partial boundaries and for complete minutes newer than the watermark, while // excluding raw rows already represented by mature cached buckets. - if _, err := db.Exec(`INSERT INTO lake.spans ( + if _, err := db.Exec(`INSERT INTO telemetry.spans ( namespace, service, start_time, duration_ms, status, http_method, http_route, operation ) VALUES ('prod','checkout',?,5.0,'OK','GET','/pay','GET /pay'), diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index d69a03ba..7bb2d629 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -226,6 +226,11 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { }}); err != nil { t.Fatalf("commit trace fixture: %v", err) } + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("trace-1", start, end, "prod", "prod", 20). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(150*time.Millisecond), "ERROR", "checkout", "payment declined", "trace-1", "root"). + AddRow(start.Add(160*time.Millisecond), "ERROR", "payments", `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, "trace-1", "child")) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "checkout", 20) if err != nil { @@ -256,6 +261,15 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { }}); err != nil { t.Fatalf("commit logs fixture: %v", err) } + mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "declined", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(2*time.Millisecond), "ERROR", "checkout", `auth declined: {"password":"[REDACTED]"}`, "trace-3", "root3"). + AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "card declined: token=[REDACTED]", "trace-2", "root2"). + AddRow(start, "ERROR", "checkout", "payment declined", "trace-1", "root")) + mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "declined"). + WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}).AddRow(start, "ERROR", int64(3))) result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "checkout", "error", "declined", 10) if err != nil { @@ -276,7 +290,7 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { } func TestLogsRetainsOnlyNewestLimit(t *testing.T) { - svc, _ := newMockService(t) + svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) logs := make([]telemetry.Log, 100) for i := range logs { @@ -285,6 +299,16 @@ func TestLogsRetainsOnlyNewestLimit(t *testing.T) { if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { t.Fatal(err) } + entryRows := sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}) + for i := 99; i >= 95; i-- { + entryRows.AddRow(start.Add(time.Duration(i)*time.Millisecond), "INFO", "", "entry", "", "") + } + mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). + WithArgs(start, start.Add(time.Hour), "prod", "prod", "", "", "", "", "", "", 5). + WillReturnRows(entryRows) + mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). + WithArgs(start, start.Add(time.Hour), "prod", "prod", "", "", "", "", "", ""). + WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}).AddRow(start, "INFO", int64(100))) result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "", "", "", 5) if err != nil { t.Fatal(err) @@ -294,24 +318,24 @@ func TestLogsRetainsOnlyNewestLimit(t *testing.T) { } } -func TestLogsFallsBackToParquetOutsideHotRetention(t *testing.T) { +func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "cold-logs", Logs: []telemetry.Log{{ + if err := svc.repository.Commit(telemetrystore.Batch{ID: "parquet-logs", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", - ServiceName: "checkout", Body: "token=secret", TraceID: "trace-cold", + ServiceName: "checkout", Body: "token=secret", TraceID: "trace-parquet", }}}); err != nil { t.Fatal(err) } if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { t.Fatal(err) } - mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). + mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(time.Minute), "ERROR", "checkout", "token=[REDACTED]", "trace-cold", "")) - mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). + AddRow(start.Add(time.Minute), "ERROR", "checkout", "token=[REDACTED]", "trace-parquet", "")) + mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", ""). WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). AddRow(start, "ERROR", int64(1))) @@ -320,14 +344,14 @@ func TestLogsFallsBackToParquetOutsideHotRetention(t *testing.T) { t.Fatal(err) } if len(result.Data.Entries) != 1 || result.Data.Entries[0].Body != "token=[REDACTED]" || result.Provenance.DataSource != "parquet" { - t.Fatalf("cold logs result = %#v", result) + t.Fatalf("Parquet logs result = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) } } -func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { +func TestLogsAreIndependentOfSpanHotPruneBoundary(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(250 * time.Millisecond) @@ -347,16 +371,17 @@ func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { t.Fatal(err) } - mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). - WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", "", 10). + mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). + WithArgs(start, end, "prod", "prod", "", "", "", "", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(100*time.Millisecond), "INFO", "", "late-old", "", ""). + AddRow(start.Add(300*time.Millisecond), "INFO", "", "newer-hot", "", ""). + AddRow(start.Add(200*time.Millisecond), "INFO", "", "late-boundary", "", ""). AddRow(start.Add(150*time.Millisecond), "INFO", "", "newer-old", "", ""). - AddRow(start.Add(200*time.Millisecond), "INFO", "", "late-boundary", "", "")) - mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). - WithArgs(start, cutoff, "prod", "prod", "", "", "", "", "", ""). + AddRow(start.Add(100*time.Millisecond), "INFO", "", "late-old", "", "")) + mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). + WithArgs(start, end, "prod", "prod", "", "", "", "", "", ""). WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). - AddRow(start, "INFO", int64(3))) + AddRow(start, "INFO", int64(4))) result, err := svc.Logs(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "", "", "", 10) if err != nil { t.Fatal(err) @@ -364,7 +389,7 @@ func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { if len(result.Data.Entries) != 4 || result.Summary != "4 logs matched the selected telemetry window" { t.Fatalf("boundary logs = %#v", result) } - if result.Data.Entries[0].Body != "newer-hot" || result.Provenance.DataSource != "fanout_segments+parquet" { + if result.Data.Entries[0].Body != "newer-hot" || result.Provenance.DataSource != "parquet" { t.Fatalf("boundary result = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { @@ -372,8 +397,8 @@ func TestLogsUsesDurablePruneBoundaryWithOverlappingSegments(t *testing.T) { } } -func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { - svc, _ := newMockService(t) +func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { + svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) batches := []telemetrystore.Batch{ {ID: "trace-latest", Spans: []telemetry.Span{{Namespace: "prod", TraceID: "trace-order", SpanID: "root", StartUnixNanos: start.UnixNano(), DurationMS: 1}}, Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(30 * time.Millisecond).UnixNano(), Body: "latest"}}}, @@ -386,33 +411,40 @@ func TestTraceLogsUseEarliestEventTimeAcrossBatches(t *testing.T) { t.Fatal(err) } } + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("trace-order", start, start.Add(time.Hour), "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). + AddRow(start.Add(10*time.Millisecond), "", "", "earliest", "trace-order", ""). + AddRow(start.Add(20*time.Millisecond), "", "", "middle", "trace-order", ""). + AddRow(start.Add(30*time.Millisecond), "", "", "latest", "trace-order", ""). + AddRow(start.Add(2*time.Minute), "", "", "unrelated later event", "trace-order", "")) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: start.Add(time.Hour)}, "trace-order", "", 10) if err != nil { t.Fatal(err) } - if len(result.Data.Logs) != 3 || result.Data.Logs[0].Body != "earliest" || result.Data.Logs[1].Body != "middle" || result.Data.Logs[2].Body != "latest" { + if len(result.Data.Logs) != 4 || result.Data.Logs[0].Body != "earliest" || result.Data.Logs[3].Body != "unrelated later event" { t.Fatalf("trace logs = %#v", result.Data.Logs) } } -func TestTraceFallsBackToParquetWhenHotSegmentsMiss(t *testing.T) { +func TestTraceUsesParquetWhenHotSegmentsMiss(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). - WithArgs("cold-trace", start, end, "prod", "prod", 10). + WithArgs("parquet-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). AddRow("root", "", "checkout", "pay", "SERVER", start, 25.0, "ERROR", "declined")) mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("cold-trace", start, end, "prod", "prod", 10). + WithArgs("parquet-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "cold-trace", "root")) - result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "cold-trace", "", 10) + AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "parquet-trace", "root")) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "parquet-trace", "", 10) if err != nil { t.Fatal(err) } if len(result.Data.Spans) != 1 || len(result.Data.Logs) != 1 || !result.Data.HasError { - t.Fatalf("cold trace detail = %#v", result.Data) + t.Fatalf("Parquet trace detail = %#v", result.Data) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) @@ -461,13 +493,13 @@ func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { var _ DB = queryrows.SQLAdapter{} -func TestLogsBoundsColdQueryWithLimitAndAggregatedBuckets(t *testing.T) { +func TestLogsBoundsParquetQueryWithLimitAndAggregatedBuckets(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-cold", Logs: []telemetry.Log{{ + if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-parquet", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", - ServiceName: "checkout", Body: "hello", TraceID: "trace-cold", + ServiceName: "checkout", Body: "hello", TraceID: "trace-parquet", }}}); err != nil { t.Fatal(err) } @@ -475,15 +507,15 @@ func TestLogsBoundsColdQueryWithLimitAndAggregatedBuckets(t *testing.T) { t.Fatal(err) } // The sample query must carry the row limit so a wide window cannot stream - // the whole cold tier through the driver. - mock.ExpectQuery(regexp.QuoteMeta(coldLogEntriesQuery)). + // the whole Parquet history through the driver. + mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). WithArgs(start, end, "prod", "prod", "", "", "", "", "", "", 2). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(3*time.Minute), "ERROR", "checkout", "newest", "trace-c", ""). AddRow(start.Add(2*time.Minute), "INFO", "checkout", "older", "trace-b", "")) // Histogram counts come back aggregated, so a million matching rows cost // one row per bucket rather than a million transfers. - mock.ExpectQuery(regexp.QuoteMeta(coldLogBucketsQuery)). + mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). WithArgs(start, end, "prod", "prod", "", "", "", "", "", ""). WillReturnRows(sqlmock.NewRows([]string{"point_time", "severity", "count"}). AddRow(start, "ERROR", int64(900000)). diff --git a/internal/observability/trace.go b/internal/observability/trace.go index 497e42c1..ed55bafe 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -6,8 +6,6 @@ import ( "sort" "strings" "time" - - "github.com/labstack/fanout/internal/telemetry" ) const recentTraceQuery = ` @@ -66,7 +64,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} hotCutoff := int64(0) if traceID != "" { - storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID) + storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID, scope.Start.UnixNano()) if readErr != nil { return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } @@ -113,34 +111,6 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin } sort.Strings(data.Services) - traceLogs := earliestLogHeap{} - logStart, logEnd := startNanos, endNanos - if !first.IsZero() { - const correlationMargin = time.Second - logStart = max(logStart, first.Add(-correlationMargin).UnixNano()) - logEnd = min(logEnd, last.Add(correlationMargin).UnixNano()) - } - logCutoff, scanErr := s.repository.ScanHotLogs(logStart, logEnd, func(row telemetry.Log) bool { - select { - case <-ctx.Done(): - return false - default: - } - if row.TraceID != traceID || (scope.Namespace != "" && row.Namespace != scope.Namespace) { - return true - } - retainEarliest(&traceLogs, LogEntry{Time: time.Unix(0, row.EventUnixNanos).UTC(), Severity: row.Severity, Service: row.ServiceName, Body: redactLogBody(row.Body), TraceID: row.TraceID, SpanID: row.SpanID}, limit) - return true - }) - if scanErr != nil { - return Result[TraceDetail]{}, fmt.Errorf("read trace logs: %w", scanErr) - } - hotCutoff = max(hotCutoff, logCutoff) - if err := ctx.Err(); err != nil { - return Result[TraceDetail]{}, err - } - data.Logs = append(data.Logs, traceLogs...) - sort.Slice(data.Logs, func(i, j int) bool { return data.Logs[i].Time.Before(data.Logs[j].Time) }) } // Any scope crossing the durable hot prune watermark may contain early trace // spans that have aged out while a late suffix remains hot. Parquet is the @@ -152,6 +122,14 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{}, err } dataSource = "parquet" + } else if traceID != "" { + // Parquet is committed atomically with the hot span index. DuckDB can + // apply trace_id and LIMIT inside its vectorized scan, avoiding a full + // Go decode while preserving clock-skewed trace events. + data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) + if err != nil { + return Result[TraceDetail]{}, err + } } summary := "No traces found in this telemetry window" @@ -205,23 +183,32 @@ func (s *Service) traceFromParquet(ctx context.Context, scope Scope, traceID str } sort.Strings(data.Services) - rows, err = s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) + data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) + if err != nil { + return TraceDetail{}, err + } + return data, nil +} + +func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID string, limit int) ([]LogEntry, error) { + rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) if err != nil { - return TraceDetail{}, fmt.Errorf("query trace parquet logs: %w", err) + return nil, fmt.Errorf("query trace parquet logs: %w", err) } + logs := make([]LogEntry, 0, limit) for rows.Next() { var entry LogEntry if err := rows.Scan(&entry.Time, &entry.Severity, &entry.Service, &entry.Body, &entry.TraceID, &entry.SpanID); err != nil { rows.Close() - return TraceDetail{}, fmt.Errorf("scan trace parquet log: %w", err) + return nil, fmt.Errorf("scan trace parquet log: %w", err) } entry.Body = redactLogBody(entry.Body) - data.Logs = append(data.Logs, entry) + logs = append(logs, entry) } if err := rows.Err(); err != nil { rows.Close() - return TraceDetail{}, fmt.Errorf("iterate trace parquet logs: %w", err) + return nil, fmt.Errorf("iterate trace parquet logs: %w", err) } rows.Close() - return data, nil + return logs, nil } diff --git a/internal/query/attr_json_test.go b/internal/query/attr_json_test.go index 00dd939c..cf6f6955 100644 --- a/internal/query/attr_json_test.go +++ b/internal/query/attr_json_test.go @@ -22,7 +22,7 @@ func TestAttrMacroAndQuotedPath(t *testing.T) { // Flat object with dotted keys and a numeric value, as attrsJSON produces. const attrs = `{"http.method":"GET","http.status_code":200,"messaging.system":"kafka"}` if _, err := db.ExecContext(ctx, - `INSERT INTO lake.spans (namespace, service, attributes_json) VALUES ('default','svc',?)`, attrs); err != nil { + `INSERT INTO telemetry.spans (namespace, service, attributes_json) VALUES ('default','svc',?)`, attrs); err != nil { t.Fatalf("insert span: %v", err) } diff --git a/internal/query/duck.go b/internal/query/duck.go index ad65ee9f..dcb20f79 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -199,7 +199,8 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: rollupLagFromConfig(cfg)} + d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(30 * time.Second)} + repository.SetParquetPublishLock(&d.parquetMu) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -956,21 +957,11 @@ SET last_ingested_unix_nano = excluded.last_ingested_unix_nano, // rollupSafetyLagNanos is how far behind the max ingested timestamp the rollup // watermark is held, covering the worst-case delay between a row being stamped at -// ingest and committed to the lake (normal flush latency plus a retry or two). +// ingest and committed to Parquet (the bounded commit retry window plus queueing). func (d *Duck) rollupSafetyLagNanos() int64 { return d.rollupLagNanos } -// rollupLagFromConfig derives the watermark safety lag from the flush interval: -// two flush cycles, with a 30s floor. -func rollupLagFromConfig(cfg config.Config) int64 { - lag := 2 * cfg.FlushInterval - if lag < 30*time.Second { - lag = 30 * time.Second - } - return lag.Nanoseconds() -} - func maxServiceRollupWatermark(ctx context.Context, tx *sql.Tx) (int64, error) { var watermark int64 err := tx.QueryRowContext(ctx, ` diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index ae3e49e1..9d54076f 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -44,7 +44,7 @@ func TestEdgeRollupBacklog(t *testing.T) { _, err := db.ExecContext(ctx, ` WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) -INSERT INTO lake.spans ( +INSERT INTO telemetry.spans ( namespace, trace_id, span_id, parent_span_id, service, operation, kind, start_time, end_time, start_unix_nano, end_unix_nano, duration_ms, @@ -74,7 +74,7 @@ FROM range(?, ?) t(i), input`, _, err = db.ExecContext(ctx, ` WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) -INSERT INTO lake.spans ( +INSERT INTO telemetry.spans ( namespace, trace_id, span_id, parent_span_id, service, operation, kind, start_time, end_time, start_unix_nano, end_unix_nano, duration_ms, @@ -128,7 +128,7 @@ FROM range(?, ?) t(i), input`, } if err := db.QueryRowContext(ctx, ` SELECT count(DISTINCT date_trunc('minute', start_time)) -FROM lake.spans`).Scan(&spanBuckets); err != nil { +FROM telemetry.spans`).Scan(&spanBuckets); err != nil { t.Fatalf("count distinct span minute buckets: %v", err) } if spanBuckets != 180 { diff --git a/internal/query/ensure_limit_test.go b/internal/query/ensure_limit_test.go index f71c8b5d..9975e292 100644 --- a/internal/query/ensure_limit_test.go +++ b/internal/query/ensure_limit_test.go @@ -3,9 +3,6 @@ package query import ( "context" "testing" - "time" - - "github.com/labstack/fanout/internal/config" ) func TestEnsureLimit_WrapsWithoutClobberingInnerLimits(t *testing.T) { @@ -82,21 +79,3 @@ func TestExecuteSQL_CapsRowsAtMaxRows(t *testing.T) { t.Errorf("RowsReturned = %d, want 5 (capped)", resp.RowsReturned) } } - -func TestRollupLagFromConfig(t *testing.T) { - sec := int64(1_000_000_000) - cases := []struct { - flushSeconds int - wantNanos int64 - }{ - {15, 30 * sec}, // 2×15s = 30s - {20, 40 * sec}, // 2×20s = 40s - {5, 30 * sec}, // 2×5s = 10s, floored to 30s - {0, 30 * sec}, // floored to 30s - } - for _, c := range cases { - if got := rollupLagFromConfig(config.Config{FlushInterval: time.Duration(c.flushSeconds) * time.Second}); got != c.wantNanos { - t.Errorf("rollupLagFromConfig(FlushInterval=%ds) = %d, want %d", c.flushSeconds, got, c.wantNanos) - } - } -} diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index e40caf62..c3093ceb 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -8,7 +8,7 @@ import ( // defaultWriterGrace bounds how long readers may keep entering ahead of a // waiting publisher. Long enough that ordinary dashboard traffic never queues // behind maintenance, short enough that retention and compaction always run. -const defaultWriterGrace = 5 * time.Second +const defaultWriterGrace = 30 * time.Second // parquetReadGate protects Parquet file publication without sync.RWMutex's // unconditional writer preference. Queuing maintenance must not stall @@ -22,11 +22,17 @@ type parquetReadGate struct { changed *sync.Cond readers int writer bool - waiting []time.Time + waiting []parquetWaiter + nextWaiter uint64 writerGrace time.Duration now func() time.Time } +type parquetWaiter struct { + id uint64 + queuedAt time.Time +} + func (g *parquetReadGate) init() { g.once.Do(func() { g.changed = sync.NewCond(&g.mu) }) } @@ -55,7 +61,7 @@ func (g *parquetReadGate) admitsReaderLocked() bool { if len(g.waiting) == 0 { return true } - return g.clock().Sub(g.waiting[0]) < g.grace() + return g.clock().Sub(g.waiting[0].queuedAt) < g.grace() } func (g *parquetReadGate) TryRLock() bool { @@ -96,16 +102,17 @@ func (g *parquetReadGate) RUnlock() { func (g *parquetReadGate) Lock() { g.init() g.mu.Lock() - queued := g.clock() - g.waiting = append(g.waiting, queued) + g.nextWaiter++ + waiter := parquetWaiter{id: g.nextWaiter, queuedAt: g.clock()} + g.waiting = append(g.waiting, waiter) // Wake any readers parked on an earlier publisher so they re-evaluate this // publisher's grace, and so the grace clock starts for readers immediately. g.changed.Broadcast() for g.writer || g.readers > 0 { g.changed.Wait() } - for i, at := range g.waiting { - if at.Equal(queued) { + for i, queued := range g.waiting { + if queued.id == waiter.id { g.waiting = append(g.waiting[:i], g.waiting[i+1:]...) break } diff --git a/internal/query/parquet_gate_test.go b/internal/query/parquet_gate_test.go index cda531a2..3d020a62 100644 --- a/internal/query/parquet_gate_test.go +++ b/internal/query/parquet_gate_test.go @@ -35,6 +35,17 @@ func waitForQueuedWriter(t *testing.T, gate *parquetReadGate) { } } +func waitForQueuedWriters(t *testing.T, gate *parquetReadGate, count int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for gate.WaitingWriters() < count { + if time.Now().After(deadline) { + t.Fatalf("publishers queued = %d, want %d", gate.WaitingWriters(), count) + } + time.Sleep(time.Millisecond) + } +} + func TestParquetGateAdmitsReadersWhileWriterGraceRuns(t *testing.T) { clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} gate := &parquetReadGate{now: clock.Now} @@ -92,3 +103,44 @@ func TestParquetGatePublishesAfterOverlappingReadersDrain(t *testing.T) { t.Fatal("publisher never ran after readers drained") } } + +func TestParquetGateDistinguishesWritersQueuedAtSameInstant(t *testing.T) { + clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} + gate := &parquetReadGate{now: clock.Now} + gate.RLock() + acquired := make(chan struct{}, 2) + release := make(chan struct{}, 2) + for range 2 { + go func() { + gate.Lock() + acquired <- struct{}{} + <-release + gate.Unlock() + }() + } + waitForQueuedWriters(t, gate, 2) + gate.RUnlock() + for range 2 { + select { + case <-acquired: + release <- struct{}{} + case <-time.After(2 * time.Second): + t.Fatal("publisher never acquired gate") + } + } + deadline := time.Now().Add(time.Second) + for gate.WaitingWriters() != 0 { + if time.Now().After(deadline) { + t.Fatalf("stale publisher remained queued: %d", gate.WaitingWriters()) + } + time.Sleep(time.Millisecond) + } + deadline = time.Now().Add(time.Second) + for !gate.TryRLock() { + if time.Now().After(deadline) { + t.Fatal("reader refused after both publishers completed") + } + time.Sleep(time.Millisecond) + } + gate.RUnlock() +} diff --git a/internal/query/rollup_test.go b/internal/query/rollup_test.go index de00ec22..4c4b3e51 100644 --- a/internal/query/rollup_test.go +++ b/internal/query/rollup_test.go @@ -247,7 +247,7 @@ func TestRollupOnceIgnoresRowsWithoutBucketTimestamp(t *testing.T) { }) if _, err := db.Exec(` -INSERT INTO lake.logs ( +INSERT INTO telemetry.logs ( namespace, log_time, time_unix_nano, @@ -259,11 +259,11 @@ INSERT INTO lake.logs ( ingested_unix_nano ) VALUES ('ns-a', NULL, 0, 'INFO', 9, 'missing time', 'checkout', now(), 200)`); err != nil { - t.Fatalf("insert lake.logs failed: %v", err) + t.Fatalf("insert telemetry.logs failed: %v", err) } if _, err := db.Exec(` -INSERT INTO lake.metrics ( +INSERT INTO telemetry.metrics ( namespace, metric_time, time_unix_nano, @@ -275,7 +275,7 @@ INSERT INTO lake.metrics ( ingested_unix_nano ) VALUES ('ns-a', NULL, 0, 'cpu.usage', 'gauge', 'checkout', 1.0, now(), 300)`); err != nil { - t.Fatalf("insert lake.metrics failed: %v", err) + t.Fatalf("insert telemetry.metrics failed: %v", err) } if _, err := d.rollupOnce(ctx); err != nil { @@ -628,7 +628,7 @@ func insertRollupTestSpan(t *testing.T, db *sql.DB, span rollupTestSpan) { end := span.start.Add(span.duration) if _, err := db.Exec(` -INSERT INTO lake.spans ( +INSERT INTO telemetry.spans ( namespace, trace_id, span_id, @@ -668,7 +668,7 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, time.Unix(0, span.ingested).UTC(), span.ingested, ); err != nil { - t.Fatalf("insert lake.spans failed: %v", err) + t.Fatalf("insert telemetry.spans failed: %v", err) } } diff --git a/internal/query/schema.go b/internal/query/schema.go index c28ec49f..d2b5b0fd 100644 --- a/internal/query/schema.go +++ b/internal/query/schema.go @@ -11,7 +11,7 @@ const schemaTemplate = ` ## Fanout Data Schema Fanout stores telemetry in indexed hot segments and open Parquet files. DuckDB -exposes the Parquet files through the read-only lake schema. The rebuildable +exposes the Parquet files through the read-only telemetry schema. The rebuildable query cache and product state live under {DATA_DIR}. Primary query surfaces: @@ -23,7 +23,7 @@ Primary query surfaces: - endpoint_rollup table: minute endpoint counts, errors, and mergeable latency histograms ### 1. Spans -Parquet relation: lake.spans +Parquet relation: telemetry.spans Preferred query surface: spans Important columns: @@ -50,7 +50,7 @@ Common queries: - Root spans only: ... WHERE parent_span_id IS NULL OR parent_span_id = '' ### 2. Logs -Base table: lake.logs +Base table: telemetry.logs Preferred query surface: logs Important columns: @@ -71,7 +71,7 @@ Common queries: - Trace-correlated logs: ... WHERE trace_id = '...' ### 3. Metrics -Base table: lake.metrics +Base table: telemetry.metrics Preferred query surface: metrics Important columns: @@ -113,7 +113,7 @@ endpoint_rollup columns: - duration_buckets (STRUCT): cumulative fixed-boundary latency counters ## Query Guidelines -1. Prefer spans, logs, and metrics over raw lake.* tables. +1. Prefer spans, logs, and metrics over raw telemetry.* tables. 2. Always add a recent time filter for large queries. 3. Filter by namespace when relevant. 4. JSON columns are flat objects keyed by the literal attribute name. Attribute keys diff --git a/internal/query/schema_test.go b/internal/query/schema_test.go index 2911cc46..affe3ab4 100644 --- a/internal/query/schema_test.go +++ b/internal/query/schema_test.go @@ -30,7 +30,7 @@ func TestGetSchema(t *testing.T) { "endpoint_rollup", "trace_id", "service", - "lake.spans", + "telemetry.spans", "json_extract_string", } diff --git a/internal/query/sql.go b/internal/query/sql.go index 8ad94d20..6776ed32 100644 --- a/internal/query/sql.go +++ b/internal/query/sql.go @@ -30,6 +30,11 @@ type SQLResponse struct { // RowMap represents a single row as a map of column name to value type RowMap map[string]interface{} +const ( + defaultQueryTimeoutMs = 15_000 + maxQueryTimeoutMs = 25_000 +) + // ExecuteSQL validates and executes a SQL query func (d *Duck) ExecuteSQL(ctx context.Context, req SQLRequest) (resp SQLResponse) { start := time.Now() @@ -53,10 +58,12 @@ func (d *Duck) ExecuteSQL(ctx context.Context, req SQLRequest) (resp SQLResponse // Set a default timeout and clamp arbitrary SQL so one request cannot occupy // a DuckDB worker indefinitely. - const maxQueryTimeoutMs = 5 * 60 * 1000 + // Keep untrusted ad-hoc reads shorter than the Parquet publisher grace period. + // Otherwise one query can make maintenance queue, close the reader gate, and + // turn its remaining runtime into an API-wide read outage. timeoutMs := req.TimeoutMs if timeoutMs <= 0 { - timeoutMs = 30000 + timeoutMs = defaultQueryTimeoutMs } if timeoutMs > maxQueryTimeoutMs { timeoutMs = maxQueryTimeoutMs diff --git a/internal/query/sql_test.go b/internal/query/sql_test.go index 6216f08c..4bcdd787 100644 --- a/internal/query/sql_test.go +++ b/internal/query/sql_test.go @@ -131,14 +131,14 @@ func TestSQLResponseQueryPlan(t *testing.T) { } func TestDefaultTimeout(t *testing.T) { - // When TimeoutMs is 0, the effective timeout should be 30000 ms. + // When TimeoutMs is 0, the effective timeout should be 15000 ms. // We test the logic directly via the guard in ExecuteSQL. timeoutMs := 0 if timeoutMs <= 0 { - timeoutMs = 30000 + timeoutMs = defaultQueryTimeoutMs } - if timeoutMs != 30000 { - t.Errorf("default timeout = %d, want 30000", timeoutMs) + if timeoutMs != 15000 { + t.Errorf("default timeout = %d, want 15000", timeoutMs) } // Custom timeout is preserved. diff --git a/internal/query/views.go b/internal/query/views.go index ff3badcb..5665f07e 100644 --- a/internal/query/views.go +++ b/internal/query/views.go @@ -7,7 +7,7 @@ import ( ) const createSpansTable = ` -CREATE TABLE IF NOT EXISTS lake.spans ( +CREATE TABLE IF NOT EXISTS telemetry.spans ( namespace VARCHAR, trace_id VARCHAR, span_id VARCHAR, @@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS lake.spans ( );` const createLogsTable = ` -CREATE TABLE IF NOT EXISTS lake.logs ( +CREATE TABLE IF NOT EXISTS telemetry.logs ( namespace VARCHAR, log_time TIMESTAMP, observed_time TIMESTAMP, @@ -69,7 +69,7 @@ CREATE TABLE IF NOT EXISTS lake.logs ( );` const createMetricsTable = ` -CREATE TABLE IF NOT EXISTS lake.metrics ( +CREATE TABLE IF NOT EXISTS telemetry.metrics ( namespace VARCHAR, metric_time TIMESTAMP, time_unix_nano BIGINT, @@ -196,7 +196,7 @@ SELECT deployment_env, exception_type, exception_message -FROM lake.spans;` +FROM telemetry.spans;` const viewLogs = ` CREATE OR REPLACE VIEW logs AS @@ -220,7 +220,7 @@ SELECT ingested_at, ingested_unix_nano, body_template -FROM lake.logs;` +FROM telemetry.logs;` const viewMetrics = ` CREATE OR REPLACE VIEW metrics AS @@ -245,7 +245,7 @@ SELECT scope_version, ingested_at, ingested_unix_nano -FROM lake.metrics;` +FROM telemetry.metrics;` const macroAttr = ` CREATE OR REPLACE MACRO attr(json_col, key) AS @@ -255,8 +255,8 @@ CREATE OR REPLACE MACRO attr(json_col, key) AS // benchmarks. Production startup never calls this function: telemetry is // exposed exclusively through CreateParquetViews. func CreateTables(db *sql.DB) error { - if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS lake`); err != nil { - return fmt.Errorf("create lake schema: %w", err) + if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS telemetry`); err != nil { + return fmt.Errorf("create telemetry schema: %w", err) } for _, stmt := range []string{createSpansTable, createLogsTable, createMetricsTable} { if _, err := db.Exec(stmt); err != nil { @@ -289,16 +289,16 @@ func CreateCacheTables(db *sql.DB) error { } // CreateParquetViews exposes the repository's open Parquet files under the -// canonical lake schema used by Fanout's SQL kernel. +// canonical telemetry schema used by Fanout's SQL kernel. func CreateParquetViews(db *sql.DB, parquetDir string) error { - if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS lake`); err != nil { + if _, err := db.Exec(`CREATE SCHEMA IF NOT EXISTS telemetry`); err != nil { return err } for _, signal := range []string{"spans", "logs", "metrics"} { pattern := filepath.ToSlash(filepath.Join(parquetDir, signal, "*.parquet")) - stmt := fmt.Sprintf(`CREATE OR REPLACE VIEW lake.%s AS SELECT * FROM read_parquet(%s, union_by_name=true)`, signal, sqlLiteral(pattern)) + stmt := fmt.Sprintf(`CREATE OR REPLACE VIEW telemetry.%s AS SELECT * FROM read_parquet(%s, union_by_name=true)`, signal, sqlLiteral(pattern)) if _, err := db.Exec(stmt); err != nil { - return fmt.Errorf("create parquet view lake.%s: %w", signal, err) + return fmt.Errorf("create parquet view telemetry.%s: %w", signal, err) } } return nil diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 81d368ad..cbd1a26e 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -76,25 +76,110 @@ func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { return stats, nil } -func (p *ParquetStore) WriteSpans(id string, rows []Span) error { - if len(rows) == 0 { - return nil +// StageBatch writes durable files that DuckDB's *.parquet views cannot see. +// Publication is a separate, rename-only step so encoding and fsync never hold +// the query read gate. +func (p *ParquetStore) StageBatch(id string, spans []Span, logs []Log, metrics []Metric) error { + for _, item := range []struct { + signal string + write func(string) error + }{ + {"spans", func(path string) error { return writeParquet(path, spanParquetColumns(), spans) }}, + {"logs", func(path string) error { return writeParquet(path, logParquetColumns(), logs) }}, + {"metrics", func(path string) error { return writeParquet(path, metricParquetColumns(), metrics) }}, + } { + rows := len(spans) + if item.signal == "logs" { + rows = len(logs) + } else if item.signal == "metrics" { + rows = len(metrics) + } + if rows == 0 { + continue + } + final := filepath.Join(p.dir, item.signal, id+".parquet") + if _, err := os.Stat(final); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := item.write(final + ".pending"); err != nil { + return fmt.Errorf("stage %s parquet: %w", item.signal, err) + } } - return writeParquet(filepath.Join(p.dir, "spans", id+".parquet"), spanParquetColumns(), rows) + return nil } -func (p *ParquetStore) WriteLogs(id string, rows []Log) error { - if len(rows) == 0 { - return nil +// PublishBatch atomically exposes all present signal files to in-process +// readers when the caller holds the Parquet publication gate. The returned +// rollback is used if the hot span projection cannot be published afterward. +func (p *ParquetStore) PublishBatch(id string, hasSpans, hasLogs, hasMetrics bool) (func() error, error) { + present := []struct { + signal string + has bool + }{{"spans", hasSpans}, {"logs", hasLogs}, {"metrics", hasMetrics}} + renamed := make([]string, 0, len(present)) + rollback := func() error { + var rollbackErr error + for i := len(renamed) - 1; i >= 0; i-- { + final := filepath.Join(p.dir, renamed[i], id+".parquet") + if err := os.Rename(final, final+".pending"); err != nil && !errors.Is(err, os.ErrNotExist) { + rollbackErr = errors.Join(rollbackErr, err) + } + } + if len(renamed) > 0 { + rollbackErr = errors.Join(rollbackErr, syncParquetSignalDirectories(p.dir, renamed)) + } + return rollbackErr + } + for _, item := range present { + if !item.has { + continue + } + final := filepath.Join(p.dir, item.signal, id+".parquet") + if _, err := os.Stat(final); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return rollback, errors.Join(err, rollback()) + } + if err := os.Rename(final+".pending", final); err != nil { + return rollback, errors.Join(err, rollback()) + } + renamed = append(renamed, item.signal) } - return writeParquet(filepath.Join(p.dir, "logs", id+".parquet"), logParquetColumns(), rows) + if err := syncParquetSignalDirectories(p.dir, renamed); err != nil { + return rollback, errors.Join(err, rollback()) + } + return rollback, nil } -func (p *ParquetStore) WriteMetrics(id string, rows []Metric) error { - if len(rows) == 0 { - return nil +// DiscardBatch removes invisible staging files for a batch that another commit +// or compaction has already consumed. +func (p *ParquetStore) DiscardBatch(id string) error { + var discardErr error + changed := make([]string, 0, 3) + for _, signal := range []string{"spans", "logs", "metrics"} { + path := filepath.Join(p.dir, signal, id+".parquet.pending") + if err := os.Remove(path); err == nil { + changed = append(changed, signal) + } else if !errors.Is(err, os.ErrNotExist) { + discardErr = errors.Join(discardErr, err) + } + } + return errors.Join(discardErr, syncParquetSignalDirectories(p.dir, changed)) +} + +func syncParquetSignalDirectories(root string, signals []string) error { + seen := make(map[string]struct{}, len(signals)) + var syncErr error + for _, signal := range signals { + if _, ok := seen[signal]; ok { + continue + } + seen[signal] = struct{}{} + syncErr = errors.Join(syncErr, syncDirectory(filepath.Join(root, signal))) } - return writeParquet(filepath.Join(p.dir, "metrics", id+".parquet"), metricParquetColumns(), rows) + return syncErr } func writeParquet[T any](path string, columns []parquetColumn[T], rows []T) error { diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go new file mode 100644 index 00000000..1306139d --- /dev/null +++ b/internal/telemetry/parquet_test.go @@ -0,0 +1,56 @@ +package telemetry + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParquetBatchStagingIsInvisibleUntilPublish(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + const id = "batch-1" + if err := store.StageBatch(id, []Span{{Namespace: "default", TraceID: "trace", SpanID: "span"}}, []Log{{Namespace: "default", Body: "body"}}, []Metric{{Namespace: "default", Name: "metric"}}); err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + final := filepath.Join(store.Dir(), signal, id+".parquet") + if _, err := os.Stat(final); !os.IsNotExist(err) { + t.Fatalf("%s final file visible before publish: %v", signal, err) + } + if _, err := os.Stat(final + ".pending"); err != nil { + t.Fatalf("%s pending file: %v", signal, err) + } + } + rollback, err := store.PublishBatch(id, true, true, true) + if err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(store.Dir(), signal, id+".parquet")); err != nil { + t.Fatalf("%s final file: %v", signal, err) + } + } + if err := rollback(); err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + final := filepath.Join(store.Dir(), signal, id+".parquet") + if _, err := os.Stat(final); !os.IsNotExist(err) { + t.Fatalf("%s final file visible after rollback: %v", signal, err) + } + if _, err := os.Stat(final + ".pending"); err != nil { + t.Fatalf("%s restored pending file: %v", signal, err) + } + } + if err := store.DiscardBatch(id); err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(store.Dir(), signal, id+".parquet.pending")); !os.IsNotExist(err) { + t.Fatalf("%s pending file remained after discard: %v", signal, err) + } + } +} diff --git a/internal/telemetry/segment/signal_store.go b/internal/telemetry/segment/signal_store.go deleted file mode 100644 index 327907a6..00000000 --- a/internal/telemetry/segment/signal_store.go +++ /dev/null @@ -1,826 +0,0 @@ -// Generic signal segments cover logs and metrics; spans add specialized indexes. -package segment - -import ( - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "os" - "path/filepath" - "reflect" - "regexp" - "sync" - "time" - - "github.com/klauspost/compress/zstd" -) - -const ( - signalMagic = "FANSIG04" - signalVersion = uint32(4) - signalHeaderSize = 64 - signalBlockSize = 32 - signalBlockRows = 2048 - signalMaxBlocks = 1 << 20 -) - -var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) - -// ValidID reports whether id can name a segment file. Callers that persist a -// batch before publishing it use this to reject an ID no projection could ever -// accept, rather than discovering it once part of the batch is already written. -func ValidID(id string) bool { return segmentIDPattern.MatchString(id) } - -type signalBlock struct { - offset uint64 - length uint32 - rows uint32 - min int64 - max int64 -} - -type signalSegment struct { - id string - path string - rows uint32 - min int64 - max int64 - fieldCount uint32 - fingerprint uint64 - blocks []signalBlock -} - -type signalManifest struct { - Version uint32 `json:"version"` - Files []string `json:"files"` -} - -type signalFile interface { - ReadAt([]byte, int64) (int, error) - Close() error -} - -// SignalStore persists one telemetry signal as immutable, independently -// compressed columns. T must be a struct containing only string, []byte, -// int32, uint32, int64, and float64 fields. -type SignalStore[T any] struct { - dir string - timeField int - codec structCodec[T] - writeMu sync.Mutex - mu sync.RWMutex - manifest signalManifest - segments []signalSegment - encoder *zstd.Encoder - decoders sync.Pool - openFile func(string) (signalFile, error) -} - -func OpenSignalStore[T any](dir, timeField string) (*SignalStore[T], error) { - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("create signal directory: %w", err) - } - codec, err := newStructCodec[T]() - if err != nil { - return nil, err - } - field, found := codec.typ.FieldByName(timeField) - if !found || field.Type.Kind() != reflect.Int64 { - return nil, fmt.Errorf("signal time field %q must be int64", timeField) - } - enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1), zstd.WithEncoderCRC(true)) - if err != nil { - return nil, fmt.Errorf("create signal encoder: %w", err) - } - s := &SignalStore[T]{dir: dir, timeField: field.Index[0], codec: codec, encoder: enc, openFile: func(path string) (signalFile, error) { return os.Open(path) }} - s.decoders.New = func() any { - dec, decErr := newSegmentDecoder() - if decErr != nil { - panic(decErr) - } - return dec - } - if err := s.load(); err != nil { - enc.Close() - return nil, err - } - return s, nil -} - -func (s *SignalStore[T]) Close() error { - s.writeMu.Lock() - defer s.writeMu.Unlock() - return s.encoder.Close() -} - -func (s *SignalStore[T]) load() error { - path := filepath.Join(s.dir, "MANIFEST.json") - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - s.manifest = signalManifest{Version: signalVersion} - return nil - } - if err != nil { - return fmt.Errorf("read signal manifest: %w", err) - } - if err := json.Unmarshal(data, &s.manifest); err != nil { - return fmt.Errorf("decode signal manifest: %w", err) - } - if s.manifest.Version != signalVersion { - return fmt.Errorf("signal manifest version %d is unsupported; expected %d", s.manifest.Version, signalVersion) - } - for _, name := range s.manifest.Files { - seg, err := openSignalSegment(filepath.Join(s.dir, name)) - if err != nil { - return fmt.Errorf("open signal segment %s: %w", name, err) - } - if seg.fieldCount != uint32(len(s.codec.fields)) || seg.fingerprint != s.codec.fingerprint { - return fmt.Errorf("signal segment %s schema does not match canonical telemetry row", name) - } - s.segments = append(s.segments, seg) - } - return nil -} - -// Append publishes rows exactly once for id. Replaying a committed transaction -// after a crash is therefore safe. -func (s *SignalStore[T]) Append(id string, rows []T) error { - if len(rows) == 0 { - return nil - } - if !segmentIDPattern.MatchString(id) { - return fmt.Errorf("invalid segment id %q", id) - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - - name := id + ".fseg" - s.mu.RLock() - for _, existing := range s.manifest.Files { - if existing == name { - s.mu.RUnlock() - return nil - } - } - current := s.manifest - s.mu.RUnlock() - - tmp := filepath.Join(s.dir, name+".tmp") - final := filepath.Join(s.dir, name) - _ = os.Remove(tmp) - seg, err := s.writeSegment(tmp, id, rows) - if err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - _ = os.Remove(tmp) - return fmt.Errorf("publish signal segment: %w", err) - } - if err := syncDir(s.dir); err != nil { - return err - } - next := current - next.Version = signalVersion - next.Files = append(append([]string(nil), current.Files...), name) - if err := writeSignalManifest(s.dir, next); err != nil { - return err - } - seg.path = final - s.mu.Lock() - s.manifest = next - s.segments = append(s.segments, seg) - s.mu.Unlock() - return nil -} - -func (s *SignalStore[T]) writeSegment(path, id string, rows []T) (signalSegment, error) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) - if err != nil { - return signalSegment{}, fmt.Errorf("create signal segment: %w", err) - } - ok := false - defer func() { - _ = f.Close() - if !ok { - _ = os.Remove(path) - } - }() - if _, err := f.Write(make([]byte, signalHeaderSize)); err != nil { - return signalSegment{}, err - } - seg := signalSegment{id: id, rows: uint32(len(rows)), min: math.MaxInt64, max: math.MinInt64, fieldCount: uint32(len(s.codec.fields)), fingerprint: s.codec.fingerprint} - offset := uint64(signalHeaderSize) - for start := 0; start < len(rows); start += signalBlockRows { - end := min(start+signalBlockRows, len(rows)) - blockMin, blockMax := int64(math.MaxInt64), int64(math.MinInt64) - for i := start; i < end; i++ { - ts := reflect.ValueOf(rows[i]).Field(s.timeField).Int() - blockMin, blockMax = min(blockMin, ts), max(blockMax, ts) - seg.min, seg.max = min(seg.min, ts), max(seg.max, ts) - } - encoded, err := s.codec.encodeBlock(s.encoder, rows[start:end]) - if err != nil { - return signalSegment{}, err - } - if _, err := f.Write(encoded); err != nil { - return signalSegment{}, err - } - seg.blocks = append(seg.blocks, signalBlock{offset: offset, length: uint32(len(encoded)), rows: uint32(end - start), min: blockMin, max: blockMax}) - offset += uint64(len(encoded)) - } - dirOffset := offset - directory := make([]byte, len(seg.blocks)*signalBlockSize) - for i, block := range seg.blocks { - entry := directory[i*signalBlockSize:] - binary.LittleEndian.PutUint64(entry[0:8], block.offset) - binary.LittleEndian.PutUint32(entry[8:12], block.length) - binary.LittleEndian.PutUint32(entry[12:16], block.rows) - binary.LittleEndian.PutUint64(entry[16:24], uint64(block.min)) - binary.LittleEndian.PutUint64(entry[24:32], uint64(block.max)) - } - if _, err := f.Write(directory); err != nil { - return signalSegment{}, err - } - var header [signalHeaderSize]byte - copy(header[0:8], signalMagic) - binary.LittleEndian.PutUint32(header[8:12], signalVersion) - binary.LittleEndian.PutUint32(header[12:16], seg.rows) - binary.LittleEndian.PutUint32(header[16:20], uint32(len(seg.blocks))) - binary.LittleEndian.PutUint32(header[20:24], uint32(len(s.codec.fields))) - binary.LittleEndian.PutUint64(header[24:32], uint64(seg.min)) - binary.LittleEndian.PutUint64(header[32:40], uint64(seg.max)) - binary.LittleEndian.PutUint64(header[40:48], dirOffset) - binary.LittleEndian.PutUint64(header[48:56], s.codec.fingerprint) - if _, err := f.WriteAt(header[:], 0); err != nil { - return signalSegment{}, err - } - if err := f.Sync(); err != nil { - return signalSegment{}, err - } - if err := f.Close(); err != nil { - return signalSegment{}, err - } - ok = true - return seg, nil -} - -// Scan visits rows in [start,end), using segment and block time pruning. -func (s *SignalStore[T]) Scan(start, end int64, visit func(T) bool) error { - s.mu.RLock() - segments := make([]signalSegment, 0, len(s.segments)) - for _, seg := range s.segments { - if seg.max < start || seg.min >= end { - continue - } - segments = append(segments, seg) - } - s.mu.RUnlock() - dec := s.decoders.Get().(*zstd.Decoder) - defer s.decoders.Put(dec) - for _, seg := range segments { - // Revalidate and open while holding the metadata read lock. Pruning must - // acquire the write lock before unlinking a retired segment, so once this - // descriptor is open the scan can safely release the lock and decode it. - s.mu.RLock() - active := false - for _, current := range s.segments { - if current.path == seg.path { - active = true - break - } - } - if !active { - s.mu.RUnlock() - continue - } - openFile := s.openFile - if openFile == nil { - openFile = func(path string) (signalFile, error) { return os.Open(path) } - } - f, err := openFile(seg.path) - s.mu.RUnlock() - if err != nil { - return err - } - stop := false - for _, block := range seg.blocks { - if block.max < start || block.min >= end { - continue - } - data := make([]byte, block.length) - if _, err := f.ReadAt(data, int64(block.offset)); err != nil { - _ = f.Close() - return err - } - rows, err := s.codec.decodeBlock(dec, data, int(block.rows)) - if err != nil { - _ = f.Close() - return fmt.Errorf("decode %s: %w", filepath.Base(seg.path), err) - } - for _, row := range rows { - ts := reflect.ValueOf(row).Field(s.timeField).Int() - if ts >= start && ts < end && !visit(row) { - stop = true - break - } - } - if stop { - break - } - } - if err := f.Close(); err != nil { - return err - } - if stop { - return nil - } - } - return nil -} - -func (s *SignalStore[T]) RowCount() uint64 { - s.mu.RLock() - defer s.mu.RUnlock() - var total uint64 - for _, seg := range s.segments { - total += uint64(seg.rows) - } - return total -} - -// Bounds returns the oldest and newest event timestamps currently covered by -// the hot acceleration tier. -func (s *SignalStore[T]) Bounds() (int64, int64, bool) { - s.mu.RLock() - defer s.mu.RUnlock() - if len(s.segments) == 0 { - return 0, 0, false - } - minTime, maxTime := s.segments[0].min, s.segments[0].max - for _, segment := range s.segments[1:] { - minTime = min(minTime, segment.min) - maxTime = max(maxTime, segment.max) - } - return minTime, maxTime, true -} - -func (s *SignalStore[T]) SegmentCount() int { - s.mu.RLock() - defer s.mu.RUnlock() - return len(s.segments) -} - -// CompactCommitted combines raw ingest segments known to be fully committed. -// Compressed column blocks are copied verbatim, so compaction is independent of -// row width and does not inflate the process heap. -func (s *SignalStore[T]) CompactCommitted(committed map[string]struct{}, maxInputs int) (int, error) { - if maxInputs < 2 { - return 0, nil - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.mu.RLock() - selected := make([]signalSegment, 0, maxInputs) - rest := make([]signalSegment, 0, len(s.segments)) - for _, seg := range s.segments { - if len(selected) < maxInputs { - if _, ok := committed[seg.id]; ok { - selected = append(selected, seg) - continue - } - } - rest = append(rest, seg) - } - s.mu.RUnlock() - if len(selected) < 2 { - return 0, nil - } - id := fmt.Sprintf("compact-%d", time.Now().UnixNano()) - name := id + ".fseg" - tmp, final := filepath.Join(s.dir, name+".tmp"), filepath.Join(s.dir, name) - replacement, err := s.writeCompactedSegment(tmp, id, selected) - if err != nil { - return 0, err - } - if err := os.Rename(tmp, final); err != nil { - _ = os.Remove(tmp) - return 0, fmt.Errorf("publish compacted signal segment: %w", err) - } - if err := syncDir(s.dir); err != nil { - return 0, err - } - next := signalManifest{Version: signalVersion, Files: make([]string, 0, len(rest)+1)} - next.Files = append(next.Files, name) - for _, seg := range rest { - next.Files = append(next.Files, filepath.Base(seg.path)) - } - if err := writeSignalManifest(s.dir, next); err != nil { - return 0, err - } - replacement.path = final - s.mu.Lock() - s.manifest = next - s.segments = append([]signalSegment{replacement}, rest...) - s.mu.Unlock() - var removeErr error - for _, seg := range selected { - if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { - removeErr = errors.Join(removeErr, err) - } - } - return len(selected), errors.Join(removeErr, syncDir(s.dir)) -} - -func (s *SignalStore[T]) writeCompactedSegment(path, id string, inputs []signalSegment) (signalSegment, error) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) - if err != nil { - return signalSegment{}, fmt.Errorf("create compacted signal segment: %w", err) - } - ok := false - defer func() { - _ = f.Close() - if !ok { - _ = os.Remove(path) - } - }() - if _, err := f.Write(make([]byte, signalHeaderSize)); err != nil { - return signalSegment{}, err - } - replacement := signalSegment{id: id, min: math.MaxInt64, max: math.MinInt64, fieldCount: uint32(len(s.codec.fields)), fingerprint: s.codec.fingerprint} - offset := uint64(signalHeaderSize) - for _, input := range inputs { - source, err := os.Open(input.path) - if err != nil { - return signalSegment{}, err - } - for _, block := range input.blocks { - if _, err := io.CopyN(f, io.NewSectionReader(source, int64(block.offset), int64(block.length)), int64(block.length)); err != nil { - _ = source.Close() - return signalSegment{}, fmt.Errorf("copy compressed signal block: %w", err) - } - replacement.blocks = append(replacement.blocks, signalBlock{offset: offset, length: block.length, rows: block.rows, min: block.min, max: block.max}) - offset += uint64(block.length) - } - if err := source.Close(); err != nil { - return signalSegment{}, err - } - replacement.rows += input.rows - replacement.min = min(replacement.min, input.min) - replacement.max = max(replacement.max, input.max) - } - dirOffset := offset - directory := make([]byte, len(replacement.blocks)*signalBlockSize) - for i, block := range replacement.blocks { - entry := directory[i*signalBlockSize:] - binary.LittleEndian.PutUint64(entry[0:8], block.offset) - binary.LittleEndian.PutUint32(entry[8:12], block.length) - binary.LittleEndian.PutUint32(entry[12:16], block.rows) - binary.LittleEndian.PutUint64(entry[16:24], uint64(block.min)) - binary.LittleEndian.PutUint64(entry[24:32], uint64(block.max)) - } - if _, err := f.Write(directory); err != nil { - return signalSegment{}, err - } - var header [signalHeaderSize]byte - copy(header[0:8], signalMagic) - binary.LittleEndian.PutUint32(header[8:12], signalVersion) - binary.LittleEndian.PutUint32(header[12:16], replacement.rows) - binary.LittleEndian.PutUint32(header[16:20], uint32(len(replacement.blocks))) - binary.LittleEndian.PutUint32(header[20:24], replacement.fieldCount) - binary.LittleEndian.PutUint64(header[24:32], uint64(replacement.min)) - binary.LittleEndian.PutUint64(header[32:40], uint64(replacement.max)) - binary.LittleEndian.PutUint64(header[40:48], dirOffset) - binary.LittleEndian.PutUint64(header[48:56], replacement.fingerprint) - if _, err := f.WriteAt(header[:], 0); err != nil { - return signalSegment{}, err - } - if err := f.Sync(); err != nil { - return signalSegment{}, err - } - if err := f.Close(); err != nil { - return signalSegment{}, err - } - ok = true - return replacement, nil -} - -func (s *SignalStore[T]) PruneBefore(cutoff int64) (int, error) { - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.mu.RLock() - current := s.manifest - segments := append([]signalSegment(nil), s.segments...) - s.mu.RUnlock() - kept := make([]signalSegment, 0, len(segments)) - removed := make([]signalSegment, 0) - for _, seg := range segments { - if seg.max < cutoff { - removed = append(removed, seg) - } else { - kept = append(kept, seg) - } - } - if len(removed) == 0 { - return 0, nil - } - next := current - next.Files = make([]string, 0, len(kept)) - for _, seg := range kept { - next.Files = append(next.Files, filepath.Base(seg.path)) - } - if err := writeSignalManifest(s.dir, next); err != nil { - return 0, err - } - s.mu.Lock() - s.manifest, s.segments = next, kept - s.mu.Unlock() - var removeErr error - for _, seg := range removed { - if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { - removeErr = errors.Join(removeErr, err) - } - } - return len(removed), errors.Join(removeErr, syncDir(s.dir)) -} - -// validateSignalBlocks bounds every decoded block entry. Scan allocates from -// block.length and decodes block.rows, so both must be known to fit the file -// before the entries are stored. Blocks live between the header and the -// directory, and their row counts must add up to the header's total. -func validateSignalBlocks(size, dirOffset uint64, blocks []signalBlock, rows uint32) error { - var counted uint64 - for _, block := range blocks { - if block.offset < signalHeaderSize || block.offset > dirOffset { - return errors.New("block offset outside the segment payload") - } - if block.length == 0 || uint64(block.length) > dirOffset-block.offset || uint64(block.length) > segmentMaxCompressedBytes { - return errors.New("block extends past the block directory") - } - if block.rows == 0 || block.rows > signalBlockRows { - return errors.New("block row count is out of range") - } - counted += uint64(block.rows) - } - if counted != uint64(rows) { - return errors.New("block row counts disagree with the segment header") - } - if dirOffset > size { - return errors.New("block directory starts past the end of the segment") - } - return nil -} - -// validateSignalDirectory bounds the block directory against the file size, -// so a torn header cannot size an allocation the file could never hold. -func validateSignalDirectory(size, dirOffset uint64, blockCount uint32) error { - if blockCount > signalMaxBlocks { - return errors.New("signal block count is out of range") - } - if dirOffset < signalHeaderSize || dirOffset > size { - return errors.New("corrupt directory offset") - } - if uint64(blockCount) > (size-dirOffset)/signalBlockSize { - return errors.New("directory does not fit in the segment") - } - return nil -} - -func openSignalSegment(path string) (signalSegment, error) { - f, err := os.Open(path) - if err != nil { - return signalSegment{}, err - } - defer f.Close() - var header [signalHeaderSize]byte - if _, err := io.ReadFull(f, header[:]); err != nil { - return signalSegment{}, err - } - if string(header[0:8]) != signalMagic || binary.LittleEndian.Uint32(header[8:12]) != signalVersion { - return signalSegment{}, errors.New("unsupported signal segment format") - } - seg := signalSegment{ - id: filepath.Base(path[:len(path)-len(filepath.Ext(path))]), path: path, - rows: binary.LittleEndian.Uint32(header[12:16]), fieldCount: binary.LittleEndian.Uint32(header[20:24]), - min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40])), - fingerprint: binary.LittleEndian.Uint64(header[48:56]), - } - blockCount := binary.LittleEndian.Uint32(header[16:20]) - dirOffset := binary.LittleEndian.Uint64(header[40:48]) - info, err := f.Stat() - if err != nil { - return signalSegment{}, err - } - if err := validateSignalDirectory(uint64(info.Size()), dirOffset, blockCount); err != nil { - return signalSegment{}, fmt.Errorf("signal segment %s: %w", filepath.Base(path), err) - } - count := int(blockCount) - directory := make([]byte, count*signalBlockSize) - if _, err := f.ReadAt(directory, int64(dirOffset)); err != nil { - return signalSegment{}, err - } - for i := range count { - entry := directory[i*signalBlockSize:] - seg.blocks = append(seg.blocks, signalBlock{ - offset: binary.LittleEndian.Uint64(entry[0:8]), length: binary.LittleEndian.Uint32(entry[8:12]), rows: binary.LittleEndian.Uint32(entry[12:16]), - min: int64(binary.LittleEndian.Uint64(entry[16:24])), max: int64(binary.LittleEndian.Uint64(entry[24:32])), - }) - } - if err := validateSignalBlocks(uint64(info.Size()), dirOffset, seg.blocks, seg.rows); err != nil { - return signalSegment{}, fmt.Errorf("signal segment %s: %w", filepath.Base(path), err) - } - return seg, nil -} - -func writeSignalManifest(dir string, manifest signalManifest) error { - data, err := json.Marshal(manifest) - if err != nil { - return err - } - tmp := filepath.Join(dir, "MANIFEST.json.tmp") - final := filepath.Join(dir, "MANIFEST.json") - f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - return err - } - if _, err := f.Write(data); err != nil { - _ = f.Close() - return err - } - if err := f.Sync(); err != nil { - _ = f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - return err - } - return syncDir(dir) -} - -type fieldKind uint8 - -const ( - fieldString fieldKind = iota - fieldBytes - fieldInt64 - fieldInt32 - fieldUint32 - fieldFloat64 -) - -type codecField struct { - name string - index int - kind fieldKind -} - -type structCodec[T any] struct { - typ reflect.Type - fields []codecField - fingerprint uint64 -} - -func newStructCodec[T any]() (structCodec[T], error) { - typ := reflect.TypeOf((*T)(nil)).Elem() - if typ.Kind() != reflect.Struct { - return structCodec[T]{}, errors.New("signal row must be a struct") - } - c := structCodec[T]{typ: typ} - var fp uint64 = 1469598103934665603 - for i := range typ.NumField() { - field := typ.Field(i) - var kind fieldKind - switch { - case field.Type.Kind() == reflect.String: - kind = fieldString - case field.Type == reflect.TypeOf([]byte(nil)): - kind = fieldBytes - case field.Type.Kind() == reflect.Int64: - kind = fieldInt64 - case field.Type.Kind() == reflect.Int32: - kind = fieldInt32 - case field.Type.Kind() == reflect.Uint32: - kind = fieldUint32 - case field.Type.Kind() == reflect.Float64: - kind = fieldFloat64 - default: - return structCodec[T]{}, fmt.Errorf("unsupported field %s (%s)", field.Name, field.Type) - } - c.fields = append(c.fields, codecField{name: field.Name, index: i, kind: kind}) - for _, b := range []byte(field.Name + ":" + field.Type.String()) { - fp ^= uint64(b) - fp *= 1099511628211 - } - } - c.fingerprint = fp - return c, nil -} - -func (c structCodec[T]) encodeBlock(enc *zstd.Encoder, rows []T) ([]byte, error) { - columns := make([][]byte, len(c.fields)) - for _, row := range rows { - value := reflect.ValueOf(row) - for i, field := range c.fields { - v := value.Field(field.index) - switch field.kind { - case fieldString: - columns[i] = appendBytes(columns[i], []byte(v.String())) - case fieldBytes: - columns[i] = appendBytes(columns[i], v.Bytes()) - case fieldInt64: - columns[i] = binary.LittleEndian.AppendUint64(columns[i], uint64(v.Int())) - case fieldInt32: - columns[i] = binary.LittleEndian.AppendUint32(columns[i], uint32(v.Int())) - case fieldUint32: - columns[i] = binary.LittleEndian.AppendUint32(columns[i], uint32(v.Uint())) - case fieldFloat64: - columns[i] = binary.LittleEndian.AppendUint64(columns[i], math.Float64bits(v.Float())) - } - } - } - headerSize := 4 + len(columns)*8 - out := make([]byte, headerSize) - binary.LittleEndian.PutUint32(out[:4], uint32(len(columns))) - offset := headerSize - for i, plain := range columns { - compressed := enc.EncodeAll(plain, nil) - entry := out[4+i*8:] - binary.LittleEndian.PutUint32(entry[:4], uint32(offset)) - binary.LittleEndian.PutUint32(entry[4:8], uint32(len(compressed))) - out = append(out, compressed...) - offset += len(compressed) - } - return out, nil -} - -func (c structCodec[T]) decodeBlock(dec *zstd.Decoder, block []byte, count int) ([]T, error) { - headerSize := 4 + len(c.fields)*8 - if len(block) < headerSize || int(binary.LittleEndian.Uint32(block[:4])) != len(c.fields) { - return nil, errors.New("invalid signal block header") - } - columns := make([][]byte, len(c.fields)) - for i := range c.fields { - entry := block[4+i*8:] - offset := int(binary.LittleEndian.Uint32(entry[:4])) - length := int(binary.LittleEndian.Uint32(entry[4:8])) - if offset < headerSize || length < 0 || offset > len(block)-length { - return nil, errors.New("invalid signal column extent") - } - plain, err := dec.DecodeAll(block[offset:offset+length], nil) - if err != nil { - return nil, err - } - columns[i] = plain - } - rows := make([]T, count) - for fieldIndex, field := range c.fields { - column := columns[fieldIndex] - for row := range count { - dst := reflect.ValueOf(&rows[row]).Elem().Field(field.index) - switch field.kind { - case fieldString, fieldBytes: - value, rest, err := consumeByteView(column) - if err != nil { - return nil, err - } - column = rest - if field.kind == fieldString { - dst.SetString(string(value)) - } else { - dst.SetBytes(append([]byte(nil), value...)) - } - case fieldInt64, fieldFloat64: - if len(column) < 8 { - return nil, io.ErrUnexpectedEOF - } - bits := binary.LittleEndian.Uint64(column[:8]) - column = column[8:] - if field.kind == fieldInt64 { - dst.SetInt(int64(bits)) - } else { - dst.SetFloat(math.Float64frombits(bits)) - } - case fieldInt32, fieldUint32: - if len(column) < 4 { - return nil, io.ErrUnexpectedEOF - } - bits := binary.LittleEndian.Uint32(column[:4]) - column = column[4:] - if field.kind == fieldInt32 { - dst.SetInt(int64(int32(bits))) - } else { - dst.SetUint(uint64(bits)) - } - } - } - if len(column) != 0 { - return nil, fmt.Errorf("column %s has trailing data", field.name) - } - } - return rows, nil -} diff --git a/internal/telemetry/segment/signal_store_test.go b/internal/telemetry/segment/signal_store_test.go deleted file mode 100644 index 99cd6210..00000000 --- a/internal/telemetry/segment/signal_store_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package segment - -import ( - "encoding/binary" - "os" - "path/filepath" - "strings" - "sync/atomic" - "testing" - - "github.com/labstack/fanout/internal/telemetry" -) - -type countedSignalFile struct { - *os.File - open *atomic.Int64 -} - -func (f *countedSignalFile) Close() error { - f.open.Add(-1) - return f.File.Close() -} - -func TestSignalStoreScanKeepsFileDescriptorsBounded(t *testing.T) { - store, err := OpenSignalStore[telemetry.Log](t.TempDir(), "EventUnixNanos") - if err != nil { - t.Fatalf("OpenSignalStore: %v", err) - } - defer store.Close() - for i := range 32 { - if err := store.Append(stringID(i), []telemetry.Log{{EventUnixNanos: int64(i + 1)}}); err != nil { - t.Fatalf("Append(%d): %v", i, err) - } - } - var open, maximum atomic.Int64 - store.openFile = func(path string) (signalFile, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - current := open.Add(1) - for { - previous := maximum.Load() - if current <= previous || maximum.CompareAndSwap(previous, current) { - break - } - } - return &countedSignalFile{File: file, open: &open}, nil - } - visited := 0 - if err := store.Scan(0, 100, func(telemetry.Log) bool { - visited++ - return true - }); err != nil { - t.Fatalf("Scan: %v", err) - } - if visited != 32 { - t.Fatalf("visited = %d, want 32", visited) - } - if got := maximum.Load(); got != 1 { - t.Fatalf("maximum simultaneous descriptors = %d, want 1", got) - } - if got := open.Load(); got != 0 { - t.Fatalf("descriptors left open = %d, want 0", got) - } -} - -func stringID(i int) string { - const digits = "0123456789abcdef" - return "batch-" + string([]byte{digits[(i>>4)&15], digits[i&15]}) -} - -func TestValidateSignalDirectoryRejectsOutOfBoundsCount(t *testing.T) { - const size = 4096 - if err := validateSignalDirectory(size, signalHeaderSize, 0xFFFFFFFF); err == nil { - t.Fatal("validateSignalDirectory accepted a block count larger than the file") - } - if err := validateSignalDirectory(size, ^uint64(0)-1024, 64); err == nil { - t.Fatal("validateSignalDirectory accepted a wrapping directory offset") - } - if err := validateSignalDirectory(size, 0, 1); err == nil { - t.Fatal("validateSignalDirectory accepted a directory inside the header") - } - if err := validateSignalDirectory(size, signalHeaderSize, 8); err != nil { - t.Fatalf("validateSignalDirectory rejected a sound header: %v", err) - } -} - -func TestValidateSignalBlocksRejectsOutOfBoundsExtents(t *testing.T) { - const size, dirOffset = 4096, uint64(2048) - sound := []signalBlock{ - {offset: signalHeaderSize, length: 512, rows: 10}, - {offset: signalHeaderSize + 512, length: 512, rows: 10}, - } - if err := validateSignalBlocks(size, dirOffset, sound, 20); err != nil { - t.Fatalf("validateSignalBlocks rejected sound blocks: %v", err) - } - tests := []struct { - name string - blocks []signalBlock - rows uint32 - }{ - {"length past the directory", []signalBlock{{offset: signalHeaderSize, length: 4096, rows: 10}}, 10}, - {"offset inside the header", []signalBlock{{offset: 0, length: 16, rows: 10}}, 10}, - {"extent wraps", []signalBlock{{offset: ^uint64(0) - 8, length: 64, rows: 10}}, 10}, - {"rows exceed the block cap", []signalBlock{{offset: signalHeaderSize, length: 16, rows: signalBlockRows + 1}}, signalBlockRows + 1}, - {"rows disagree with the header", []signalBlock{{offset: signalHeaderSize, length: 16, rows: 10}}, 11}, - {"empty block", []signalBlock{{offset: signalHeaderSize, length: 0, rows: 0}}, 0}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := validateSignalBlocks(size, dirOffset, test.blocks, test.rows); err == nil { - t.Fatal("validateSignalBlocks accepted a corrupt block directory") - } - }) - } -} - -func TestOpenSignalStoreRejectsCorruptBlockEntry(t *testing.T) { - dir := t.TempDir() - store, err := OpenSignalStore[telemetry.Log](dir, "EventUnixNanos") - if err != nil { - t.Fatal(err) - } - if err := store.Append("seg-block", []telemetry.Log{{EventUnixNanos: 100, Body: "hello"}}); err != nil { - t.Fatal(err) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - path := filepath.Join(dir, "seg-block.fseg") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - dirOffset := binary.LittleEndian.Uint64(data[40:48]) - binary.LittleEndian.PutUint32(data[int(dirOffset)+8:], 1<<30) - if err := os.WriteFile(path, data, 0o644); err != nil { - t.Fatal(err) - } - _, err = OpenSignalStore[telemetry.Log](dir, "EventUnixNanos") - if err == nil { - t.Fatal("OpenSignalStore accepted a segment whose block extends past its directory") - } - if !strings.Contains(err.Error(), "block extends past the block directory") { - t.Fatalf("OpenSignalStore error = %v, want the block-extent guard to reject it", err) - } -} diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index e7eb0c55..ad9078af 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "sort" "strconv" "strings" @@ -37,6 +38,11 @@ const ( segmentMaxBlocks = 1 << 20 ) +var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) + +// ValidID reports whether id can safely name a durable segment and WAL file. +func ValidID(id string) bool { return segmentIDPattern.MatchString(id) } + // newSegmentDecoder builds a decoder bounded to segmentDecoderMaxMemory. Every // segment read goes through it, so no on-disk frame can size an allocation. func newSegmentDecoder() (*zstd.Decoder, error) { @@ -251,6 +257,7 @@ func (s *Store) Append(rows []Span) error { name := fmt.Sprintf("%020d.fseg", id) tmp := filepath.Join(s.dir, name+".tmp") final := filepath.Join(s.dir, name) + _ = os.Remove(tmp) seg, err := s.writeSegment(tmp, rows) if err != nil { _ = os.Remove(tmp) @@ -964,7 +971,7 @@ func (s *Store) readColumns(f *os.File, block blockDir, wanted []int) (map[int][ return columns, err } -// isErrorStatus matches both status spellings the lake carries: OTLP ingest +// isErrorStatus matches both status spellings telemetry carries: OTLP ingest // stores Status.Code.String() ("STATUS_CODE_ERROR"), while other producers and // older rows use the bare code. The DuckDB rollups compare against the same // pair. diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 9f77a5c0..be9daec5 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -28,6 +28,8 @@ const minCompactionInputs = 8 var parquetSignals = [...]string{"spans", "logs", "metrics"} +var renameCompactionFile = os.Rename + // CompactParquet combines the oldest small atomic batches into larger files. // A durable marker makes the multi-signal swap recoverable after a crash. func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { @@ -222,14 +224,20 @@ func (r *Repository) recoverCompaction() error { return r.completeCompaction(marker) } -func (r *Repository) completeCompaction(marker compactionMarker) error { +func (r *Repository) completeCompaction(marker compactionMarker) (resultErr error) { r.commitMu.Lock() defer r.commitMu.Unlock() r.mu.Lock() defer r.mu.Unlock() stageDir := filepath.Join(r.root, marker.ID) + committed := r.batchConsumedLocked(marker.ID) + defer func() { + if resultErr != nil && !committed { + resultErr = errors.Join(resultErr, r.rollbackCompactionSwap(marker, stageDir)) + } + }() if err := r.validateCompactionOutputs(marker, stageDir); err != nil { - return errors.Join(err, r.restoreCompactionInputs(marker)) + return err } for _, signal := range marker.Signals { dir := filepath.Join(r.Parquet.Dir(), signal) @@ -237,7 +245,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { input := filepath.Join(dir, id+".parquet") retired := input + ".retired-" + marker.ID if _, err := os.Stat(input); err == nil { - if err := os.Rename(input, retired); err != nil { + if err := renameCompactionFile(input, retired); err != nil { return err } } else if !errors.Is(err, os.ErrNotExist) { @@ -247,7 +255,7 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { stage := filepath.Join(stageDir, signal+".parquet") final := filepath.Join(dir, marker.ID+".parquet") if _, err := os.Stat(stage); err == nil { - if err := os.Rename(stage, final); err != nil { + if err := renameCompactionFile(stage, final); err != nil { return err } } @@ -274,11 +282,15 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { } } kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation, Sources: append([]string(nil), marker.Sources...)}) - next := repositoryManifest{Version: 1, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} - if err := writeRepositoryManifest(r.root, next); err != nil { + next := repositoryManifest{Version: repositoryVersion, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} + if err := r.checkpointManifestLocked(next); err != nil { + // checkpointManifestLocked may fail while truncating the superseded journal + // after the new snapshot is already durable and installed in memory. In that + // case the compacted files are committed and must not be rolled back. + committed = r.batchConsumedLocked(marker.ID) return err } - r.manifest = next + committed = true for _, signal := range marker.Signals { for _, id := range marker.Inputs { _ = os.Remove(filepath.Join(r.Parquet.Dir(), signal, id+".parquet.retired-"+marker.ID)) @@ -291,6 +303,39 @@ func (r *Repository) completeCompaction(marker compactionMarker) error { return syncDirectory(r.root) } +func (r *Repository) rollbackCompactionSwap(marker compactionMarker, stageDir string) error { + var rollbackErr error + for _, signal := range marker.Signals { + stage := filepath.Join(stageDir, signal+".parquet") + final := filepath.Join(r.Parquet.Dir(), signal, marker.ID+".parquet") + finalExists, err := pathExists(final) + if err != nil { + rollbackErr = errors.Join(rollbackErr, err) + continue + } + if !finalExists { + continue + } + stageExists, err := pathExists(stage) + if err != nil { + rollbackErr = errors.Join(rollbackErr, err) + continue + } + if stageExists { + if err := os.Remove(final); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } + } else if err := os.Rename(final, stage); err != nil { + rollbackErr = errors.Join(rollbackErr, err) + } + } + rollbackErr = errors.Join(rollbackErr, r.restoreCompactionInputs(marker)) + if err := syncDirectory(stageDir); err != nil && !errors.Is(err, os.ErrNotExist) { + rollbackErr = errors.Join(rollbackErr, err) + } + return rollbackErr +} + func (r *Repository) validateCompactionOutputs(marker compactionMarker, stageDir string) error { if len(marker.Signals) == 0 { return errors.New("compaction marker has no required signals") diff --git a/internal/telemetry/store/publication_test.go b/internal/telemetry/store/publication_test.go new file mode 100644 index 00000000..18434704 --- /dev/null +++ b/internal/telemetry/store/publication_test.go @@ -0,0 +1,59 @@ +package store + +import ( + "os" + "path/filepath" + "testing" + + "github.com/labstack/fanout/internal/telemetry" +) + +type publicationInspectLock struct { + t *testing.T + repository *Repository + id string + locked bool +} + +func (l *publicationInspectLock) Lock() { + l.locked = true + for _, signal := range []string{"spans", "logs", "metrics"} { + final := filepath.Join(l.repository.Parquet.Dir(), signal, l.id+".parquet") + if _, err := os.Stat(final); !os.IsNotExist(err) { + l.t.Fatalf("%s became visible before publication lock: %v", signal, err) + } + if _, err := os.Stat(final + ".pending"); err != nil { + l.t.Fatalf("%s was not durably staged before publication lock: %v", signal, err) + } + } +} + +func (l *publicationInspectLock) Unlock() { l.locked = false } + +func TestCommitStagesOutsidePublicationLock(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + const id = "atomic-batch" + lock := &publicationInspectLock{t: t, repository: repository, id: id} + repository.SetParquetPublishLock(lock) + batch := Batch{ + ID: id, + Spans: []telemetry.Span{{Namespace: "default", TraceID: "00000000000000000000000000000001", SpanID: "0000000000000001"}}, + Logs: []telemetry.Log{{Namespace: "default", Body: "body"}}, + Metrics: []telemetry.Metric{{Namespace: "default", Name: "metric"}}, + } + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + if lock.locked { + t.Fatal("publication lock remained held after commit") + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(repository.Parquet.Dir(), signal, id+".parquet")); err != nil { + t.Fatalf("%s final file: %v", signal, err) + } + } +} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index ebd15d3d..683a764e 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -31,8 +31,10 @@ type Batch struct { } const ( - maxBatchRows = 50_000 - walDecoderMaxMemory = 128 << 20 + maxBatchRows = 50_000 + walDecoderMaxMemory = 128 << 20 + repositoryVersion = 2 + manifestCheckpointRecords = 4_096 ) type batchMetadata struct { @@ -50,41 +52,50 @@ type batchMetadata struct { type repositoryManifest struct { Version uint32 `json:"version"` + Epoch uint64 `json:"epoch"` HotCutoffNanos int64 `json:"hot_cutoff_nanos"` Batches []batchMetadata `json:"batches"` } +type repositoryJournalRecord struct { + Epoch uint64 `json:"epoch"` + Batch batchMetadata `json:"batch"` +} + type Repository struct { mu sync.RWMutex // hotMu makes the persisted prune watermark and the hot-segment snapshot one // atomic read boundary. A query can never observe an old watermark after the // corresponding segments have been retired. - hotMu sync.RWMutex - commitMu sync.Mutex - compactionMu sync.Mutex - root string - walDir string - Spans *segment.Store - Logs *segment.SignalStore[telemetry.Log] - Metrics *segment.SignalStore[telemetry.Metric] - Parquet *telemetry.ParquetStore - manifest repositoryManifest + hotMu sync.RWMutex + stageMu sync.Mutex + commitMu sync.Mutex + compactionMu sync.Mutex + parquetPublish sync.Locker + root string + walDir string + Spans *segment.Store + Parquet *telemetry.ParquetStore + manifest repositoryManifest + consumed map[string]struct{} + journal *os.File + journalRecords int +} + +// SetParquetPublishLock connects repository publication to the query engine's +// read gate. It must be called during startup, before the commit worker runs. +func (r *Repository) SetParquetPublishLock(lock sync.Locker) { + r.parquetPublish = lock } func Open(root string) (*Repository, error) { - legacyCatalog := filepath.Join(root, "ducklake.sqlite") - if _, err := os.Stat(legacyCatalog); err == nil { - return nil, fmt.Errorf("legacy DuckLake catalog %s is unsupported; start Fanout with a clean storage.data_dir", legacyCatalog) - } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("inspect legacy DuckLake catalog: %w", err) - } walDir := filepath.Join(root, "wal") - for _, dir := range []string{root, walDir, filepath.Join(root, "hot", "spans"), filepath.Join(root, "hot", "logs"), filepath.Join(root, "hot", "metrics")} { + for _, dir := range []string{root, walDir, filepath.Join(root, "hot", "spans")} { if err := os.MkdirAll(dir, 0o755); err != nil { return nil, err } } - spans, logs, metricsStore, err := openHotStores(root) + spans, err := openHotStore(root) hotRebuilt := false if err != nil { quarantine := filepath.Join(root, fmt.Sprintf("hot.corrupt-%d", time.Now().UnixNano())) @@ -94,7 +105,7 @@ func Open(root string) (*Repository, error) { if syncErr := syncDirectory(root); syncErr != nil { return nil, fmt.Errorf("sync quarantined hot tier: %w", syncErr) } - spans, logs, metricsStore, err = openHotStores(root) + spans, err = openHotStore(root) if err != nil { return nil, fmt.Errorf("rebuild hot telemetry tier: %w", err) } @@ -103,19 +114,17 @@ func Open(root string) (*Repository, error) { } parquet, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { - _ = metricsStore.Close() - _ = logs.Close() _ = spans.Close() return nil, err } - r := &Repository{root: root, walDir: walDir, Spans: spans, Logs: logs, Metrics: metricsStore, Parquet: parquet} + r := &Repository{root: root, walDir: walDir, Spans: spans, Parquet: parquet} if err := r.loadManifest(); err != nil { _ = r.Close() return nil, fmt.Errorf("load telemetry manifest: %w", err) } if hotRebuilt { r.manifest.HotCutoffNanos = max(r.manifest.HotCutoffNanos, time.Now().UnixNano()) - if err := writeRepositoryManifest(r.root, r.manifest); err != nil { + if err := r.checkpointManifestLocked(r.manifest); err != nil { _ = r.Close() return nil, fmt.Errorf("publish rebuilt hot-tier cutoff: %w", err) } @@ -131,38 +140,24 @@ func Open(root string) (*Repository, error) { return r, nil } -func openHotStores(root string) (*segment.Store, *segment.SignalStore[telemetry.Log], *segment.SignalStore[telemetry.Metric], error) { - hot := filepath.Join(root, "hot") - for _, signal := range []string{"spans", "logs", "metrics"} { - if err := os.MkdirAll(filepath.Join(hot, signal), 0o755); err != nil { - return nil, nil, nil, err - } - } - spans, err := segment.Open(filepath.Join(hot, "spans")) - if err != nil { - return nil, nil, nil, err - } - logs, err := segment.OpenSignalStore[telemetry.Log](filepath.Join(hot, "logs"), "EventUnixNanos") - if err != nil { - _ = spans.Close() - return nil, nil, nil, err - } - metricsStore, err := segment.OpenSignalStore[telemetry.Metric](filepath.Join(hot, "metrics"), "EventUnixNanos") - if err != nil { - _ = logs.Close() - _ = spans.Close() - return nil, nil, nil, err - } - return spans, logs, metricsStore, nil +func openHotStore(root string) (*segment.Store, error) { + return segment.Open(filepath.Join(root, "hot", "spans")) } func (r *Repository) Close() error { - return errors.Join(r.Spans.Close(), r.Logs.Close(), r.Metrics.Close()) + var journalErr error + if r.journal != nil { + journalErr = r.journal.Close() + r.journal = nil + } + return errors.Join(journalErr, r.Spans.Close()) } // PruneHot removes acceleration segments older than cutoff. Parquet remains // authoritative for longer retention and SQL queries. func (r *Repository) PruneHot(cutoff int64) (int, error) { + r.commitMu.Lock() + defer r.commitMu.Unlock() r.hotMu.Lock() defer r.hotMu.Unlock() @@ -170,17 +165,14 @@ func (r *Repository) PruneHot(cutoff int64) (int, error) { // therefore create only harmless overlap (Parquet below the boundary and hot // segments above it), never a hole after restart. publishCutoff := func() error { - r.commitMu.Lock() - defer r.commitMu.Unlock() r.mu.Lock() defer r.mu.Unlock() if cutoff > r.manifest.HotCutoffNanos { - next := r.manifest + next := cloneRepositoryManifest(r.manifest) next.HotCutoffNanos = cutoff - if err := writeRepositoryManifest(r.root, next); err != nil { + if err := r.checkpointManifestLocked(next); err != nil { return err } - r.manifest = next } return nil } @@ -188,24 +180,17 @@ func (r *Repository) PruneHot(cutoff int64) (int, error) { return 0, fmt.Errorf("publish hot prune cutoff: %w", err) } - spans, spanErr := r.Spans.PruneBefore(cutoff) - logs, logErr := r.Logs.PruneBefore(cutoff) - metricRows, metricErr := r.Metrics.PruneBefore(cutoff) - return spans + logs + metricRows, errors.Join(spanErr, logErr, metricErr) + return r.Spans.PruneBefore(cutoff) } -// CompactHot drains committed raw segments into larger immutable files for all -// three signals. It intentionally excludes any segment not present in the +// CompactHot drains committed raw span-index segments into larger immutable +// files. It intentionally excludes any segment not present in the // repository manifest, because that file may belong to a partially applied WAL // transaction that still needs exact-ID replay. func (r *Repository) CompactHot(maxInputs int) (int, error) { if maxInputs < 2 { return 0, nil } - r.hotMu.Lock() - defer r.hotMu.Unlock() - r.commitMu.Lock() - defer r.commitMu.Unlock() r.mu.RLock() committed := make(map[string]struct{}, len(r.manifest.Batches)) for _, batch := range r.manifest.Batches { @@ -225,45 +210,20 @@ func (r *Repository) CompactHot(maxInputs int) (int, error) { break } } - for { - n, err := r.Logs.CompactCommitted(committed, maxInputs) - total += n - compactErr = errors.Join(compactErr, err) - if err != nil || n < 2 { - break - } - } - for { - n, err := r.Metrics.CompactCommitted(committed, maxInputs) - total += n - compactErr = errors.Join(compactErr, err) - if err != nil || n < 2 { - break - } - } return total, compactErr } -// ScanHotLogs reads the portion of [start,end) that is guaranteed complete in -// the hot tier and returns the durable boundary below which Parquet is -// authoritative. The boundary and scan are serialized with PruneHot. -func (r *Repository) ScanHotLogs(start, end int64, visit func(telemetry.Log) bool) (int64, error) { - r.hotMu.RLock() - defer r.hotMu.RUnlock() - r.mu.RLock() - cutoff := r.manifest.HotCutoffNanos - r.mu.RUnlock() - return cutoff, r.Logs.Scan(max(start, cutoff), end, visit) -} - // HotTrace returns the hot trace snapshot and the durable prune boundary that // was in force for that snapshot. -func (r *Repository) HotTrace(traceID string) ([]telemetry.Span, int64, error) { +func (r *Repository) HotTrace(traceID string, scopeStartNanos int64) ([]telemetry.Span, int64, error) { r.hotMu.RLock() defer r.hotMu.RUnlock() r.mu.RLock() cutoff := r.manifest.HotCutoffNanos r.mu.RUnlock() + if scopeStartNanos < cutoff { + return nil, cutoff, nil + } spans, err := r.Spans.Trace(traceID) return spans, cutoff, err } @@ -304,11 +264,10 @@ func (r *Repository) PruneParquet(cutoff int64) (int, error) { if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { return 0, errors.Join(removeErr, err) } - next := repositoryManifest{Version: 1, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} - if err := writeRepositoryManifest(r.root, next); err != nil { + next := repositoryManifest{Version: repositoryVersion, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} + if err := r.checkpointManifestLocked(next); err != nil { return 0, errors.Join(removeErr, err) } - r.manifest = next return removed, removeErr } @@ -319,19 +278,28 @@ func (r *Repository) Commit(batch Batch) error { if err := validateBatch(batch); err != nil { return err } - // Commits are serialized, but their segment and Parquet fsyncs do not hold - // the repository metadata lock. Each projection has its own atomic publish - // protocol; the WAL keeps a partially applied transaction replayable. - r.commitMu.Lock() - defer r.commitMu.Unlock() - consumed := r.batchConsumedLocked(batch.ID) - if consumed { - return r.removeWAL(batch.ID) + r.stageMu.Lock() + err := r.writeWAL(batch) + r.stageMu.Unlock() + if err != nil { + return err } - if err := r.writeWAL(batch); err != nil { + // Parquet encoding and fsync happen before either the query publication gate + // or commit mutex is acquired. Only the final renames and manifest append are + // serialized with readers and maintenance. + if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { return err } - err := r.apply(batch) + if r.parquetPublish != nil { + r.parquetPublish.Lock() + defer r.parquetPublish.Unlock() + } + r.commitMu.Lock() + defer r.commitMu.Unlock() + if r.batchConsumedLocked(batch.ID) { + return errors.Join(r.Parquet.DiscardBatch(batch.ID), r.removeWAL(batch.ID)) + } + err = r.publish(batch) if err == nil { r.mu.Lock() err = r.recordBatch(batch) @@ -351,9 +319,12 @@ func (r *Repository) Stage(batch Batch) error { if err := validateBatch(batch); err != nil { return err } - r.commitMu.Lock() - defer r.commitMu.Unlock() - consumed := r.batchConsumedLocked(batch.ID) + // WAL publication is independent from projection publication. Keeping this + // lock separate lets the next OTLP request become durable while the commit + // worker writes span indexes and Parquet for an earlier request. + r.stageMu.Lock() + defer r.stageMu.Unlock() + consumed := r.batchConsumed(batch.ID) if consumed { return r.removeWAL(batch.ID) } @@ -379,24 +350,15 @@ func validateBatch(batch Batch) error { return nil } -func (r *Repository) apply(batch Batch) error { - if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { - return fmt.Errorf("commit span segment: %w", err) - } - if err := r.Logs.Append(batch.ID, batch.Logs); err != nil { - return fmt.Errorf("commit log segment: %w", err) - } - if err := r.Metrics.Append(batch.ID, batch.Metrics); err != nil { - return fmt.Errorf("commit metric segment: %w", err) - } - if err := r.Parquet.WriteSpans(batch.ID, batch.Spans); err != nil { - return fmt.Errorf("commit span parquet: %w", err) - } - if err := r.Parquet.WriteLogs(batch.ID, batch.Logs); err != nil { - return fmt.Errorf("commit log parquet: %w", err) +func (r *Repository) publish(batch Batch) error { + r.hotMu.Lock() + defer r.hotMu.Unlock() + rollback, err := r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0) + if err != nil { + return fmt.Errorf("publish parquet batch: %w", err) } - if err := r.Parquet.WriteMetrics(batch.ID, batch.Metrics); err != nil { - return fmt.Errorf("commit metric parquet: %w", err) + if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { + return errors.Join(fmt.Errorf("commit span segment: %w", err), rollback()) } return nil } @@ -505,7 +467,7 @@ func (r *Repository) recover() error { } // Poison is decided before anything is published: a payload the // projections can never accept is moved aside so it cannot abort every - // boot. An apply or manifest failure after that point is environmental, + // boot. A publication or manifest failure after that point is environmental, // so the WAL is retained and startup fails loudly — a later healthy boot // must still be able to finish the batch, including one whose projection // prefix this attempt already published. @@ -516,7 +478,10 @@ func (r *Repository) recover() error { } continue } - if err := r.apply(batch); err != nil { + if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { + return fmt.Errorf("stage replay %s: %w", name, err) + } + if err := r.publish(batch); err != nil { return fmt.Errorf("replay %s: %w", name, err) } if err := r.recordBatch(batch); err != nil { @@ -551,41 +516,154 @@ func (r *Repository) loadManifest() error { path := filepath.Join(r.root, "MANIFEST.json") data, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { - r.manifest = repositoryManifest{Version: 1} - return writeRepositoryManifest(r.root, r.manifest) + r.manifest = repositoryManifest{Version: repositoryVersion, Epoch: 1} + if err := writeRepositoryManifest(r.root, r.manifest); err != nil { + return err + } + } else if err != nil { + return err + } else { + if err := json.Unmarshal(data, &r.manifest); err != nil { + return err + } + if r.manifest.Version != repositoryVersion || r.manifest.Epoch == 0 { + return fmt.Errorf("unsupported telemetry manifest version %d epoch %d", r.manifest.Version, r.manifest.Epoch) + } } - if err != nil { + r.rebuildConsumedLocked() + + journalPath := filepath.Join(r.root, "MANIFEST.log") + journalData, err := os.ReadFile(journalPath) + journalNew := errors.Is(err, os.ErrNotExist) + if err != nil && !journalNew { return err } - if err := json.Unmarshal(data, &r.manifest); err != nil { + validBytes := 0 + for validBytes < len(journalData) { + relativeEnd := bytes.IndexByte(journalData[validBytes:], '\n') + if relativeEnd < 0 { + break + } + lineStart := validBytes + lineEnd := lineStart + relativeEnd + line := journalData[lineStart:lineEnd] + validBytes = lineEnd + 1 + if len(bytes.TrimSpace(line)) == 0 { + continue + } + var record repositoryJournalRecord + if err := json.Unmarshal(line, &record); err != nil { + return fmt.Errorf("decode telemetry manifest journal at byte %d: %w", lineStart, err) + } + if record.Epoch != r.manifest.Epoch || r.batchConsumedLocked(record.Batch.ID) { + continue + } + if record.Batch.ID == "" || !segment.ValidID(record.Batch.ID) { + return fmt.Errorf("telemetry manifest journal contains invalid batch ID %q", record.Batch.ID) + } + r.manifest.Batches = append(r.manifest.Batches, record.Batch) + r.addConsumedLocked(record.Batch) + r.journalRecords++ + } + r.journal, err = os.OpenFile(journalPath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644) + if err != nil { return err } - if r.manifest.Version != 1 { - return fmt.Errorf("unsupported telemetry manifest version %d", r.manifest.Version) + if validBytes != len(journalData) { + if err := r.journal.Truncate(int64(validBytes)); err != nil { + return fmt.Errorf("truncate partial telemetry manifest journal: %w", err) + } + if err := r.journal.Sync(); err != nil { + return fmt.Errorf("sync repaired telemetry manifest journal: %w", err) + } + } + if journalNew { + return syncDirectory(r.root) } return nil } func (r *Repository) recordBatch(batch Batch) error { - for _, existing := range r.manifest.Batches { - if existing.ID == batch.ID { - return nil - } - for _, source := range existing.Sources { - if source == batch.ID { - return nil - } + if r.batchConsumedLocked(batch.ID) { + return nil + } + metadata := batchMetadata{ID: batch.ID, MinNanos: batchMinNanos(batch), MaxNanos: batchMaxNanos(batch)} + line, err := json.Marshal(repositoryJournalRecord{Epoch: r.manifest.Epoch, Batch: metadata}) + if err != nil { + return err + } + line = append(line, '\n') + if r.journal == nil { + return errors.New("telemetry manifest journal is closed") + } + if _, err := r.journal.Write(line); err != nil { + return fmt.Errorf("append telemetry manifest journal: %w", err) + } + if err := r.journal.Sync(); err != nil { + return fmt.Errorf("sync telemetry manifest journal: %w", err) + } + r.manifest.Batches = append(r.manifest.Batches, metadata) + r.addConsumedLocked(metadata) + r.journalRecords++ + if r.journalRecords >= manifestCheckpointRecords { + if err := r.checkpointManifestLocked(r.manifest); err != nil { + return fmt.Errorf("checkpoint telemetry manifest journal: %w", err) } } - next := r.manifest - next.Batches = append(append([]batchMetadata(nil), r.manifest.Batches...), batchMetadata{ID: batch.ID, MinNanos: batchMinNanos(batch), MaxNanos: batchMaxNanos(batch)}) + return nil +} + +func (r *Repository) checkpointManifestLocked(next repositoryManifest) error { + next = cloneRepositoryManifest(next) + next.Version = repositoryVersion + next.Epoch = max(next.Epoch, r.manifest.Epoch+1) if err := writeRepositoryManifest(r.root, next); err != nil { return err } r.manifest = next + r.rebuildConsumedLocked() + r.journalRecords = 0 + if r.journal == nil { + return nil + } + if err := r.journal.Truncate(0); err != nil { + return fmt.Errorf("truncate telemetry manifest journal: %w", err) + } + if _, err := r.journal.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind telemetry manifest journal: %w", err) + } + if err := r.journal.Sync(); err != nil { + return fmt.Errorf("sync telemetry manifest journal checkpoint: %w", err) + } return nil } +func cloneRepositoryManifest(manifest repositoryManifest) repositoryManifest { + clone := manifest + clone.Batches = append([]batchMetadata(nil), manifest.Batches...) + for i := range clone.Batches { + clone.Batches[i].Sources = append([]string(nil), manifest.Batches[i].Sources...) + } + return clone +} + +func (r *Repository) rebuildConsumedLocked() { + r.consumed = make(map[string]struct{}, len(r.manifest.Batches)) + for _, batch := range r.manifest.Batches { + r.addConsumedLocked(batch) + } +} + +func (r *Repository) addConsumedLocked(batch batchMetadata) { + if r.consumed == nil { + r.consumed = make(map[string]struct{}) + } + r.consumed[batch.ID] = struct{}{} + for _, source := range batch.Sources { + r.consumed[source] = struct{}{} + } +} + func (r *Repository) batchConsumed(id string) bool { r.mu.RLock() defer r.mu.RUnlock() @@ -593,17 +671,8 @@ func (r *Repository) batchConsumed(id string) bool { } func (r *Repository) batchConsumedLocked(id string) bool { - for _, batch := range r.manifest.Batches { - if batch.ID == id { - return true - } - for _, source := range batch.Sources { - if source == id { - return true - } - } - } - return false + _, exists := r.consumed[id] + return exists } func (r *Repository) removeWAL(id string) error { @@ -684,7 +753,7 @@ func normalizeBatch(batch *Batch) { for i := range batch.Spans { batch.Spans[i].Namespace = telemetry.NormalizeNamespace(batch.Spans[i].Namespace) if batch.Spans[i].StartUnixNanos == 0 { - // Mirror the Parquet start_time coalesce so hot segments and cold SQL + // Mirror the Parquet start_time coalesce so hot segments and SQL scans // key a zero-start span on the same instant. batch.Spans[i].StartUnixNanos = batch.Spans[i].IngestedAt } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 017db66e..d7d62283 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -4,11 +4,11 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "github.com/klauspost/compress/zstd" "os" "path/filepath" - "slices" "strings" "testing" "time" @@ -42,11 +42,10 @@ func TestRepositoryCommitIsIdempotentAndQueryable(t *testing.T) { if got := repository.Spans.RowCount(); got != 1 { t.Fatalf("span rows = %d", got) } - if got := repository.Logs.RowCount(); got != 1 { - t.Fatalf("log rows = %d", got) - } - if got := repository.Metrics.RowCount(); got != 1 { - t.Fatalf("metric rows = %d", got) + for _, signal := range []string{"logs", "metrics"} { + if _, err := os.Stat(filepath.Join(dir, "hot", signal)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unused hot %s copy exists: %v", signal, err) + } } if err := repository.Close(); err != nil { t.Fatal(err) @@ -105,6 +104,112 @@ func TestRepositoryCommitIODoesNotHoldMetadataLock(t *testing.T) { } } +func TestRepositoryStageDoesNotWaitForProjectionCommitLock(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + repository.commitMu.Lock() + staged := make(chan error, 1) + go func() { + staged <- repository.Stage(Batch{ID: "independent-stage", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}) + }() + select { + case err := <-staged: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + repository.commitMu.Unlock() + t.Fatal("WAL staging waited for projection commit I/O") + } + repository.commitMu.Unlock() +} + +func TestCompactHotDoesNotTakeRepositoryIngestOrReadLocks(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + repository.hotMu.Lock() + repository.commitMu.Lock() + done := make(chan error, 1) + go func() { + _, err := repository.CompactHot(2) + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + repository.commitMu.Unlock() + repository.hotMu.Unlock() + t.Fatal("hot compaction acquired a repository-wide ingest or read lock") + } + repository.commitMu.Unlock() + repository.hotMu.Unlock() +} + +func TestRepositoryManifestJournalReplaysAndRepairsPartialTail(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + for i := range 3 { + batch := testBatch() + batch.ID = fmt.Sprintf("journal-%d", i) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + var snapshot repositoryManifest + data, err := os.ReadFile(filepath.Join(dir, "MANIFEST.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &snapshot); err != nil { + t.Fatal(err) + } + if len(snapshot.Batches) != 0 { + t.Fatalf("per-commit path rewrote manifest snapshot with %d batches", len(snapshot.Batches)) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + journalPath := filepath.Join(dir, "MANIFEST.log") + journal, err := os.OpenFile(journalPath, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + if _, err := journal.WriteString(`{"epoch":1,"batch":`); err != nil { + t.Fatal(err) + } + if err := journal.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if len(reopened.manifest.Batches) != 3 || !reopened.batchConsumed("journal-2") { + t.Fatalf("journal replay batches = %#v", reopened.manifest.Batches) + } + repaired, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(repaired), `"batch":`) && !strings.HasSuffix(string(repaired), "}\n") { + t.Fatalf("partial journal tail was not truncated: %q", repaired) + } +} + func TestRepositoryReplaysDurableWAL(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -124,8 +229,13 @@ func TestRepositoryReplaysDurableWAL(t *testing.T) { t.Fatal(err) } defer recovered.Close() - if recovered.Spans.RowCount() != 1 || recovered.Logs.RowCount() != 1 || recovered.Metrics.RowCount() != 1 { - t.Fatal("WAL recovery did not restore every signal") + if recovered.Spans.RowCount() != 1 { + t.Fatal("WAL recovery did not restore the span index") + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if _, err := os.Stat(filepath.Join(dir, "parquet", signal, batch.ID+".parquet")); err != nil { + t.Fatalf("WAL recovery did not restore %s parquet: %v", signal, err) + } } entries, err := filepath.Glob(filepath.Join(dir, "wal", "*.wal")) if err != nil { @@ -136,17 +246,6 @@ func TestRepositoryReplaysDurableWAL(t *testing.T) { } } -func TestRepositoryRejectsLegacyDuckLakeCatalog(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "ducklake.sqlite"), []byte("legacy"), 0o600); err != nil { - t.Fatal(err) - } - _, err := Open(dir) - if err == nil || !strings.Contains(err.Error(), "clean storage.data_dir") { - t.Fatalf("Open error = %v, want explicit clean-data-dir failure", err) - } -} - func TestRepositoryQuarantinesCorruptWAL(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -213,7 +312,7 @@ func TestRepositoryPersistsHotPruneBoundary(t *testing.T) { } batch := testBatch() batch.ID = "hot-boundary" - batch.Logs = []telemetry.Log{{EventUnixNanos: 100}, {EventUnixNanos: 300}} + batch.Spans = []telemetry.Span{{TraceID: "trace-boundary", SpanID: "span", StartUnixNanos: 300}} if err := repository.Commit(batch); err != nil { t.Fatal(err) } @@ -228,16 +327,19 @@ func TestRepositoryPersistsHotPruneBoundary(t *testing.T) { t.Fatal(err) } defer reopened.Close() - var timestamps []int64 - cutoff, err := reopened.ScanHotLogs(0, 400, func(row telemetry.Log) bool { - timestamps = append(timestamps, row.EventUnixNanos) - return true - }) + spans, cutoff, err := reopened.HotTrace("trace-boundary", 250) + if err != nil { + t.Fatal(err) + } + if cutoff != 250 || len(spans) != 1 { + t.Fatalf("cutoff=%d spans=%d, want cutoff 250 and one retained boundary span", cutoff, len(spans)) + } + skipped, cutoff, err := reopened.HotTrace("trace-boundary", 249) if err != nil { t.Fatal(err) } - if cutoff != 250 || !slices.Equal(timestamps, []int64{300}) { - t.Fatalf("cutoff=%d timestamps=%v, want cutoff 250 and only hot timestamp 300", cutoff, timestamps) + if cutoff != 250 || len(skipped) != 0 { + t.Fatalf("cross-boundary lookup cutoff=%d spans=%d, want Parquet handoff without a hot scan", cutoff, len(skipped)) } } @@ -519,6 +621,64 @@ func TestRepositoryCompactionRecoveryRestoresRetiredInputsWhenStageMissing(t *te } } +func TestRepositoryCompactionRollsBackMidSwapFailure(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + marker := compactionMarker{ID: "compact-rollback", Inputs: []string{"rollback-a", "rollback-b"}, Signals: []string{"spans", "logs"}, MinNanos: 100, MaxNanos: 120, Generation: 1} + for _, id := range marker.Inputs { + batch := testBatch() + batch.ID = id + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + stageDir := filepath.Join(dir, marker.ID) + if err := os.Mkdir(stageDir, 0o755); err != nil { + t.Fatal(err) + } + for _, signal := range marker.Signals { + data, err := os.ReadFile(filepath.Join(repository.Parquet.Dir(), signal, marker.Inputs[0]+".parquet")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stageDir, signal+".parquet"), data, 0o644); err != nil { + t.Fatal(err) + } + } + originalRename := renameCompactionFile + renameCompactionFile = func(oldPath, newPath string) error { + if oldPath == filepath.Join(stageDir, "logs.parquet") { + return errors.New("injected log publish failure") + } + return os.Rename(oldPath, newPath) + } + defer func() { renameCompactionFile = originalRename }() + if err := repository.completeCompaction(marker); err == nil || !strings.Contains(err.Error(), "injected log publish failure") { + t.Fatalf("complete compaction error = %v", err) + } + for _, signal := range marker.Signals { + for _, id := range marker.Inputs { + input := filepath.Join(repository.Parquet.Dir(), signal, id+".parquet") + if _, err := os.Stat(input); err != nil { + t.Fatalf("restored %s input %s: %v", signal, id, err) + } + if _, err := os.Stat(input + ".retired-" + marker.ID); !os.IsNotExist(err) { + t.Fatalf("retired %s input %s remains: %v", signal, id, err) + } + } + if _, err := os.Stat(filepath.Join(repository.Parquet.Dir(), signal, marker.ID+".parquet")); !os.IsNotExist(err) { + t.Fatalf("partial compacted %s output remains: %v", signal, err) + } + if _, err := os.Stat(filepath.Join(stageDir, signal+".parquet")); err != nil { + t.Fatalf("restaged %s output: %v", signal, err) + } + } +} + func testBatchAt(timestamp int64) Batch { batch := testBatch() batch.Spans[0].StartUnixNanos = timestamp @@ -685,7 +845,7 @@ func TestWALWriterRejectsBatchLargerThanDecoderBudget(t *testing.T) { } } -func TestCompactHotCompactsEverySignalAndPreservesRows(t *testing.T) { +func TestCompactHotCompactsSpanIndexAndPreservesRows(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) @@ -709,14 +869,8 @@ func TestCompactHotCompactsEverySignalAndPreservesRows(t *testing.T) { if got := repository.Spans.SegmentCount(); got != 2 { t.Fatalf("span segments = %d, want 2 compacted files", got) } - if got := repository.Logs.SegmentCount(); got != 2 { - t.Fatalf("log segments = %d, want 2 compacted files", got) - } - if got := repository.Metrics.SegmentCount(); got != 2 { - t.Fatalf("metric segments = %d, want 2 compacted files", got) - } - if repository.Spans.RowCount() != 6 || repository.Logs.RowCount() != 6 || repository.Metrics.RowCount() != 6 { - t.Fatal("hot compaction changed row counts") + if repository.Spans.RowCount() != 6 { + t.Fatal("hot span compaction changed row counts") } } diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 6b02c077..964a7a56 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -9,12 +9,10 @@ import ( "github.com/google/uuid" "github.com/labstack/fanout/internal/metrics" - "github.com/labstack/fanout/internal/telemetry" ) const ( - flushQueueDepth = 4 - carryBatches = 3 + commitQueueDepth = 4 commitRetryLimit = 8 writerShutdownGrace = 5 * time.Second submissionQueueDepth = 256 @@ -27,14 +25,7 @@ type batchCommitter interface { type Writer struct { repository batchCommitter - interval time.Duration batchSize int - spans <-chan telemetry.Span - logs <-chan telemetry.Log - metricRows <-chan telemetry.Metric - bufSpans []telemetry.Span - bufLogs []telemetry.Log - bufMetrics []telemetry.Metric retryDelay func(int) time.Duration shutdownGrace time.Duration done chan struct{} @@ -46,8 +37,8 @@ type submission struct { ack chan error } -func NewWriter(repository *Repository, interval time.Duration, batchSize int, spans <-chan telemetry.Span, logs <-chan telemetry.Log, metricRows <-chan telemetry.Metric) *Writer { - return &Writer{repository: repository, interval: interval, batchSize: batchSize, spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth)} +func NewWriter(repository *Repository, batchSize int) *Writer { + return &Writer{repository: repository, batchSize: batchSize, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth)} } func (w *Writer) Wait() { <-w.done } @@ -79,21 +70,13 @@ func (w *Writer) Submit(ctx context.Context, batch Batch) error { func (w *Writer) Run(ctx context.Context) error { defer close(w.done) - flushes := make(chan Batch, flushQueueDepth) + commits := make(chan Batch, commitQueueDepth) workerDone := make(chan error, 1) workerCtx, cancelWorker := context.WithCancel(context.Background()) defer cancelWorker() - go w.flushWorker(workerCtx, flushes, workerDone) - ticker := time.NewTicker(w.interval) - defer ticker.Stop() - spans, logs, metricRows := w.spans, w.logs, w.metricRows - legacyInputs := spans != nil || logs != nil || metricRows != nil + go w.commitWorker(workerCtx, commits, workerDone) finish := func() error { - w.drain(&spans, &logs, &metricRows) - if err := w.flushBuffered(flushes, workerDone, true); err != nil { - return err - } - close(flushes) + close(commits) return <-workerDone } finishBounded := func() error { @@ -108,32 +91,7 @@ func (w *Writer) Run(ctx context.Context) error { for { select { case request := <-w.submissions: - if err := w.stageSubmission(request, flushes, workerDone); err != nil { - return err - } - case row, ok := <-spans: - if !ok { - spans = nil - } else { - w.bufSpans = append(w.bufSpans, row) - metrics.RecordIngest("spans", 1) - } - case row, ok := <-logs: - if !ok { - logs = nil - } else { - w.bufLogs = append(w.bufLogs, row) - metrics.RecordIngest("logs", 1) - } - case row, ok := <-metricRows: - if !ok { - metricRows = nil - } else { - w.bufMetrics = append(w.bufMetrics, row) - metrics.RecordIngest("metrics", 1) - } - case <-ticker.C: - if err := w.flush(flushes, workerDone); err != nil { + if err := w.stageSubmission(request, commits, workerDone); err != nil { return err } case <-ctx.Done(): @@ -141,14 +99,6 @@ func (w *Writer) Run(ctx context.Context) error { case err := <-workerDone: return err } - if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) >= w.batchSize { - if err := w.flush(flushes, workerDone); err != nil { - return err - } - } - if legacyInputs && spans == nil && logs == nil && metricRows == nil { - return finish() - } } } @@ -163,17 +113,51 @@ func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDon draining = false } } - limit := min(w.batchSize, maxBatchRows) - if limit <= 0 { - limit = maxBatchRows - } + limit := w.batchLimit() for len(requests) > 0 { + if batchRows(requests[0].batch) > limit { + oversized := requests[0] + requests = requests[1:] + chunks := splitBatch(oversized.batch, limit) + staged := chunks[:0] + var stageErr error + for _, chunk := range chunks { + chunk.ID = uuid.NewString() + if stageErr = w.repository.Stage(chunk); stageErr != nil { + metrics.FlushErrors.WithLabelValues("stage").Inc() + break + } + staged = append(staged, chunk) + } + if stageErr != nil { + // Already-staged chunks remain replayable. Do not enqueue only a + // prefix for live publication; recovery will publish that durable + // prefix after the storage fault is repaired. + oversized.ack <- stageErr + if len(staged) > 0 { + return fmt.Errorf("stage oversized telemetry request after %d durable chunks: %w", len(staged), stageErr) + } + continue + } + metrics.RecordIngest("spans", len(oversized.batch.Spans)) + metrics.RecordIngest("logs", len(oversized.batch.Logs)) + metrics.RecordIngest("metrics", len(oversized.batch.Metrics)) + oversized.ack <- nil + for _, chunk := range staged { + select { + case out <- chunk: + case err := <-workerDone: + return err + } + } + continue + } batch := Batch{ID: uuid.NewString()} group := make([]submission, 0, len(requests)) rows := 0 for len(requests) > 0 { next := requests[0] - nextRows := len(next.batch.Spans) + len(next.batch.Logs) + len(next.batch.Metrics) + nextRows := batchRows(next.batch) if len(group) > 0 && rows+nextRows > limit { break } @@ -209,44 +193,42 @@ func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDon return nil } -func (w *Writer) flush(out chan<- Batch, workerDone <-chan error) error { - return w.flushBuffered(out, workerDone, false) +func batchRows(batch Batch) int { + return len(batch.Spans) + len(batch.Logs) + len(batch.Metrics) } -// flushBuffered publishes the buffered rows. Until a batch is staged in the -// WAL no durable copy exists, so a failed staging attempt keeps the rows -// buffered for the next tick rather than discarding them — bounded by -// carryBatches so a long storage outage cannot grow the buffer without limit. -// The final flush has no next tick, so there the rows are accounted as dropped. -func (w *Writer) flushBuffered(out chan<- Batch, workerDone <-chan error, final bool) error { - if len(w.bufSpans)+len(w.bufLogs)+len(w.bufMetrics) == 0 { - return nil +func (w *Writer) batchLimit() int { + limit := min(w.batchSize, maxBatchRows) + if limit <= 0 { + return maxBatchRows } - batch := Batch{ID: uuid.NewString(), Spans: append([]telemetry.Span(nil), w.bufSpans...), Logs: append([]telemetry.Log(nil), w.bufLogs...), Metrics: append([]telemetry.Metric(nil), w.bufMetrics...)} - if err := w.repository.Stage(batch); err != nil { - metrics.FlushErrors.WithLabelValues("stage").Inc() - if final { - recordDroppedBatch(batch) - slog.Error("telemetry batch could not be staged durably during shutdown; dropping", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) - w.resetBuffers() - return nil + return limit +} + +// splitBatch partitions one request without copying telemetry payloads. Each +// chunk is independently WAL-safe and no chunk exceeds the projection limit. +func splitBatch(batch Batch, limit int) []Batch { + chunks := make([]Batch, 0, (batchRows(batch)+limit-1)/limit) + for batchRows(batch) > 0 { + chunk := Batch{} + remaining := limit + if count := min(remaining, len(batch.Spans)); count > 0 { + chunk.Spans, batch.Spans = batch.Spans[:count], batch.Spans[count:] + remaining -= count } - slog.Warn("telemetry batch could not be staged durably; retrying on the next flush", "batch_id", batch.ID, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) - w.trimCarry() - return nil - } - select { - case out <- batch: - w.bufSpans = w.bufSpans[:0] - w.bufLogs = w.bufLogs[:0] - w.bufMetrics = w.bufMetrics[:0] - return nil - case err := <-workerDone: - return err + if count := min(remaining, len(batch.Logs)); count > 0 { + chunk.Logs, batch.Logs = batch.Logs[:count], batch.Logs[count:] + remaining -= count + } + if count := min(remaining, len(batch.Metrics)); count > 0 { + chunk.Metrics, batch.Metrics = batch.Metrics[:count], batch.Metrics[count:] + } + chunks = append(chunks, chunk) } + return chunks } -func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- error) { +func (w *Writer) commitWorker(ctx context.Context, in <-chan Batch, done chan<- error) { for { var batch Batch select { @@ -299,10 +281,9 @@ func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- e } } if !committed { - // The batch stays in the WAL. Its projections may be partly published, - // and only replay can finish the transaction and register the batch in - // the manifest, so deleting the WAL here would both lose the rows and - // strand any parquet file the failed attempt already wrote. + // The batch stays in the WAL. Its invisible Parquet staging files may + // already be durable, and replay can finish publication and register the + // batch without re-encoding them. metrics.FlushErrors.WithLabelValues("deferred").Inc() slog.Error("telemetry batch commit failed after bounded retries; stopping ingest with WAL retained for replay", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) done <- fmt.Errorf("commit telemetry batch %s after %d attempts: %w", batch.ID, commitRetryLimit, lastErr) @@ -311,90 +292,7 @@ func (w *Writer) flushWorker(ctx context.Context, in <-chan Batch, done chan<- e } } -func (w *Writer) resetBuffers() { - w.bufSpans = w.bufSpans[:0] - w.bufLogs = w.bufLogs[:0] - w.bufMetrics = w.bufMetrics[:0] -} - -// trimCarry bounds the rows held for a later staging attempt. Once the carry -// exceeds carryBatches worth of rows the oldest are dropped with accounting, -// so an unwritable WAL degrades to visible loss instead of unbounded memory. -func (w *Writer) trimCarry() { - limit := w.batchSize * carryBatches - if limit <= 0 { - return - } - spans, logs, metricRows := 0, 0, 0 - if overflow := len(w.bufSpans) - limit; overflow > 0 { - spans = overflow - w.bufSpans = append(w.bufSpans[:0], w.bufSpans[overflow:]...) - } - if overflow := len(w.bufLogs) - limit; overflow > 0 { - logs = overflow - w.bufLogs = append(w.bufLogs[:0], w.bufLogs[overflow:]...) - } - if overflow := len(w.bufMetrics) - limit; overflow > 0 { - metricRows = overflow - w.bufMetrics = append(w.bufMetrics[:0], w.bufMetrics[overflow:]...) - } - if spans+logs+metricRows == 0 { - return - } - recordDropped(spans, logs, metricRows) - slog.Error("telemetry carry buffer is full; dropping the oldest rows", "spans", spans, "logs", logs, "metrics", metricRows) -} - -func recordDroppedBatch(batch Batch) { - recordDropped(len(batch.Spans), len(batch.Logs), len(batch.Metrics)) -} - -func recordDropped(spans, logs, metricRows int) { - metrics.RowsDropped.WithLabelValues("spans").Add(float64(spans)) - metrics.RowsDropped.WithLabelValues("logs").Add(float64(logs)) - metrics.RowsDropped.WithLabelValues("metrics").Add(float64(metricRows)) -} - func defaultCommitRetryDelay(attempt int) time.Duration { shift := min(attempt, 6) return min(100*time.Millisecond*time.Duration(1< maxBatchRows { + t.Fatalf("staged batch rows = %d", rows) + } + total += batchRows(batch) + } + if total != maxBatchRows+1 { + t.Fatalf("staged rows = %d, want %d", total, maxBatchRows+1) + } } -func (c *recoveringCommitter) Commit(batch Batch) error { +type secondStageFailCommitter struct { + mu sync.Mutex + stages []Batch +} + +func (c *secondStageFailCommitter) Stage(batch Batch) error { c.mu.Lock() defer c.mu.Unlock() - c.calls++ - if c.calls <= c.failures { - return errors.New("disk temporarily unavailable") + if len(c.stages) == 1 { + return errors.New("WAL device failed mid-request") } - c.batches = append(c.batches, batch) + c.stages = append(c.stages, batch) return nil } +func (*secondStageFailCommitter) Commit(Batch) error { return nil } -func TestWriterRetainsBatchAcrossCommitFailures(t *testing.T) { - spans := make(chan telemetry.Span, 1) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} - close(spans) - close(logs) - close(metricRows) - committer := &recoveringCommitter{failures: 6} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return 0 }, +func TestWriterStopsAfterPartialOversizedSubmissionStage(t *testing.T) { + committer := &secondStageFailCommitter{} + w := testWriter(committer, maxBatchRows) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(context.Background()) }() + if err := w.Submit(context.Background(), Batch{Spans: make([]telemetry.Span, maxBatchRows+1)}); err == nil { + t.Fatal("Submit returned nil after only a prefix became durable") } - if err := w.Run(context.Background()); err != nil { - t.Fatalf("Run error = %v", err) + if err := <-runDone; err == nil || !strings.Contains(err.Error(), "durable chunks") { + t.Fatalf("Run error = %v, want fatal partial-stage error", err) } - if committer.calls != 7 { - t.Fatalf("Commit calls = %d, want 7", committer.calls) +} + +func TestWriterRetainsBatchAcrossCommitFailures(t *testing.T) { + committer := &recoveringCommitter{failures: 6} + w := testWriter(committer, 1) + w.retryDelay = func(int) time.Duration { return 0 } + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { + t.Fatal(err) } - if len(committer.staged) != 1 { - t.Fatalf("staged batches = %d, want 1", len(committer.staged)) + cancel() + if err := <-runDone; err != nil { + t.Fatal(err) } - if len(committer.batches) != 1 || len(committer.batches[0].Spans) != 1 { - t.Fatalf("committed batches = %#v, want original batch exactly once", committer.batches) + if committer.calls != 7 || len(committer.staged) != 1 || len(committer.batches) != 1 { + t.Fatalf("calls=%d staged=%d committed=%d", committer.calls, len(committer.staged), len(committer.batches)) } } type durableFailCommitter struct { repository *Repository attempted chan struct{} - staged chan struct{} once sync.Once } -func (c *durableFailCommitter) Stage(batch Batch) error { - if err := c.repository.Stage(batch); err != nil { - return err - } - c.staged <- struct{}{} - return nil -} - +func (c *durableFailCommitter) Stage(batch Batch) error { return c.repository.Stage(batch) } func (c *durableFailCommitter) Commit(Batch) error { c.once.Do(func() { close(c.attempted) }) return errors.New("storage stalled") } -func TestWriterShutdownReplaysEveryStagedBatch(t *testing.T) { +func TestWriterShutdownReplaysEveryAcknowledgedBatch(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } - spans := make(chan telemetry.Span, 5) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - for i := range 5 { - spans <- telemetry.Span{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: int64(100 + i)} - } - committer := &durableFailCommitter{repository: repository, attempted: make(chan struct{}), staged: make(chan struct{}, 5)} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return time.Hour }, shutdownGrace: 25 * time.Millisecond, - } + committer := &durableFailCommitter{repository: repository, attempted: make(chan struct{})} + w := testWriter(committer, 1) + w.retryDelay = func(int) time.Duration { return time.Hour } + w.shutdownGrace = 25 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) finished := make(chan error, 1) go func() { finished <- w.Run(ctx) }() - for range 5 { - select { - case <-committer.staged: - case <-time.After(time.Second): - t.Fatal("queued batch was not staged") + for i := range 5 { + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: int64(100 + i)}}}); err != nil { + t.Fatal(err) } } - select { - case <-committer.attempted: - case <-time.After(time.Second): - t.Fatal("commit attempt did not start") - } + <-committer.attempted cancel() if err := <-finished; !errors.Is(err, context.Canceled) { t.Fatalf("Run error = %v, want context canceled", err) @@ -217,44 +249,33 @@ func TestWriterShutdownReplaysEveryStagedBatch(t *testing.T) { } func TestWriterSurfacesPermanentCommitFailureAfterBoundedRetries(t *testing.T) { - spans := make(chan telemetry.Span, 1) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} - close(spans) - close(logs) - close(metricRows) committer := &recoveringCommitter{failures: commitRetryLimit + 1} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return 0 }, + w := testWriter(committer, 1) + w.retryDelay = func(int) time.Duration { return 0 } + runDone := make(chan error, 1) + go func() { runDone <- w.Run(context.Background()) }() + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { + t.Fatal(err) } - if err := w.Run(context.Background()); err == nil { + if err := <-runDone; err == nil { t.Fatal("Run returned nil after permanent commit failure") } - if committer.calls != commitRetryLimit { - t.Fatalf("Commit calls = %d, want %d", committer.calls, commitRetryLimit) - } - if len(committer.batches) != 0 { - t.Fatalf("committed poison batches = %#v", committer.batches) + if committer.calls != commitRetryLimit || len(committer.batches) != 0 { + t.Fatalf("calls=%d committed=%d", committer.calls, len(committer.batches)) } } func TestWriterCancellationInterruptsCommitBackoff(t *testing.T) { - spans := make(chan telemetry.Span, 1) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} committer := &recoveringCommitter{failures: commitRetryLimit + 1} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return time.Hour }, shutdownGrace: 25 * time.Millisecond, - } + w := testWriter(committer, 1) + w.retryDelay = func(int) time.Duration { return time.Hour } + w.shutdownGrace = 25 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) finished := make(chan error, 1) go func() { finished <- w.Run(ctx) }() + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { + t.Fatal(err) + } deadline := time.Now().Add(time.Second) for { committer.mu.Lock() @@ -279,65 +300,36 @@ func TestWriterCancellationInterruptsCommitBackoff(t *testing.T) { } } -type stageFailCommitter struct { - mu sync.Mutex - stages int - commits int -} +type stageFailCommitter struct{ stages, commits int } func (c *stageFailCommitter) Stage(Batch) error { - c.mu.Lock() - defer c.mu.Unlock() c.stages++ return errors.New("wal device unavailable") } +func (c *stageFailCommitter) Commit(Batch) error { c.commits++; return nil } -func (c *stageFailCommitter) Commit(Batch) error { - c.mu.Lock() - defer c.mu.Unlock() - c.commits++ - return nil -} - -func TestWriterSurvivesStageFailure(t *testing.T) { - spans := make(chan telemetry.Span, 1) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "span"} - close(spans) - close(logs) - close(metricRows) +func TestWriterSurvivesRequestStageFailure(t *testing.T) { committer := &stageFailCommitter{} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return 0 }, - } - if err := w.Run(context.Background()); err != nil { - t.Fatalf("Run error = %v, want nil: an unstageable batch must be dropped with accounting, not kill the writer", err) + w := testWriter(committer, 1) + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err == nil { + t.Fatal("Submit returned nil after Stage failed") } - if committer.stages == 0 { - t.Fatal("Stage was never attempted") + cancel() + if err := <-runDone; err != nil { + t.Fatal(err) } - if committer.commits != 0 { - t.Fatalf("Commit calls = %d, want 0 for an unstaged batch", committer.commits) + if committer.stages != 1 || committer.commits != 0 { + t.Fatalf("stages=%d commits=%d", committer.stages, committer.commits) } } -type ioFailCommitter struct { - repository *Repository - mu sync.Mutex - attempts int -} +type ioFailCommitter struct{ repository *Repository } func (c *ioFailCommitter) Stage(batch Batch) error { return c.repository.Stage(batch) } - -func (c *ioFailCommitter) Commit(Batch) error { - c.mu.Lock() - defer c.mu.Unlock() - c.attempts++ - return errors.New("storage unavailable") -} +func (*ioFailCommitter) Commit(Batch) error { return errors.New("storage unavailable") } func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { dir := t.TempDir() @@ -345,20 +337,14 @@ func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { if err != nil { t.Fatal(err) } - spans := make(chan telemetry.Span, 1) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100} - close(spans) - close(logs) - close(metricRows) - committer := &ioFailCommitter{repository: repository} - w := &Writer{ - repository: committer, interval: time.Hour, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return 0 }, - } - if err := w.Run(context.Background()); err == nil { + w := testWriter(&ioFailCommitter{repository: repository}, 1) + w.retryDelay = func(int) time.Duration { return 0 } + runDone := make(chan error, 1) + go func() { runDone <- w.Run(context.Background()) }() + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}}); err != nil { + t.Fatal(err) + } + if err := <-runDone; err == nil { t.Fatal("Run returned nil after permanent commit failure") } entries, err := os.ReadDir(filepath.Join(dir, "wal")) @@ -372,14 +358,13 @@ func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { } } if kept != 1 { - t.Fatalf("retained WAL files = %d, want 1: an I/O failure must leave the batch replayable, not delete its only durable copy", kept) + t.Fatalf("retained WAL files = %d, want 1", kept) } } type transientStageCommitter struct { repository *Repository mu sync.Mutex - failures int stages int committed []Batch } @@ -387,72 +372,48 @@ type transientStageCommitter struct { func (c *transientStageCommitter) Stage(batch Batch) error { c.mu.Lock() c.stages++ - fail := c.stages <= c.failures + first := c.stages == 1 c.mu.Unlock() - if fail { + if first { return errors.New("wal device busy") } return c.repository.Stage(batch) } - func (c *transientStageCommitter) Commit(batch Batch) error { if err := c.repository.Commit(batch); err != nil { return err } c.mu.Lock() - defer c.mu.Unlock() c.committed = append(c.committed, batch) + c.mu.Unlock() return nil } -func TestWriterCarriesRowsForwardAcrossTransientStageFailure(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) +func TestWriterAcceptsCallerRetryAfterTransientStageFailure(t *testing.T) { + repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) } defer repository.Close() - spans := make(chan telemetry.Span, 2) - logs := make(chan telemetry.Log) - metricRows := make(chan telemetry.Metric) - spans <- telemetry.Span{TraceID: "trace", SpanID: "a", StartUnixNanos: 100, IngestedAt: 100} - committer := &transientStageCommitter{repository: repository, failures: 1} - w := &Writer{ - repository: committer, interval: 5 * time.Millisecond, batchSize: 1, - spans: spans, logs: logs, metricRows: metricRows, done: make(chan struct{}), - retryDelay: func(int) time.Duration { return 0 }, - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - finished := make(chan error, 1) - go func() { finished <- w.Run(ctx) }() - deadline := time.Now().Add(time.Second) - for { - committer.mu.Lock() - done := len(committer.committed) - committer.mu.Unlock() - if done > 0 { - break - } - if time.Now().After(deadline) { - cancel() - <-finished - t.Fatal("rows were not retried after a transient Stage failure; they were dropped instead of carried forward") - } - time.Sleep(time.Millisecond) + committer := &transientStageCommitter{repository: repository} + w := testWriter(committer, 1) + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + batch := Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}} + if err := w.Submit(context.Background(), batch); err == nil { + t.Fatal("first Submit returned nil") + } + if err := w.Submit(context.Background(), batch); err != nil { + t.Fatal(err) } - close(spans) - close(logs) - close(metricRows) cancel() - <-finished + if err := <-runDone; err != nil { + t.Fatal(err) + } committer.mu.Lock() defer committer.mu.Unlock() - total := 0 - for _, batch := range committer.committed { - total += len(batch.Spans) - } - if total != 1 { - t.Fatalf("committed spans = %d, want the single carried-forward row", total) + if len(committer.committed) != 1 { + t.Fatalf("committed batches = %d, want 1", len(committer.committed)) } } diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index 1d131d37..ac14db82 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -1,71 +1,62 @@ --- title: How storage works -description: What Fanout writes to disk, how it is queried, and why the data directory is one unit. -summary: Parquet under a DuckLake catalog, queried by DuckDB in-process, with SQLite holding application state. +description: What Fanout writes to disk, how it is queried, and why each component exists. +summary: Durable WAL ingestion, authoritative Parquet, DuckDB queries, and a focused hot span index. read_when: - You are sizing disk, or deciding what to back up. - - You want to know why a rollup lags the raw telemetry. + - You want to understand ingest durability or query performance. status: preview --- -Telemetry lands on disk as Parquet. A DuckLake catalog tracks those files, and -DuckDB queries them in the same process that wrote them. Application state — -users, sessions, dashboards, alert rules, agent history — lives in a separate -SQLite database. +Fanout uses a small embedded storage stack, not a data lake. Telemetry is retained +as ordinary Parquet files and queried by DuckDB in the same process. Application +state—users, sessions, dashboards, alert rules, and agent history—lives in SQLite. -Under the data directory: - -| Path | Holds | +| Component | Purpose | |---|---| -| `telemetry/parquet` | The telemetry itself | -| `telemetry/ducklake.sqlite` | The catalog tracking those files | -| `query/catalog.duckdb` | Query-side state | -| `control/fanout.sqlite` | Users, sessions, dashboards, alerts, agent history | - -These reference each other, which is why [backup](/guides/back-up-and-restore) -takes the whole directory. A copy of `telemetry/parquet` without its catalog is -a directory of files nothing can find. - -## Why two databases +| WAL | Acknowledges an OTLP request only after its complete bounded batch is durable | +| Parquet | Authoritative retained spans, logs, and metrics | +| DuckDB | SQL, log filtering, aggregation, and broad analytical reads | +| Hot span index | Fast trace lookup and service/endpoint rollups for recent spans | +| Manifest journal | Constant-time commit ledger and crash-safe Parquet file lifecycle | +| SQLite | Transactional application and identity state | -They have different shapes. Telemetry is append-heavy, queried analytically over -wide time ranges, and never updated — which is what Parquet and DuckDB are good -at. Application state is small, transactional, and updated constantly, which -they are not. Using one engine for both would make one of the two jobs worse. +There is no Iceberg, DuckLake, external catalog, or server database in the +telemetry path. -## Writes are batched +## Writes are durable immediately -Incoming telemetry is buffered and flushed on `FANOUT_FLUSH_INTERVAL` or when -`FANOUT_FLUSH_BATCH_SIZE` rows accumulate. Nothing is queryable until it is -flushed, so the flush interval is the floor on how quickly new telemetry becomes -visible — and the amount of data at risk if the process dies uncleanly. +Each decoded OTLP request goes directly to the WAL. Concurrent small requests +may share one group commit; a request larger than `FANOUT_INGEST_BATCH_SIZE` is +split into independently recoverable chunks. Fanout returns success only after +every chunk is fsynced. -## Small files are the thing to manage +An asynchronous worker then writes Parquet and updates the hot span index. A +crash or permanent I/O failure leaves the WAL in place, and startup replays it +idempotently. There is no timer-based flush window and no acknowledged telemetry +that exists only in memory. -Frequent flushes produce many small Parquet files, and scan cost tracks file -count as much as it tracks bytes. Two passes address it: a frequent merge that -consolidates the newest small files and deletes nothing, and an hourly -maintenance cycle that also applies retention. [Tuning -retention](/guides/tune-retention) covers both. +## One authoritative copy for logs and metrics -## Rollups lag, on purpose +Logs and metrics are written once, to Parquet. DuckDB applies filters, ordering, +`LIMIT`, and aggregation inside its vectorized scan. Keeping separate custom hot +copies would add write latency and compaction work without serving a production +query. -The overview and the alert engine read pre-aggregated rollups rather than raw -telemetry, recomputed on `FANOUT_ROLLUP_INTERVAL`. A rollup deliberately trails -the newest data by a safety margin, because telemetry arrives out of order — -aggregating right up to the current instant would produce numbers that change -after the fact as late spans land. +Spans additionally use a compact purpose-built index because trace lookup and +service rollups benefit from it. Parquet remains authoritative when the recent +span index is pruned or rebuilt. -The practical consequence: the newest few seconds are queryable as raw -telemetry before they appear in an overview or fire an alert. That is a -correctness choice, not lag to be tuned away. +## Small files are bounded by compaction -## Everything serialises through one write gate +Request-level durability can create many Parquet files. Maintenance drains every +eligible compaction group, combines files within bounded day/generation levels, +and applies retention. Active DuckDB readers pin their immutable files while a +publish is in progress, so reads see a consistent set. -Flushes, rollups and maintenance all commit through a shared gate, so DuckDB's -single writer is never contended. Reads run concurrently — the catalog is opened -in WAL mode, which is what makes a connection pool larger than one safe. +## Rollups lag deliberately -This is the mechanism behind the trade in [why one -binary](/explanation/why-one-binary): maintenance that runs harder takes gate -time that ingest is not getting. +Overview and alert queries read rebuildable DuckDB rollups. Their watermark +trails committed telemetry by a fixed safety window so out-of-order events from +bounded commit retries are not skipped. Raw committed telemetry is queryable +before it appears in a rollup. diff --git a/site/src/content/docs/guides/back-up-and-restore.mdx b/site/src/content/docs/guides/back-up-and-restore.mdx index 328ff540..5c6c466e 100644 --- a/site/src/content/docs/guides/back-up-and-restore.mdx +++ b/site/src/content/docs/guides/back-up-and-restore.mdx @@ -11,18 +11,17 @@ status: shipped Fanout is one process and one persistent data directory. Back up the directory as a unit. -That is not a simplification. The directory holds the telemetry catalog and its -Parquet files, query state, and the control SQLite database, and they reference -each other. Copying one subdirectory produces something that looks like a backup -and does not restore. +The directory holds the telemetry WAL, commit manifest, Parquet files, +rebuildable query state, and the control SQLite database. Copying one +subdirectory produces something that looks like a backup and does not restore. ## Take a cold backup 1. Record the running version and configuration, keeping secrets out of ordinary logs and tickets. 2. Stop Fanout cleanly and wait for the process to exit. Shutdown closes both - OTLP listeners before draining the lake writer, so a clean exit is what makes - the files on disk consistent. + OTLP listeners before draining the telemetry commit worker, so a clean exit + leaves no accepted request waiting for publication. 3. Copy or snapshot the complete data directory, preserving ownership and permissions. 4. Start Fanout and confirm `/readyz`. @@ -32,8 +31,8 @@ and does not restore. :::caution[Not while it is running] Do not take a recursive copy of a live data directory. A storage-level snapshot is acceptable only if it gives a crash-consistent point-in-time view of the -whole directory at once — a copy that walks the tree while the writer moves -through it captures a catalog and Parquet files from different moments. +whole directory at once — a copy that walks the tree while commits publish can +capture related files from different moments. ::: ## Restore diff --git a/site/src/content/docs/guides/tune-retention.mdx b/site/src/content/docs/guides/tune-retention.mdx index 58e276b0..1c2bae68 100644 --- a/site/src/content/docs/guides/tune-retention.mdx +++ b/site/src/content/docs/guides/tune-retention.mdx @@ -46,25 +46,26 @@ If query latency is climbing on an instance whose data volume has not changed, this is the first thing to look at: a high file count from many small writes costs more per scan than the same bytes in fewer files. -## Flush behaviour +## Publication batching -How often data reaches disk in the first place: +The maximum number of telemetry rows in one WAL and Parquet batch: ```sh -FANOUT_FLUSH_INTERVAL=15s -FANOUT_FLUSH_BATCH_SIZE=50000 +FANOUT_INGEST_BATCH_SIZE=50000 ``` -A longer flush interval produces fewer, larger files — less compaction work, at -the cost of newly ingested telemetry taking longer to become queryable, and more -of it being lost if the process dies uncleanly. +A larger batch can improve sustained write throughput and create fewer files +under concurrent ingest. Every request is still acknowledged only after its WAL +record is durable; this setting does not create a timer window or put accepted +telemetry at risk. Requests larger than the limit are split into recoverable +chunks. ## What the knobs interact with -Maintenance and rollups serialise against ingest through the same write gate. -Running maintenance much more often on a busy instance therefore trades disk -against ingest headroom rather than being free. Change one setting at a time and -watch `/-/metrics`. +Maintenance briefly gates Parquet publication while it swaps immutable files; +WAL acknowledgement and Parquet encoding continue independently. Running +maintenance much more often still creates extra disk and CPU work, so change +one setting at a time and watch `/-/metrics`. The full list, with defaults and types, is in the [storage settings](/reference/settings/storage) reference. diff --git a/site/src/content/docs/reference/data-layout.mdx b/site/src/content/docs/reference/data-layout.mdx index 6415ee33..406212e3 100644 --- a/site/src/content/docs/reference/data-layout.mdx +++ b/site/src/content/docs/reference/data-layout.mdx @@ -13,17 +13,20 @@ Everything Fanout persists lives under `FANOUT_DATA_DIR` (`./data` by default, | Path | Holds | |---|---| -| `telemetry/parquet/` | The telemetry itself, as Parquet files | -| `telemetry/ducklake.sqlite` | The DuckLake catalog tracking those files | -| `query/catalog.duckdb` | Query-side DuckDB state | +| `telemetry/wal/` | Durable requests waiting for publication or cleanup | +| `telemetry/parquet/{spans,logs,metrics}/` | Authoritative retained telemetry | +| `telemetry/hot/spans/` | Rebuildable recent-span index | +| `telemetry/MANIFEST.json` | Checkpoint of published WAL batches | +| `telemetry/MANIFEST.log` | Append-only commit journal since the checkpoint | +| `query/catalog.duckdb` | Rebuildable DuckDB rollups and query state | | `query/tmp/` | Spill space for queries that exceed the memory cap | | `control/fanout.sqlite` | Users, sessions, dashboards, alert rules, agent history | ## Why it is one unit -The catalog records which Parquet files exist and what they contain. Separated -from it, `telemetry/parquet/` is a directory of files nothing can locate, and -the catalog alone is an index of files that are not there. +The WAL, Parquet files, and manifest form one crash-recovery unit. The hot span +index and DuckDB rollups can be rebuilt, but keeping the complete directory is +the supported and fastest restore path. `control/fanout.sqlite` is independent of both in format but not in meaning: it holds the dashboards and alert rules that refer to the telemetry, and the @@ -33,11 +36,11 @@ So the unit of backup is the whole directory. [Back up and restore](/guides/back-up-and-restore) covers doing that safely, which mostly means doing it while the process is stopped. -## Two databases, on purpose +## Two engines, on purpose Telemetry is append-heavy, queried analytically over wide ranges, and never -updated. Application state is small, transactional and updated constantly. One -engine would serve one of those two badly — [how storage +updated. Application state is small, transactional and updated constantly. +Parquet with DuckDB serves the former; SQLite serves the latter — [how storage works](/explanation/storage-model) covers the reasoning and the write path. ## `query/tmp/` diff --git a/site/src/content/docs/reference/settings/ingest.mdx b/site/src/content/docs/reference/settings/ingest.mdx index 2fda2a27..63d3cd64 100644 --- a/site/src/content/docs/reference/settings/ingest.mdx +++ b/site/src/content/docs/reference/settings/ingest.mdx @@ -19,9 +19,8 @@ as a refusal to start rather than as a default nobody chose. | Setting | Environment variable | Type | Default | |---|---|---|---| | `ingest.advertised_endpoint` | `FANOUT_INGEST_ADVERTISED_ENDPOINT` | string | — | +| `ingest.batch_size` | `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | | `ingest.default_namespace` | `FANOUT_DEFAULT_NAMESPACE` | string | `default` | -| `ingest.flush_batch_size` | `FANOUT_FLUSH_BATCH_SIZE` | integer | `50000` | -| `ingest.flush_interval` | `FANOUT_FLUSH_INTERVAL` | duration | `15s` | | `ingest.otlp_grpc_addr` | `FANOUT_OTLP_GRPC_ADDR` | string | `127.0.0.1:4317` | | `ingest.otlp_http_addr` | `FANOUT_OTLP_HTTP_ADDR` | string | `127.0.0.1:4318` | diff --git a/site/src/content/docs/start/first-boot.mdx b/site/src/content/docs/start/first-boot.mdx index f55f29c1..4d177d59 100644 --- a/site/src/content/docs/start/first-boot.mdx +++ b/site/src/content/docs/start/first-boot.mdx @@ -19,7 +19,7 @@ Every duration and size has a floor, and the auth settings are checked as a group rather than individually: - Addresses and the data directory must not be empty. -- `FANOUT_FLUSH_INTERVAL` and `FANOUT_ROLLUP_INTERVAL` must be at least `1s`; +- `FANOUT_ROLLUP_INTERVAL` must be at least `1s`; `FANOUT_MERGE_INTERVAL` must be `0s` or at least `1s`. - `FANOUT_AUTH_MODE` must be `local` or `oidc`. - `FANOUT_SESSION_IDLE_TTL` must be at least `5m`, and the absolute TTL must be From f18a19d9c4c61ab783ec6b92b1cbb97189783e56 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 06:45:26 -0700 Subject: [PATCH 12/31] perf(storage)!: bound hot index memory Keep trace indexes disk-resident and remove unused segment rollups so hot retention does not scale heap with trace cardinality. Limit the publish gate to Parquet visibility changes and harden write and recovery boundaries. BREAKING CHANGE: hot span segments use format v4. Existing hot indexes are discarded; authoritative Parquet remains intact. --- bench/storage/main.go | 24 +- docs/storage-architecture-options.md | 117 ++-- docs/storage-benchmark.md | 37 +- internal/api/health.go | 5 +- internal/config/config.go | 4 +- internal/config/config_test.go | 9 + internal/observability/logs.go | 3 + internal/observability/service_test.go | 69 +- internal/observability/trace.go | 27 +- internal/query/duck.go | 8 +- internal/query/rollup_watermark_test.go | 7 + internal/telemetry/segment/span_columnar.go | 54 +- internal/telemetry/segment/span_store.go | 623 +++++++----------- internal/telemetry/segment/span_store_test.go | 119 ++-- internal/telemetry/store/compaction.go | 8 +- internal/telemetry/store/publication_test.go | 20 +- internal/telemetry/store/repository.go | 49 +- internal/telemetry/store/repository_test.go | 38 +- internal/telemetry/store/writer.go | 2 +- .../docs/explanation/storage-model.mdx | 9 +- 20 files changed, 656 insertions(+), 576 deletions(-) diff --git a/bench/storage/main.go b/bench/storage/main.go index 81fb4b90..f28a3f1f 100644 --- a/bench/storage/main.go +++ b/bench/storage/main.go @@ -104,7 +104,7 @@ func main() { if r.rollupBuild > 0 { rollup = formatDuration(r.rollupBuild) } - fmt.Printf("%-20s %14.0f %13s %12s %12.1f %12s %12s %12s\n", r.name, r.writeRate, rollup, formatDuration(r.maintenance), float64(r.diskBytes)/(1<<20), formatDuration(r.endpoint), formatDuration(r.trace), formatDuration(r.rawService)) + fmt.Printf("%-20s %14.0f %13s %12s %12.1f %12s %12s %12s\n", r.name, r.writeRate, rollup, formatDuration(r.maintenance), float64(r.diskBytes)/(1<<20), formatOptionalDuration(r.endpoint), formatDuration(r.trace), formatDuration(r.rawService)) } fmt.Println("\nMixed load: committed writes plus full trace reads at 100 qps") fmt.Printf("%-20s %14s %14s\n", "storage / execution", "write rows/s", "trace p95") @@ -136,11 +136,6 @@ func runRepository(dir string, total, batch, repeats, mixedRows int, base, start } } writeElapsed := time.Since(writeStart) - var endpointSink []segment.Endpoint - endpoint := median(repeats, func() error { - endpointSink = repository.Spans.Endpoints("default", "service-00", start, end, 20) - return nil - }) var traceSink []segment.Span trace := median(repeats, func() (err error) { traceSink, err = repository.Spans.Trace(targetTrace); return err }) var aggregateSink segment.Aggregate @@ -148,7 +143,7 @@ func runRepository(dir string, total, batch, repeats, mixedRows int, base, start aggregateSink, err = repository.Spans.ScanService("default", "service-00", start, end) return err }) - if len(endpointSink) == 0 || aggregateSink.Calls == 0 { + if aggregateSink.Calls == 0 { return result{}, errors.New("production repository queries returned no rows") } disk, err := directoryBytes(dir) @@ -170,7 +165,7 @@ func runRepository(dir string, total, batch, repeats, mixedRows int, base, start if err != nil { return result{}, err } - return result{name: "Fanout + Parquet", writeRate: float64(total) / writeElapsed.Seconds(), diskBytes: disk, endpoint: endpoint, trace: trace, rawService: raw, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil + return result{name: "Fanout + Parquet", writeRate: float64(total) / writeElapsed.Seconds(), diskBytes: disk, trace: trace, rawService: raw, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil } func directoryBytes(root string) (int64, error) { @@ -224,20 +219,18 @@ func runCustom(dir string, total, batch, repeats, mixedRows int, base, start, en recovery := time.Since(reopenStart) defer store.Close() - var endpointSink []segment.Endpoint - endpoint := median(repeats, func() error { endpointSink = store.Endpoints("default", "service-00", start, end, 20); return nil }) var traceSink []segment.Span trace := median(repeats, func() (err error) { traceSink, err = store.Trace(targetTrace); return err }) var aggSink segment.Aggregate raw := median(repeats, func() (err error) { aggSink, err = store.ScanService("default", "service-00", start, end); return err }) - if len(endpointSink) == 0 || aggSink.Calls == 0 { + if aggSink.Calls == 0 { return result{}, fmt.Errorf("queries returned no rows") } mixedWrite, mixedP95, err := mixedCustom(store, total, mixedRows, batch, base, targetTrace) if err != nil { return result{}, err } - return result{name: "fanseg + direct", writeRate: float64(total) / writeElapsed.Seconds(), maintenance: maintenance, diskBytes: disk, endpoint: endpoint, trace: trace, rawService: raw, recovery: recovery, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil + return result{name: "fanseg + direct", writeRate: float64(total) / writeElapsed.Seconds(), maintenance: maintenance, diskBytes: disk, trace: trace, rawService: raw, recovery: recovery, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil } func runDuck(dbPath, parquetDir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, result, error) { @@ -522,4 +515,11 @@ func formatDuration(value time.Duration) string { return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) } +func formatOptionalDuration(value time.Duration) string { + if value == 0 { + return "n/a" + } + return formatDuration(value) +} + func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-bench:", err); os.Exit(1) } diff --git a/docs/storage-architecture-options.md b/docs/storage-architecture-options.md index 2eff0234..ce4b541d 100644 --- a/docs/storage-architecture-options.md +++ b/docs/storage-architecture-options.md @@ -10,9 +10,10 @@ trace lookup, attribute filtering, retention, and ad-hoc SQL Use a hybrid architecture: -1. **Fanout columnar segments** for hot telemetry. -2. **Direct Fanout execution** for known product queries. -3. **Parquet** as the durable open SQL copy, written in the same commit. +1. **Fanout columnar segments** as a rebuildable recent-span trace index. +2. **Direct Fanout execution** for recent trace lookup. +3. **Parquet** as the authoritative durable telemetry format, written in the + same commit. 4. **DuckDB** for arbitrary SQL over Parquet. 5. **SQLite** for control-plane data only. 6. **Do not use DuckLake, Iceberg, or chDB initially.** @@ -22,11 +23,10 @@ OTLP ingestion │ ▼ Fanout hot columnar store (.fseg) - ├── trace and promoted-attribute indexes - ├── ingestion-time service/endpoint rollups - ├── direct dashboard and trace execution + ├── disk-resident trace index + ├── direct recent-trace execution └── atomic manifest + streaming compaction - │ same durable commit + │ same WAL-backed transaction ▼ Parquet files │ @@ -67,26 +67,28 @@ Important distinctions: ## Measured result -The normalized benchmark used one million complete Fanout-shaped spans, 50,000-row -commits, live endpoint rollups, complete trace reads, and another 200,000 rows -under concurrent trace load at 100 queries per second. +The normalized benchmark used one million complete Fanout-shaped spans, +50,000-row commits, complete trace reads, a raw service aggregation, and +another 200,000 rows under concurrent trace load at 100 queries per second. +General-purpose engines also ran an endpoint-rollup query. -| Storage / execution | Write rows/s | Endpoint | Full trace | Raw scan | Mixed write | Mixed trace p95 | Active disk | Peak RSS | -|---|---:|---:|---:|---:|---:|---:|---:|---:| -| **Production repository: Fanout + Parquet** | **165,875** | **0.172 ms** | **0.517 ms** | 26.18 ms | **167,353/s** | **0.76 ms** | 56.6 MiB | Not isolated | -| **Fanout columnar + direct** | **520,879** | **0.224 ms** | **0.507 ms** | 26.98 ms | **528,668/s** | **1.33 ms** | 34.4 MiB | **197 MiB** | -| DuckDB native | 98,030 | 0.905 ms | 1.54 ms | **1.44 ms** | 85,514/s | 2.14 ms | 47.5 MiB | 1,693 MiB | -| Zstd Parquet + DuckDB | 94,824 effective | 1.35 ms | 10.53 ms | 3.88 ms | n/a | n/a | **21.8 MiB** | Included in DuckDB process | -| chDB MergeTree | 129,724 | 2.81 ms | 7.39 ms | 6.06 ms | 118,722/s | 9.61 ms | 38.1 MiB | 699 MiB | +| Storage / execution | Write rows/s | Endpoint | Full trace | Raw scan | Mixed write | Mixed trace p95 | Active disk | +|---|---:|---:|---:|---:|---:|---:|---:| +| **Production repository: Fanout + Parquet** | 161,115 | n/a | 0.846 ms | 28.63 ms | 162,544/s | 1.79 ms | 56.7 MiB | +| **Fanout columnar + direct** | **548,714** | n/a | **0.663 ms** | 34.93 ms | **538,201/s** | **0.984 ms** | 34.6 MiB | +| DuckDB native | 98,307 | **0.880 ms** | 1.45 ms | **1.21 ms** | 72,420/s | 2.08 ms | 47.8 MiB | +| Zstd Parquet + DuckDB | 95,305 effective | 1.21 ms | 9.44 ms | 3.45 ms | n/a | n/a | **21.8 MiB** | +| chDB MergeTree | 125,388 | 2.91 ms | 7.10 ms | 6.10 ms | 121,775/s | 12.41 ms | 38.1 MiB | Maintenance measurements: | Operation | Time | |---|---:| -| Fanout compressed-block compaction | **135 ms** | -| DuckDB endpoint-rollup build | 274 ms | -| Parquet export | 345 ms | -| chDB forced optimization | 4.13 s | +| Fanout compressed-block and index compaction | **142 ms** | +| DuckDB endpoint-rollup build | 166 ms | +| DuckDB checkpoint | 4.59 ms | +| Parquet export | 320 ms | +| chDB forced optimization | 4.05 s | The production-repository row includes the real atomic WAL + hot-segment + Parquet commit path and was rerun on 2026-08-26. The isolated rows measure each @@ -98,7 +100,7 @@ not published capacity claims. The detailed methodology and reproduction command | Option | Writes | Product reads | Ad-hoc SQL | Open data | Complexity | Verdict | |---|---|---|---|---|---|---| -| Fanout hot + Parquet cold + DuckDB | **Best** | **Best** | Strong | Yes for cold data | Medium | **Recommended** | +| Fanout hot index + Parquet + DuckDB | **Best** | **Best** | Strong | Yes | Medium | **Recommended** | | DuckDB native | Medium | Strong | **Best** | Export required | Low | Good simpler alternative | | DuckLake + DuckDB + Parquet | Medium | Strong | Strong | Yes | Medium-high | Remove from new design | | Iceberg v3 + Parquet + DuckDB | Medium-low | Strong | Strong | **Best** | High | Add only for shared object storage | @@ -111,18 +113,17 @@ not published capacity claims. The detailed methodology and reproduction command - Fanout-owned immutable columnar hot segments. - Atomic Fanout manifest and crash recovery. -- Trace, tenant, service, and other promoted indexes. -- Service and endpoint rollups created during ingestion. +- A fixed-width on-disk trace index, searched lazily without a retention-sized + resident map. - Streaming compaction that copies compressed blocks without decoding rows. - Parquet files committed alongside each hot segment. -- DuckDB for ad-hoc SQL over cold files. +- DuckDB for dashboards, broad scans, and ad-hoc SQL over Parquet. - SQLite for control data. ### Benefits - Highest measured ingestion throughput. - Lowest measured indexed-query latency. -- Lowest measured peak memory. - No C++ call in the ingestion hot path. - Fanout can optimize precisely for append-only telemetry and TTL retention. - Parquet preserves interoperability for the complete retained dataset. @@ -132,11 +133,11 @@ not published capacity claims. The detailed methodology and reproduction command - Fanout owns file-format compatibility, checksums, recovery, retention, and compaction correctness. -- Hot and cold data use different physical formats. -- Product queries spanning hot and cold data must merge two result streams. +- Recent spans have a second, rebuildable physical representation. +- Trace queries that cross the hot-retention boundary fall back to the + authoritative Parquet view. - The current benchmark's broad scan is much slower than DuckDB. -- Further promoted-attribute indexes and long-run compaction tuning remain - workload-driven optimizations. +- Long-run compaction tuning remains workload-driven. ### Decision @@ -163,16 +164,18 @@ SQL and interoperable cold storage to established components. - Ingestion was approximately five times slower than the custom hot store in the full-shape benchmark. -- Peak RSS was much higher in the isolated comparison. +- Peak RSS was much higher in a previous isolated comparison; the current + normalized rerun did not repeat RSS measurement. - Scheduled rollup work remains outside ingestion. - Native files are not an interoperable telemetry format. - Export is required for other engines to consume the data. ### Decision -**Best fallback if owning a hot format becomes too expensive.** It is preferable -to a more complicated DuckLake or Iceberg deployment when everything remains -inside one Fanout process. +**Best simpler replacement if owning a hot format becomes too expensive.** It +is preferable to a more complicated DuckLake or Iceberg deployment when +everything remains inside one Fanout process. It is an architecture choice, +not a runtime fallback path. ## Option C: DuckLake + DuckDB + Parquet @@ -322,34 +325,32 @@ It provides: - straightforward export and backup; - independence from Fanout's hot-format evolution. -Parquet should not be used for every small ingest flush. Fanout should first -write hot segments, then create reasonably sized Parquet files during aging or -cold compaction. +Parquet is published in every durable repository commit. Background compaction +combines small files without changing the authoritative format. ## Proposed data lifecycle ```text 1. Receive OTLP batch 2. Normalize and promote indexed attributes once -3. Append a crash-safe Fanout hot segment -4. Publish the segment through an atomic manifest -5. Answer dashboards and trace lookup directly -6. Stream-compact small hot segments -7. Age completed time partitions into Parquet -8. Atomically publish cold files and retire superseded hot segments -9. Query cold/ad-hoc data with DuckDB -10. Delete expired whole files through manifest commits +3. Durably stage the WAL and authoritative Parquet +4. Atomically publish Parquet under the reader gate +5. Publish the hot segment and commit journal, then remove the WAL +6. Answer recent complete traces through the hot index +7. Query dashboards, broad scans, and ad-hoc SQL with DuckDB +8. Stream-compact small Parquet and hot-segment files +9. Delete expired whole files through manifest commits ``` ## Query routing -| Query | Hot data | Cold data | +| Query | Recent path | Authoritative path | |---|---|---| -| Trace by ID | Fanout trace index | Parquet sidecar index, then DuckDB or direct reader | -| Service/endpoint dashboard | Fanout ingestion-time rollups | Parquet rollups through DuckDB | -| Promoted attribute filter | Fanout attribute index | DuckDB predicate pushdown | -| Log text search | Fanout text/token index | DuckDB scan initially; specialized cold index if required | -| Arbitrary SQL | Optional limited direct projection | DuckDB over Parquet | +| Trace by ID | Fanout trace index | DuckDB over Parquet | +| Service/endpoint dashboard | DuckDB rollup cache | DuckDB rollup cache | +| Promoted attribute filter | DuckDB predicate pushdown | DuckDB predicate pushdown | +| Log text search | DuckDB scan | DuckDB scan | +| Arbitrary SQL | DuckDB over Parquet | DuckDB over Parquet | | Export | Parquet writer | Existing Parquet files | ## Single-binary implications @@ -367,19 +368,15 @@ No external database daemon is required by the recommended design. ## Production gates -Do not replace the existing telemetry path until all gates pass: +The implementation should remain gated on these production checks: -- [ ] Add complete log and metric columnar formats. -- [ ] Add promoted tenant and high-value attribute indexes. - [ ] Add per-block and per-file checksums. - [ ] Test torn writes and corruption at every commit boundary. - [ ] Run continuous kill/restart recovery tests. - [ ] Prove retention and compaction are safe under active readers. - [ ] Bound memory during multi-day compaction. -- [ ] Add hot/cold query result merging. - [ ] Benchmark on the target Linux 4-vCPU/8-GB host. - [ ] Run a long concurrent ingest/query/retention soak. -- [ ] Validate upgrade and format-version handling. - [ ] Benchmark realistic high-cardinality attributes and large exception data. ## Final decision table @@ -387,11 +384,11 @@ Do not replace the existing telemetry path until all gates pass: | Component | Initial decision | Revisit when | |---|---|---| | Fanout hot columnar format | **Use** | If ownership cost exceeds its measured advantage | -| Fanout direct query paths | **Use** | Always retain benchmarks against DuckDB | -| Parquet cold format | **Use** | No expected replacement | +| Fanout direct trace path | **Use** | Always retain benchmarks against DuckDB | +| Parquet authoritative format | **Use** | No expected replacement | | DuckDB query engine | **Use** | If another embedded engine wins normalized SQL tests materially | | SQLite control database | **Use** | No overlap with telemetry storage | -| DuckDB native telemetry tables | Do not use as primary | Fallback if the custom store fails production gates | +| DuckDB native telemetry tables | Do not use | Reconsider only as a deliberate architecture replacement | | DuckLake | **Remove** | If DuckDB again becomes authoritative over mutable Parquet tables | | Iceberg v3 | Not initially | Shared object storage, multiple writers, or multi-engine tables | | chDB | **Remove** | Only if its binding, footprint, and normalized results improve materially | @@ -404,7 +401,7 @@ Fanout does not need every lakehouse layer. The smallest architecture that satisfies the product is: ```text -Fanout hot columnar store + Parquet cold files + DuckDB SQL + SQLite control +Fanout hot trace index + authoritative Parquet + DuckDB SQL + SQLite control ``` DuckLake and Iceberg overlap with lifecycle management that Fanout already must diff --git a/docs/storage-benchmark.md b/docs/storage-benchmark.md index 94572b7a..70ac6224 100644 --- a/docs/storage-benchmark.md +++ b/docs/storage-benchmark.md @@ -16,40 +16,46 @@ matching Fanout's current 35-column analytical shape. Each run uses one million spans, 50,000-row durable commits, 50 services, 20 routes, 200 tenants, five spans per trace, and 24 hours of event time. -Queries return complete 35-column traces, endpoint rollups, and a raw service -aggregation. The mixed test writes another 200,000 committed rows while full -trace reads run at 100 queries per second. +Queries return complete 35-column traces and a raw service aggregation. The +general-purpose engines also run an endpoint rollup. The mixed test writes +another 200,000 committed rows while full trace reads run at 100 queries per +second. ## Fanout segment design - immutable segments containing 2,048-row columnar blocks; - every column compressed independently with Zstandard; - block min/max event-time metadata; -- compact per-block trace indexes with full-ID verification; -- five-minute endpoint histograms built during ingestion; +- a fixed-width on-disk trace index, binary-searched lazily with full-ID + verification; - atomic manifest replacement with file and directory `fsync` ordering; - orphan detection after a crash between segment and manifest publication; - streaming compaction that copies compressed blocks without materializing - rows, then atomically replaces the input segments; + rows and merges indexes with bounded memory, then atomically replaces the + input segments; - direct execution for indexed Fanout operations. ## Normalized result Collected on Darwin/arm64, Apple M3 Max, 14 logical CPUs. This is a development -comparison, not a published Fanout capacity claim. Peak RSS was measured in an -isolated process for each embedded engine. +comparison, not a published Fanout capacity claim. | Storage / execution | Write rows/s | Maintenance | Active disk | Endpoint | Full trace | Raw service | Mixed write | Mixed trace p95 | |---|---:|---:|---:|---:|---:|---:|---:|---:| -| **Production Fanout + Parquet** | 154,848 | live | 56.6 MiB | **0.171 ms** | **0.508 ms** | 27.42 ms | 156,547/s | 0.864 ms | -| Fanout columnar experiment | **511,113** | **94 ms** | 34.4 MiB | 0.210 ms | 0.551 ms | 27.99 ms | **513,927/s** | **0.816 ms** | -| DuckDB native | 93,189 | 143 ms rollup + 5 ms checkpoint | 47.5 MiB | 0.899 ms | 1.52 ms | **1.16 ms** | 80,255/s | 2.02 ms | -| Zstd Parquet + DuckDB | 90,215 effective | 354 ms export | **21.8 MiB** | 1.35 ms | 9.40 ms | 3.64 ms | n/a | n/a | -| chDB MergeTree | 78,434 | 4.64 s optimize | 38.1 MiB active | 3.47 ms | 9.47 ms | 6.88 ms | 97,271/s | 12.38 ms | +| **Production Fanout + Parquet** | 161,115 | live | 56.7 MiB | n/a | 0.846 ms | 28.63 ms | 162,544/s | 1.79 ms | +| Fanout columnar experiment | **548,714** | **142 ms** | 34.6 MiB | n/a | **0.663 ms** | 34.93 ms | **538,201/s** | **0.984 ms** | +| DuckDB native | 98,307 | 166 ms rollup + 4.59 ms checkpoint | 47.8 MiB | **0.880 ms** | 1.45 ms | **1.21 ms** | 72,420/s | 2.08 ms | +| Zstd Parquet + DuckDB | 95,305 effective | 320 ms export | **21.8 MiB** | 1.21 ms | 9.44 ms | 3.45 ms | n/a | n/a | +| chDB MergeTree | 125,388 | 4.05 s optimize | 38.1 MiB active | 2.91 ms | 7.10 ms | 6.10 ms | 121,775/s | 12.41 ms | The chDB directory occupied 106.7 MiB after forced merges because inactive and engine-internal files remain present; the table's active parts occupied 38.1 -MiB. Its embedded-engine initialization took 747 ms in the measured run. +MiB. Its embedded-engine initialization took 418 ms in the measured run. + +Endpoint is `n/a` for the Fanout rows because the span segment deliberately +contains only the production trace-lookup primitive. Production endpoint +dashboards use the rebuildable DuckDB `endpoint_rollup` cache; the DuckDB row +measures that query shape. Iceberg is not listed as an execution engine. Its data plane is Parquet; table metadata, snapshots, manifests, deletion vectors, and planning would sit above @@ -60,7 +66,8 @@ the Parquet/DuckDB result and add capabilities plus some overhead. The custom span experiment establishes the upper bound behind the earlier roughly 500k rows/s figure. It is not the production write rate: it omits the authoritative Parquet projection for logs and metrics and is intentionally not -a general SQL store. +a general SQL store. Its trace index stays on disk and is searched lazily, so +retention does not create a resident trace-ID or rollup map. The production design keeps the useful parts without taking on a home-grown database: diff --git a/internal/api/health.go b/internal/api/health.go index a29bd054..5cf7c52c 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -190,7 +190,10 @@ func (h *HealthHandler) checkTelemetry() CheckResult { defer cancel() var one int - err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM telemetry.spans LIMIT 1") + // Readiness must not queue behind routine Parquet publication. Maintenance + // health is reported separately; this probe only verifies that DuckDB and the + // telemetry view can plan and execute. + err := h.duck.DB.QueryRowContext(ctx, "SELECT 1 FROM telemetry.spans LIMIT 1").Scan(&one) if err != nil && err != sql.ErrNoRows { return CheckResult{ Status: "unhealthy", diff --git a/internal/config/config.go b/internal/config/config.go index d1615760..0ec2c97b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -192,8 +192,8 @@ func (c Config) Validate() error { if c.RetentionDays < 0 { return fmt.Errorf("storage.retention_days must be >= 0, got %d", c.RetentionDays) } - if c.HotRetention < 24*time.Hour { - return fmt.Errorf("storage.hot_retention must be at least 24h, got %s", c.HotRetention) + if c.HotRetention <= 0 { + return fmt.Errorf("storage.hot_retention must be positive, got %s", c.HotRetention) } if c.MaintenanceInterval < time.Second { return fmt.Errorf("storage.maintenance_interval must be at least 1s, got %s", c.MaintenanceInterval) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e574b2b7..6872141d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -642,6 +642,7 @@ func TestValidate(t *testing.T) { {"RollupInterval=0", func(c *Config) { c.RollupInterval = 0 }}, {"RollupInterval=999ms", func(c *Config) { c.RollupInterval = 999 * time.Millisecond }}, {"RetentionDays=-1", func(c *Config) { c.RetentionDays = -1 }}, + {"HotRetention=0", func(c *Config) { c.HotRetention = 0 }}, {"HTTPAddr empty", func(c *Config) { c.HTTPAddr = "" }}, {"OTLPGRPCAddr empty", func(c *Config) { c.OTLPGRPCAddr = "" }}, {"OTLPHTTPAddr empty", func(c *Config) { c.OTLPHTTPAddr = "" }}, @@ -698,6 +699,14 @@ func TestValidate(t *testing.T) { } }) + t.Run("short hot retention valid", func(t *testing.T) { + c := valid + c.HotRetention = 15 * time.Minute + if err := c.Validate(); err != nil { + t.Errorf("HotRetention=15m should be valid: %v", err) + } + }) + t.Run("local mode allows absent SMTP and agent", func(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 3ea3c68a..216ee973 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -54,6 +54,9 @@ func (s *Service) Logs(ctx context.Context, scope Scope, service, severity, sear rows.Close() return Result[Logs]{}, fmt.Errorf("scan log: %w", err) } + // Keep Go redaction as a defense-in-depth boundary even though DuckDB + // applies the equivalent expression before filtering and transfer. + entry.Body = redactLogBody(entry.Body) data.Entries = append(data.Entries, entry) } if err := rows.Err(); err != nil { diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 7bb2d629..e26d2f4d 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -264,8 +264,10 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "declined", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(2*time.Millisecond), "ERROR", "checkout", `auth declined: {"password":"[REDACTED]"}`, "trace-3", "root3"). - AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "card declined: token=[REDACTED]", "trace-2", "root2"). + // Return raw values to prove the Go boundary still redacts even if the + // SQL expression and driver ever diverge. + AddRow(start.Add(2*time.Millisecond), "ERROR", "checkout", `auth declined: {"password":"hunter2"}`, "trace-3", "root3"). + AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "card declined: token=abc123", "trace-2", "root2"). AddRow(start, "ERROR", "checkout", "payment declined", "trace-1", "root")) mock.ExpectQuery(regexp.QuoteMeta(logBucketsQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "declined", "declined"). @@ -491,6 +493,69 @@ func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { } } +func TestTraceUsesRebuiltHotTierWhenRootIsAboveCutoff(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + cutoff := start.Add(30 * time.Minute) + end := start.Add(time.Hour) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-after-rebuild", Spans: []telemetry.Span{{ + Namespace: "prod", TraceID: "new-trace", SpanID: "root", ServiceName: "frontend", + StartUnixNanos: cutoff.Add(time.Minute).UnixNano(), DurationMS: 10, StatusCode: "OK", + }}}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("new-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "new-trace", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Spans) != 1 || result.Data.Spans[0].SpanID != "root" || result.Provenance.DataSource != "fanout_segments" { + t.Fatalf("rebuilt hot trace = %#v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestTraceDoesNotUseRootFromAnotherNamespaceAsHotCoverage(t *testing.T) { + svc, mock := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + cutoff := start.Add(30 * time.Minute) + end := start.Add(time.Hour) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-cross-namespace", Spans: []telemetry.Span{ + {Namespace: "prod", TraceID: "shared-trace", SpanID: "child", ParentSpanID: "old-root", StartUnixNanos: cutoff.Add(time.Minute).UnixNano()}, + {Namespace: "staging", TraceID: "shared-trace", SpanID: "root", StartUnixNanos: cutoff.Add(2 * time.Minute).UnixNano()}, + }}); err != nil { + t.Fatal(err) + } + if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). + WithArgs("shared-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). + AddRow("old-root", "", "frontend", "request", "SERVER", start.Add(10*time.Minute), 10.0, "OK", ""). + AddRow("child", "old-root", "backend", "work", "CLIENT", cutoff.Add(time.Minute), 5.0, "OK", "")) + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). + WithArgs("shared-trace", start, end, "prod", "prod", 10). + WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) + result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "shared-trace", "", 10) + if err != nil { + t.Fatal(err) + } + if len(result.Data.Spans) != 2 || result.Provenance.DataSource != "parquet" { + t.Fatalf("cross-namespace trace = %#v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + var _ DB = queryrows.SQLAdapter{} func TestLogsBoundsParquetQueryWithLimitAndAggregatedBuckets(t *testing.T) { diff --git a/internal/observability/trace.go b/internal/observability/trace.go index ed55bafe..e08b7abb 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -63,16 +63,22 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin dataSource := "fanout_segments" data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} hotCutoff := int64(0) + hotHasRoot := false if traceID != "" { - storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID, scope.Start.UnixNano()) + storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID) if readErr != nil { return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) } hotCutoff = spanCutoff startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() for _, row := range storedSpans { + matchesNamespace := scope.Namespace == "" || row.Namespace == scope.Namespace + if row.ParentSpanID == "" && row.StartUnixNanos >= hotCutoff && + row.StartUnixNanos < endNanos && matchesNamespace { + hotHasRoot = true + } if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || - (scope.Namespace != "" && row.Namespace != scope.Namespace) { + !matchesNamespace { continue } data.Spans = append(data.Spans, TraceSpan{SpanID: row.SpanID, ParentSpanID: row.ParentSpanID, Service: row.ServiceName, Operation: row.Name, Kind: row.Kind, Start: time.Unix(0, row.StartUnixNanos).UTC(), DurationMS: row.DurationMS, Status: row.StatusCode, StatusMessage: row.StatusMsg}) @@ -112,20 +118,21 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin sort.Strings(data.Services) } - // Any scope crossing the durable hot prune watermark may contain early trace - // spans that have aged out while a late suffix remains hot. Parquet is the - // authoritative complete trace in that case; a zero-span hot miss uses the - // same path. - if traceID != "" && (len(data.Spans) == 0 || scope.Start.UnixNano() < hotCutoff) { + // A root retained at or above the durable cutoff proves the hot index contains + // the beginning of this trace, including immediately after a tier rebuild. A + // crossing scope without that root may contain only a suffix and must use the + // authoritative Parquet copy. + hotComplete := scope.Start.UnixNano() >= hotCutoff || hotHasRoot + if traceID != "" && (len(data.Spans) == 0 || !hotComplete) { data, err = s.traceFromParquet(ctx, scope, traceID, limit) if err != nil { return Result[TraceDetail]{}, err } dataSource = "parquet" } else if traceID != "" { - // Parquet is committed atomically with the hot span index. DuckDB can - // apply trace_id and LIMIT inside its vectorized scan, avoiding a full - // Go decode while preserving clock-skewed trace events. + // Parquet is authoritative and published before the disposable hot index. + // DuckDB can apply trace_id and LIMIT to the associated logs without a + // full Go decode while preserving clock-skewed trace events. data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) if err != nil { return Result[TraceDetail]{}, err diff --git a/internal/query/duck.go b/internal/query/duck.go index dcb20f79..eff5cbe2 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -70,6 +70,12 @@ const ( defaultDuckDBPoolSize = 1 ) +// rollupPublicationSafetyLag covers the maximum public SQL hold, publication +// grace, bounded commit retries, and queued segment encoding with headroom for +// a busy disk. Rows stamped at request receipt remain inside the recomputed +// tail until their Parquet commit becomes visible. +const rollupPublicationSafetyLag = 5 * time.Minute + // WriteGate returns the gate that serializes writes to DuckDB's rebuildable // rollup cache. func (d *Duck) WriteGate() *writegate.WriteGate { return &d.writeGate } @@ -199,7 +205,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(30 * time.Second)} + d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} repository.SetParquetPublishLock(&d.parquetMu) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's diff --git a/internal/query/rollup_watermark_test.go b/internal/query/rollup_watermark_test.go index be7cf8b6..daf92331 100644 --- a/internal/query/rollup_watermark_test.go +++ b/internal/query/rollup_watermark_test.go @@ -8,6 +8,13 @@ import ( "github.com/labstack/fanout/internal/config" ) +func TestRollupPublicationLagExceedsReaderAndPublisherWindow(t *testing.T) { + minimum := defaultWriterGrace + time.Duration(maxQueryTimeoutMs)*time.Millisecond + if rollupPublicationSafetyLag <= minimum { + t.Fatalf("rollup publication lag = %s, must exceed reader+publisher window %s", rollupPublicationSafetyLag, minimum) + } +} + // TestRollupWatermarkPicksUpLateLowIngestedRow reproduces the silent-data-loss // bug: a row that commits with an ingested timestamp below a watermark already // advanced by another signal would be excluded from the rollup forever. The diff --git a/internal/telemetry/segment/span_columnar.go b/internal/telemetry/segment/span_columnar.go index 65168c77..9ae82b09 100644 --- a/internal/telemetry/segment/span_columnar.go +++ b/internal/telemetry/segment/span_columnar.go @@ -58,7 +58,7 @@ var allColumns = func() []int { return out }() -func encodeColumnarBlock(enc *zstd.Encoder, rows []Span) []byte { +func encodeColumnarBlock(enc *zstd.Encoder, rows []Span) ([]byte, error) { columns := make([][]byte, columnCount) for _, row := range rows { columns[colNamespace] = appendString(columns[colNamespace], row.Namespace) @@ -99,14 +99,64 @@ func encodeColumnarBlock(enc *zstd.Encoder, rows []Span) []byte { binary.LittleEndian.PutUint32(out[0:4], columnCount) offset := columnarHeaderSize for id, plain := range columns { + if len(plain) > segmentDecoderMaxMemory { + return nil, fmt.Errorf("column %d exceeds decoder memory limit", id) + } compressed := enc.EncodeAll(plain, nil) + if len(compressed) > math.MaxUint32 || offset > math.MaxUint32-len(compressed) { + return nil, fmt.Errorf("column %d exceeds segment extent limit", id) + } entry := out[4+id*8:] binary.LittleEndian.PutUint32(entry[0:4], uint32(offset)) binary.LittleEndian.PutUint32(entry[4:8], uint32(len(compressed))) out = append(out, compressed...) offset += len(compressed) } - return out + if len(out) > segmentMaxCompressedBytes { + return nil, fmt.Errorf("encoded block uses %d bytes; maximum is %d", len(out), segmentMaxCompressedBytes) + } + return out, nil +} + +// spanColumnPlainSizes mirrors encodeColumnarBlock without allocating. The WAL +// validator uses it to guarantee that every acknowledged block fits the same +// per-column and aggregate budgets enforced by the decoder. +func spanColumnPlainSizes(rows []Span) [columnCount]uint64 { + var sizes [columnCount]uint64 + framed := func(valueLen int) uint64 { + var scratch [binary.MaxVarintLen64]byte + return uint64(binary.PutUvarint(scratch[:], uint64(valueLen)) + valueLen) + } + for _, row := range rows { + for id, value := range []string{ + row.Namespace, row.TraceID, row.SpanID, row.ParentSpanID, + row.ServiceName, row.Name, row.Kind, + } { + sizes[id] += framed(len(value)) + } + sizes[colStartUnixNanos] += 8 + sizes[colEndUnixNanos] += 8 + sizes[colDurationMS] += 8 + for offset, value := range []string{row.StatusCode, row.StatusMsg} { + sizes[colStatusCode+offset] += framed(len(value)) + } + for offset, value := range [][]byte{row.ResourceJSON, row.AttributesJSON, row.EventsJSON, row.LinksJSON} { + sizes[colResourceJSON+offset] += framed(len(value)) + } + sizes[colTraceState] += framed(len(row.TraceState)) + sizes[colFlags] += 4 + sizes[colScopeName] += framed(len(row.ScopeName)) + sizes[colScopeVersion] += framed(len(row.ScopeVersion)) + sizes[colIngestedAt] += 8 + for offset, value := range []string{ + row.HTTPMethod, row.HTTPStatusCode, row.HTTPRoute, row.DBSystem, + row.RPCMethod, row.RPCService, row.PeerService, row.ServiceVersion, + row.DeploymentEnv, row.ExceptionType, row.ExceptionMessage, + } { + sizes[colHTTPMethod+offset] += framed(len(value)) + } + } + return sizes } func decodeColumns(dec *zstd.Decoder, block []byte, wanted []int) (map[int][]byte, error) { diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go index ad9078af..f05131e8 100644 --- a/internal/telemetry/segment/span_store.go +++ b/internal/telemetry/segment/span_store.go @@ -6,6 +6,7 @@ package segment import ( "bufio" "bytes" + "container/heap" "encoding/binary" "encoding/json" "errors" @@ -20,7 +21,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/klauspost/compress/zstd" "github.com/labstack/fanout/internal/telemetry" @@ -33,7 +33,6 @@ import ( // long before zstd's 64 GiB default would. const ( segmentDecoderMaxMemory = 128 << 20 - maxRollupKeyBytes = 32 << 20 segmentMaxCompressedBytes = segmentDecoderMaxMemory + (1 << 20) segmentMaxBlocks = 1 << 20 ) @@ -54,64 +53,44 @@ func newSegmentDecoderWithLimit(limit uint64) (*zstd.Decoder, error) { } const ( - segmentMagic = "FANSEG03" - segmentVersion = uint32(3) + segmentMagic = "FANSEG04" + segmentVersion = uint32(4) headerSize = 64 blockDirSize = 32 - traceEntrySize = 16 + traceEntrySize = 12 rowsPerBlock = 2048 - rollupBins = 32 - rollupWindow = 5 * time.Minute ) type Span = telemetry.Span -// ValidateSpanRows rejects rows whose derived rollup representation could not -// be reopened within the production decoder budget. It must run before WAL -// staging so deterministic format errors never become boot-blocking WALs. +// ValidateSpanRows rejects rows whose columnar blocks could not be reopened +// within the production decoder budget. It must run before WAL staging so a +// deterministic format error never becomes a boot-blocking WAL. func ValidateSpanRows(rows []Span) error { - return validateSpanRowsWithLimits(rows, maxRollupKeyBytes, segmentDecoderMaxMemory) + return validateSpanRowsWithLimit(rows, segmentDecoderMaxMemory) } -func validateSpanRowsWithLimits(rows []Span, maxKeyBytes, maxSectionBytes uint64) error { - keys := make(map[rollupKey]struct{}, len(rows)) - var sectionBytes uint64 - for _, row := range rows { - key := rollupKey{ - bucket: row.StartUnixNanos - row.StartUnixNanos%int64(rollupWindow), - namespace: row.Namespace, service: row.ServiceName, method: row.HTTPMethod, route: row.HTTPRoute, - } - if _, exists := keys[key]; exists { - continue - } - keys[key] = struct{}{} - keyBytes := uint64(len(key.namespace)) + uint64(len(key.service)) + uint64(len(key.method)) + uint64(len(key.route)) - if keyBytes > maxKeyBytes { - return fmt.Errorf("span rollup key uses %d bytes; maximum is %d", keyBytes, maxKeyBytes) - } - size := uint64(160) - var scratch [binary.MaxVarintLen64]byte - for _, value := range []string{key.namespace, key.service, key.method, key.route} { - size += uint64(binary.PutUvarint(scratch[:], uint64(len(value)))) + uint64(len(value)) - } - if size > maxSectionBytes-sectionBytes { - return fmt.Errorf("span rollups exceed %d-byte decoder budget", maxSectionBytes) +func validateSpanRowsWithLimit(rows []Span, maxBlockBytes uint64) error { + if maxBlockBytes < columnarHeaderSize { + return fmt.Errorf("span block budget %d is smaller than its %d-byte header", maxBlockBytes, columnarHeaderSize) + } + for start := 0; start < len(rows); start += rowsPerBlock { + end := min(start+rowsPerBlock, len(rows)) + sizes := spanColumnPlainSizes(rows[start:end]) + total := uint64(columnarHeaderSize) + for id, size := range sizes { + if size > maxBlockBytes { + return fmt.Errorf("span block column %d uses %d bytes; maximum is %d", id, size, maxBlockBytes) + } + if size > maxBlockBytes-total { + return fmt.Errorf("span block uses more than %d bytes", maxBlockBytes) + } + total += size } - sectionBytes += size } return nil } -type Endpoint struct { - Service string - Method string - Route string - Calls uint64 - Errors uint64 - AverageMS float64 - P95MS float64 -} - type Aggregate struct { Calls uint64 Errors uint64 @@ -129,33 +108,42 @@ type blockDir struct { type traceEntry struct { hash uint64 block uint32 - row uint32 } -type rollupKey struct { - bucket int64 - namespace string - service string - method string - route string +type indexCursor struct { + file *os.File + segment segment + blockBase uint32 + position uint32 + entry traceEntry } -type rollup struct { - key rollupKey - calls uint64 - errors uint64 - duration float64 - bins [rollupBins]uint32 +type indexCursorHeap []*indexCursor + +func (h indexCursorHeap) Len() int { return len(h) } +func (h indexCursorHeap) Less(i, j int) bool { + if h[i].entry.hash != h[j].entry.hash { + return h[i].entry.hash < h[j].entry.hash + } + return h[i].entry.block < h[j].entry.block +} +func (h indexCursorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *indexCursorHeap) Push(value any) { *h = append(*h, value.(*indexCursor)) } +func (h *indexCursorHeap) Pop() any { + old := *h + last := old[len(old)-1] + *h = old[:len(old)-1] + return last } type segment struct { - path string - rows uint32 - min int64 - max int64 - blocks []blockDir - traceIndex []traceEntry - rollups []rollup + path string + rows uint32 + min int64 + max int64 + blocks []blockDir + indexOffset uint64 + indexCount uint32 } type manifest struct { @@ -242,7 +230,7 @@ func (s *Store) loadManifest() error { } // Append writes one crash-safe immutable segment and atomically publishes it. -// Rollups and the trace index are built in the same pass as block encoding. +// The on-disk trace index is built in the same pass as block encoding. func (s *Store) Append(rows []Span) error { if len(rows) == 0 { return nil @@ -501,7 +489,6 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { seg := segment{rows: uint32(len(rows)), min: math.MaxInt64, max: math.MinInt64} index := make([]traceEntry, 0, len(rows)) - rollups := make(map[rollupKey]*rollup) var offset = uint64(headerSize) for blockStart := 0; blockStart < len(rows); blockStart += rowsPerBlock { blockEnd := min(blockStart+rowsPerBlock, len(rows)) @@ -518,23 +505,11 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { index = append(index, traceEntry{hash: traceHash, block: uint32(len(seg.blocks))}) blockTraces[traceHash] = struct{}{} } - key := rollupKey{ - bucket: row.StartUnixNanos - row.StartUnixNanos%int64(rollupWindow), - namespace: row.Namespace, service: row.ServiceName, method: row.HTTPMethod, route: row.HTTPRoute, - } - r := rollups[key] - if r == nil { - r = &rollup{key: key} - rollups[key] = r - } - r.calls++ - if isErrorStatus(row.StatusCode) { - r.errors++ - } - r.duration += row.DurationMS - r.bins[durationBin(row.DurationMS)]++ } - encoded := encodeColumnarBlock(s.encoder, rows[blockStart:blockEnd]) + encoded, err := encodeColumnarBlock(s.encoder, rows[blockStart:blockEnd]) + if err != nil { + return segment{}, fmt.Errorf("encode block: %w", err) + } if _, err := f.Write(encoded); err != nil { return segment{}, fmt.Errorf("write block: %w", err) } @@ -546,31 +521,7 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { if index[i].hash != index[j].hash { return index[i].hash < index[j].hash } - if index[i].block != index[j].block { - return index[i].block < index[j].block - } - return index[i].row < index[j].row - }) - seg.traceIndex = index - seg.rollups = make([]rollup, 0, len(rollups)) - for _, r := range rollups { - seg.rollups = append(seg.rollups, *r) - } - sort.Slice(seg.rollups, func(i, j int) bool { - a, b := seg.rollups[i].key, seg.rollups[j].key - if a.bucket != b.bucket { - return a.bucket < b.bucket - } - if a.namespace != b.namespace { - return a.namespace < b.namespace - } - if a.service != b.service { - return a.service < b.service - } - if a.method != b.method { - return a.method < b.method - } - return a.route < b.route + return index[i].block < index[j].block }) dirOffset := offset @@ -588,43 +539,29 @@ func (s *Store) writeSegment(path string, rows []Span) (segment, error) { } indexOffset, _ := f.Seek(0, io.SeekCurrent) indexPlain := make([]byte, len(index)*traceEntrySize) - if len(indexPlain) > segmentDecoderMaxMemory { - return segment{}, errors.New("trace index exceeds decoder memory limit") - } for i, entry := range index { buf := indexPlain[i*traceEntrySize:] binary.LittleEndian.PutUint64(buf[0:8], entry.hash) binary.LittleEndian.PutUint32(buf[8:12], entry.block) - binary.LittleEndian.PutUint32(buf[12:16], entry.row) } - if _, err := f.Write(s.encoder.EncodeAll(indexPlain, nil)); err != nil { + if _, err := f.Write(indexPlain); err != nil { return segment{}, fmt.Errorf("write trace index: %w", err) } - rollupOffset, _ := f.Seek(0, io.SeekCurrent) - var rollupPlain bytes.Buffer - for _, r := range seg.rollups { - if err := writeRollup(&rollupPlain, r); err != nil { - return segment{}, err - } - } - if rollupPlain.Len() > segmentDecoderMaxMemory { - return segment{}, errors.New("rollup section exceeds decoder memory limit") - } - if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { - return segment{}, fmt.Errorf("write rollups: %w", err) - } + indexEnd, _ := f.Seek(0, io.SeekCurrent) + seg.indexOffset = uint64(indexOffset) + seg.indexCount = uint32(len(index)) var header [headerSize]byte copy(header[0:8], segmentMagic) binary.LittleEndian.PutUint32(header[8:12], segmentVersion) binary.LittleEndian.PutUint32(header[12:16], seg.rows) binary.LittleEndian.PutUint32(header[16:20], uint32(len(seg.blocks))) - binary.LittleEndian.PutUint32(header[20:24], uint32(len(seg.rollups))) + binary.LittleEndian.PutUint32(header[20:24], seg.indexCount) binary.LittleEndian.PutUint64(header[24:32], uint64(seg.min)) binary.LittleEndian.PutUint64(header[32:40], uint64(seg.max)) binary.LittleEndian.PutUint64(header[40:48], dirOffset) binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) - binary.LittleEndian.PutUint64(header[56:64], uint64(rollupOffset)) + binary.LittleEndian.PutUint64(header[56:64], uint64(indexEnd)) if _, err := f.WriteAt(header[:], 0); err != nil { return segment{}, fmt.Errorf("write header: %w", err) } @@ -654,13 +591,15 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e return segment{}, err } replacement := segment{min: math.MaxInt64, max: math.MinInt64} + blockBases := make([]uint32, len(inputs)) var offset = uint64(headerSize) - for _, input := range inputs { + for i, input := range inputs { source, err := os.Open(input.path) if err != nil { return segment{}, err } blockBase := uint32(len(replacement.blocks)) + blockBases[i] = blockBase for _, block := range input.blocks { if _, err := io.CopyN(f, io.NewSectionReader(source, int64(block.offset), int64(block.length)), int64(block.length)); err != nil { _ = source.Close() @@ -672,41 +611,10 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e if err := source.Close(); err != nil { return segment{}, err } - for _, entry := range input.traceIndex { - entry.block += blockBase - replacement.traceIndex = append(replacement.traceIndex, entry) - } - replacement.rollups = append(replacement.rollups, input.rollups...) replacement.rows += input.rows replacement.min = min(replacement.min, input.min) replacement.max = max(replacement.max, input.max) } - sort.Slice(replacement.traceIndex, func(i, j int) bool { - a, b := replacement.traceIndex[i], replacement.traceIndex[j] - if a.hash != b.hash { - return a.hash < b.hash - } - if a.block != b.block { - return a.block < b.block - } - return a.row < b.row - }) - sort.Slice(replacement.rollups, func(i, j int) bool { - a, b := replacement.rollups[i].key, replacement.rollups[j].key - if a.bucket != b.bucket { - return a.bucket < b.bucket - } - if a.namespace != b.namespace { - return a.namespace < b.namespace - } - if a.service != b.service { - return a.service < b.service - } - if a.method != b.method { - return a.method < b.method - } - return a.route < b.route - }) dirOffset := offset directory := make([]byte, len(replacement.blocks)*blockDirSize) @@ -722,43 +630,24 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e return segment{}, err } indexOffset, _ := f.Seek(0, io.SeekCurrent) - indexPlain := make([]byte, len(replacement.traceIndex)*traceEntrySize) - if len(indexPlain) > segmentDecoderMaxMemory { - return segment{}, errors.New("compacted trace index exceeds decoder memory limit") - } - for i, entry := range replacement.traceIndex { - buf := indexPlain[i*traceEntrySize:] - binary.LittleEndian.PutUint64(buf[0:8], entry.hash) - binary.LittleEndian.PutUint32(buf[8:12], entry.block) - binary.LittleEndian.PutUint32(buf[12:16], entry.row) - } - if _, err := f.Write(s.encoder.EncodeAll(indexPlain, nil)); err != nil { - return segment{}, err - } - rollupOffset, _ := f.Seek(0, io.SeekCurrent) - var rollupPlain bytes.Buffer - for _, r := range replacement.rollups { - if err := writeRollup(&rollupPlain, r); err != nil { - return segment{}, err - } - } - if rollupPlain.Len() > segmentDecoderMaxMemory { - return segment{}, errors.New("compacted rollup section exceeds decoder memory limit") - } - if _, err := f.Write(s.encoder.EncodeAll(rollupPlain.Bytes(), nil)); err != nil { + indexCount, err := mergeTraceIndexes(f, inputs, blockBases) + if err != nil { return segment{}, err } + indexEnd, _ := f.Seek(0, io.SeekCurrent) + replacement.indexOffset = uint64(indexOffset) + replacement.indexCount = indexCount var header [headerSize]byte copy(header[0:8], segmentMagic) binary.LittleEndian.PutUint32(header[8:12], segmentVersion) binary.LittleEndian.PutUint32(header[12:16], replacement.rows) binary.LittleEndian.PutUint32(header[16:20], uint32(len(replacement.blocks))) - binary.LittleEndian.PutUint32(header[20:24], uint32(len(replacement.rollups))) + binary.LittleEndian.PutUint32(header[20:24], replacement.indexCount) binary.LittleEndian.PutUint64(header[24:32], uint64(replacement.min)) binary.LittleEndian.PutUint64(header[32:40], uint64(replacement.max)) binary.LittleEndian.PutUint64(header[40:48], dirOffset) binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) - binary.LittleEndian.PutUint64(header[56:64], uint64(rollupOffset)) + binary.LittleEndian.PutUint64(header[56:64], uint64(indexEnd)) if _, err := f.WriteAt(header[:], 0); err != nil { return segment{}, err } @@ -772,6 +661,129 @@ func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, e return replacement, nil } +// mergeTraceIndexes performs a bounded-memory k-way merge of fixed-width, +// sorted on-disk indexes. Compaction therefore scales with file count rather +// than retaining every trace entry from every generation in the Go heap. +func mergeTraceIndexes(dst io.Writer, inputs []segment, blockBases []uint32) (uint32, error) { + if len(inputs) != len(blockBases) { + return 0, errors.New("trace-index inputs and block bases disagree") + } + cursors := make(indexCursorHeap, 0, len(inputs)) + files := make([]*os.File, 0, len(inputs)) + defer func() { + for _, file := range files { + _ = file.Close() + } + }() + var total uint64 + for i, input := range inputs { + total += uint64(input.indexCount) + if total > math.MaxUint32 { + return 0, errors.New("compacted trace index exceeds entry limit") + } + if input.indexCount == 0 { + continue + } + f, err := os.Open(input.path) + if err != nil { + return 0, err + } + files = append(files, f) + cursor := &indexCursor{file: f, segment: input, blockBase: blockBases[i]} + entry, err := readTraceEntry(f, input, 0) + if err != nil { + _ = f.Close() + return 0, err + } + if entry.block > math.MaxUint32-cursor.blockBase { + _ = f.Close() + return 0, errors.New("compacted trace block id overflows") + } + entry.block += cursor.blockBase + cursor.entry = entry + cursors = append(cursors, cursor) + } + heap.Init(&cursors) + writer := bufio.NewWriterSize(dst, 64<<10) + var encoded [traceEntrySize]byte + for cursors.Len() > 0 { + cursor := heap.Pop(&cursors).(*indexCursor) + binary.LittleEndian.PutUint64(encoded[0:8], cursor.entry.hash) + binary.LittleEndian.PutUint32(encoded[8:12], cursor.entry.block) + if _, err := writer.Write(encoded[:]); err != nil { + return 0, err + } + cursor.position++ + if cursor.position < cursor.segment.indexCount { + entry, err := readTraceEntry(cursor.file, cursor.segment, cursor.position) + if err != nil { + return 0, err + } + if entry.block > math.MaxUint32-cursor.blockBase { + return 0, errors.New("compacted trace block id overflows") + } + entry.block += cursor.blockBase + cursor.entry = entry + heap.Push(&cursors, cursor) + } + } + if err := writer.Flush(); err != nil { + return 0, err + } + return uint32(total), nil +} + +func readTraceEntry(f *os.File, seg segment, position uint32) (traceEntry, error) { + if position >= seg.indexCount { + return traceEntry{}, io.EOF + } + var encoded [traceEntrySize]byte + offset := seg.indexOffset + uint64(position)*traceEntrySize + if _, err := f.ReadAt(encoded[:], int64(offset)); err != nil { + return traceEntry{}, err + } + entry := traceEntry{hash: binary.LittleEndian.Uint64(encoded[0:8]), block: binary.LittleEndian.Uint32(encoded[8:12])} + if int(entry.block) >= len(seg.blocks) { + return traceEntry{}, errors.New("trace index references a missing block") + } + return entry, nil +} + +func traceBlocks(seg segment, hash uint64) ([]uint32, error) { + f, err := os.Open(seg.path) + if err != nil { + return nil, err + } + defer f.Close() + low, high := uint32(0), seg.indexCount + for low < high { + middle := low + (high-low)/2 + entry, err := readTraceEntry(f, seg, middle) + if err != nil { + return nil, err + } + if entry.hash < hash { + low = middle + 1 + } else { + high = middle + } + } + blocks := make([]uint32, 0, 1) + for position := low; position < seg.indexCount; position++ { + entry, err := readTraceEntry(f, seg, position) + if err != nil { + return nil, err + } + if entry.hash != hash { + break + } + if len(blocks) == 0 || blocks[len(blocks)-1] != entry.block { + blocks = append(blocks, entry.block) + } + } + return blocks, nil +} + // Trace performs a hash-index lookup and decompresses only the blocks that can // contain the requested trace. The full trace ID is checked after hashing. func (s *Store) Trace(traceID string) ([]Span, error) { @@ -780,15 +792,13 @@ func (s *Store) Trace(traceID string) ([]Span, error) { hash := xxh3.HashString(traceID) var out []Span for i := range s.segments { - seg := &s.segments[i] - start := sort.Search(len(seg.traceIndex), func(j int) bool { return seg.traceIndex[j].hash >= hash }) - blocks := make(map[uint32][]uint32) - for j := start; j < len(seg.traceIndex) && seg.traceIndex[j].hash == hash; j++ { - entry := seg.traceIndex[j] - blocks[entry.block] = append(blocks[entry.block], entry.row) + seg := s.segments[i] + blocks, err := traceBlocks(seg, hash) + if err != nil { + return nil, err } - for blockID := range blocks { - rows, err := s.readTraceBlock(*seg, blockID, traceID) + for _, blockID := range blocks { + rows, err := s.readTraceBlock(seg, blockID, traceID) if err != nil { return nil, err } @@ -799,65 +809,6 @@ func (s *Store) Trace(traceID string) ([]Span, error) { return out, nil } -// Endpoints answers Fanout's dashboard query entirely from ingestion-time -// rollups. Quantiles use the same bounded-histogram approximation style as the -// existing Fanout endpoint cache. -func (s *Store) Endpoints(namespace, service string, start, end int64, limit int) []Endpoint { - s.mu.RLock() - defer s.mu.RUnlock() - type value struct { - calls, errors uint64 - duration float64 - bins [rollupBins]uint32 - } - values := make(map[string]*value) - keys := make(map[string]rollupKey) - for i := range s.segments { - seg := &s.segments[i] - if seg.max < start || seg.min >= end { - continue - } - for _, r := range seg.rollups { - if r.key.bucket < start-start%int64(rollupWindow) || r.key.bucket >= end { - continue - } - if namespace != "" && r.key.namespace != namespace { - continue - } - if service != "" && r.key.service != service { - continue - } - key := r.key.service + "\x00" + r.key.method + "\x00" + r.key.route - v := values[key] - if v == nil { - v = &value{} - values[key] = v - keys[key] = r.key - } - v.calls += r.calls - v.errors += r.errors - v.duration += r.duration - for j := range v.bins { - v.bins[j] += r.bins[j] - } - } - } - out := make([]Endpoint, 0, len(values)) - for key, v := range values { - k := keys[key] - average := 0.0 - if v.calls > 0 { - average = v.duration / float64(v.calls) - } - out = append(out, Endpoint{Service: k.service, Method: k.method, Route: k.route, Calls: v.calls, Errors: v.errors, AverageMS: average, P95MS: histogramQuantile(v.bins, v.calls, .95)}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Calls > out[j].Calls }) - if limit > 0 && len(out) > limit { - out = out[:limit] - } - return out -} - // ScanService is the deliberately expensive raw path. It demonstrates block // time pruning and provides a fairer comparison with a general query engine. func (s *Store) ScanService(namespace, service string, start, end int64) (Aggregate, error) { @@ -983,7 +934,7 @@ func isErrorStatus(status string) bool { // any of them is used to size an allocation. All comparisons are written as // subtractions against size so a corrupt offset near the top of the address // space cannot wrap past the check. -func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, blockCount uint32) error { +func validateSegmentSections(size, dirOffset, indexOffset, indexEnd uint64, blockCount, indexCount uint32) error { if blockCount > segmentMaxBlocks { return errors.New("segment block count is out of range") } @@ -993,11 +944,14 @@ func validateSegmentSections(size, dirOffset, indexOffset, rollupOffset uint64, if indexOffset < dirOffset+uint64(blockCount)*blockDirSize || indexOffset > size { return errors.New("corrupt trace index offset") } - if rollupOffset < indexOffset || rollupOffset > size { - return errors.New("corrupt rollup offset") + if indexEnd < indexOffset || indexEnd > size { + return errors.New("corrupt trace index end") + } + if uint64(indexCount) > (indexEnd-indexOffset)/traceEntrySize || indexEnd-indexOffset != uint64(indexCount)*traceEntrySize { + return errors.New("trace index size disagrees with the segment header") } - if rollupOffset-indexOffset > segmentMaxCompressedBytes || size-rollupOffset > segmentMaxCompressedBytes { - return errors.New("compressed segment section exceeds memory limit") + if indexEnd != size { + return errors.New("segment has data past the trace index") } return nil } @@ -1058,15 +1012,15 @@ func openSegment(path string) (segment, error) { } seg := segment{path: path, rows: binary.LittleEndian.Uint32(header[12:16]), min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40]))} blockCount := binary.LittleEndian.Uint32(header[16:20]) - rollupCount := binary.LittleEndian.Uint32(header[20:24]) + indexCount := binary.LittleEndian.Uint32(header[20:24]) dirOffset := binary.LittleEndian.Uint64(header[40:48]) indexOffset := binary.LittleEndian.Uint64(header[48:56]) - rollupOffset := binary.LittleEndian.Uint64(header[56:64]) + indexEnd := binary.LittleEndian.Uint64(header[56:64]) info, err := f.Stat() if err != nil { return segment{}, err } - if err := validateSegmentSections(uint64(info.Size()), dirOffset, indexOffset, rollupOffset, blockCount); err != nil { + if err := validateSegmentSections(uint64(info.Size()), dirOffset, indexOffset, indexEnd, blockCount, indexCount); err != nil { return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) } seg.blocks = make([]blockDir, blockCount) @@ -1081,50 +1035,32 @@ func openSegment(path string) (segment, error) { if err := validateSegmentBlocks(uint64(info.Size()), dirOffset, seg.blocks, seg.rows); err != nil { return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) } - indexCompressed := make([]byte, int(rollupOffset-indexOffset)) - if _, err := f.ReadAt(indexCompressed, int64(indexOffset)); err != nil { - return segment{}, err - } - dec, err := newSegmentDecoder() - if err != nil { - return segment{}, err - } - indexBytes, err := dec.DecodeAll(indexCompressed, nil) - if err != nil { - dec.Close() - return segment{}, fmt.Errorf("decode trace index: %w", err) - } - if len(indexBytes)%traceEntrySize != 0 { - dec.Close() - return segment{}, fmt.Errorf("trace index size %d is not entry-aligned", len(indexBytes)) - } - seg.traceIndex = make([]traceEntry, len(indexBytes)/traceEntrySize) - for i := range seg.traceIndex { - b := indexBytes[i*traceEntrySize:] - seg.traceIndex[i] = traceEntry{hash: binary.LittleEndian.Uint64(b[0:8]), block: binary.LittleEndian.Uint32(b[8:12]), row: binary.LittleEndian.Uint32(b[12:16])} - } - rollupCompressed := make([]byte, info.Size()-int64(rollupOffset)) - if _, err := f.ReadAt(rollupCompressed, int64(rollupOffset)); err != nil { - dec.Close() - return segment{}, err + seg.indexOffset = indexOffset + seg.indexCount = indexCount + if err := validateTraceIndex(f, seg); err != nil { + return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) } - rollupBytes, err := dec.DecodeAll(rollupCompressed, nil) - dec.Close() - if err != nil { - return segment{}, fmt.Errorf("decode rollups: %w", err) - } - reader := bufio.NewReader(bytes.NewReader(rollupBytes)) - // rollupCount comes from the same untrusted header, so the slice grows with - // the rollups actually decoded rather than being sized from it up front. - seg.rollups = nil - for range rollupCount { - r, err := readRollup(reader) - if err != nil { - return segment{}, err + return seg, nil +} + +func validateTraceIndex(f *os.File, seg segment) error { + reader := bufio.NewReaderSize(io.NewSectionReader(f, int64(seg.indexOffset), int64(seg.indexCount)*traceEntrySize), 64<<10) + var encoded [traceEntrySize]byte + var previous traceEntry + for position := uint32(0); position < seg.indexCount; position++ { + if _, err := io.ReadFull(reader, encoded[:]); err != nil { + return err + } + entry := traceEntry{hash: binary.LittleEndian.Uint64(encoded[0:8]), block: binary.LittleEndian.Uint32(encoded[8:12])} + if int(entry.block) >= len(seg.blocks) { + return errors.New("trace index references a missing block") } - seg.rollups = append(seg.rollups, r) + if position > 0 && (entry.hash < previous.hash || entry.hash == previous.hash && entry.block < previous.block) { + return errors.New("trace index is not sorted") + } + previous = entry } - return seg, nil + return nil } func appendString(dst []byte, value string) []byte { @@ -1144,89 +1080,6 @@ func consumeByteView(src []byte) ([]byte, []byte, error) { return src[:length], src[length:], nil } -func writeRollup(w io.Writer, r rollup) error { - var fixed [160]byte - binary.LittleEndian.PutUint64(fixed[0:8], uint64(r.key.bucket)) - binary.LittleEndian.PutUint64(fixed[8:16], r.calls) - binary.LittleEndian.PutUint64(fixed[16:24], r.errors) - binary.LittleEndian.PutUint64(fixed[24:32], math.Float64bits(r.duration)) - for i, count := range r.bins { - binary.LittleEndian.PutUint32(fixed[32+i*4:], count) - } - if _, err := w.Write(fixed[:]); err != nil { - return fmt.Errorf("write rollup: %w", err) - } - // Key parts are varint-framed like every other string in the format: a - // pathological route must never make a batch unpublishable, because the WAL - // would then abort every restart with no way to make progress. - for _, value := range []string{r.key.namespace, r.key.service, r.key.method, r.key.route} { - var length [binary.MaxVarintLen64]byte - n := binary.PutUvarint(length[:], uint64(len(value))) - if _, err := w.Write(length[:n]); err != nil { - return err - } - if _, err := io.WriteString(w, value); err != nil { - return err - } - } - return nil -} - -func readRollup(r *bufio.Reader) (rollup, error) { - return readRollupWithLimit(r, maxRollupKeyBytes) -} - -func readRollupWithLimit(r *bufio.Reader, maxKeyBytes uint64) (rollup, error) { - var fixed [160]byte - if _, err := io.ReadFull(r, fixed[:]); err != nil { - return rollup{}, err - } - out := rollup{key: rollupKey{bucket: int64(binary.LittleEndian.Uint64(fixed[0:8]))}, calls: binary.LittleEndian.Uint64(fixed[8:16]), errors: binary.LittleEndian.Uint64(fixed[16:24]), duration: math.Float64frombits(binary.LittleEndian.Uint64(fixed[24:32]))} - for i := range out.bins { - out.bins[i] = binary.LittleEndian.Uint32(fixed[32+i*4:]) - } - remaining := maxKeyBytes - for _, target := range []*string{&out.key.namespace, &out.key.service, &out.key.method, &out.key.route} { - length, err := binary.ReadUvarint(r) - if err != nil { - return rollup{}, err - } - if length > remaining { - return rollup{}, errors.New("rollup key length is out of range") - } - remaining -= length - value := make([]byte, length) - if _, err := io.ReadFull(r, value); err != nil { - return rollup{}, err - } - *target = string(value) - } - return out, nil -} - -func durationBin(ms float64) int { - if ms <= 0 { - return 0 - } - bin := int(math.Log2(ms*1000 + 1)) - return min(bin, rollupBins-1) -} - -func histogramQuantile(bins [rollupBins]uint32, total uint64, q float64) float64 { - if total == 0 { - return 0 - } - target := uint64(math.Ceil(float64(total) * q)) - var seen uint64 - for i, count := range bins { - seen += uint64(count) - if seen >= target { - return (math.Pow(2, float64(i+1)) - 1) / 1000 - } - } - return (math.Pow(2, rollupBins) - 1) / 1000 -} - func writeManifest(dir string, m manifest) error { data, err := json.Marshal(m) if err != nil { diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go index 2d65932a..d062c443 100644 --- a/internal/telemetry/segment/span_store_test.go +++ b/internal/telemetry/segment/span_store_test.go @@ -2,15 +2,15 @@ package segment import ( - "bufio" - "bytes" "encoding/binary" - "github.com/klauspost/compress/zstd" "os" "path/filepath" "strings" "testing" "time" + + "github.com/klauspost/compress/zstd" + "github.com/zeebo/xxh3" ) func TestStoreCommitReopenAndQueries(t *testing.T) { @@ -58,10 +58,6 @@ func TestStoreCommitReopenAndQueries(t *testing.T) { if len(trace) != 2 || !EqualSpan(trace[0], rows[0]) || !EqualSpan(trace[1], rows[1]) { t.Fatalf("trace result = %#v", trace) } - endpoints := store.Endpoints("default", "api", base, base+int64(5*time.Minute), 10) - if len(endpoints) != 1 || endpoints[0].Calls != 2 || endpoints[0].Errors != 1 { - t.Fatalf("endpoint result = %#v", endpoints) - } agg, err := store.ScanService("default", "api", base, base+int64(5*time.Minute)) if err != nil { t.Fatal(err) @@ -159,6 +155,44 @@ func TestStoreCompactionPreservesRowsAndIndexes(t *testing.T) { } } +func TestTraceIndexIsReadLazilyFromDisk(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err := store.AppendID("lazy-index", []Span{{TraceID: "trace-a", SpanID: "span-a", StartUnixNanos: 1}}); err != nil { + t.Fatal(err) + } + store.mu.RLock() + seg := store.segments[0] + store.mu.RUnlock() + if seg.indexCount != 1 { + t.Fatalf("index count = %d, want 1", seg.indexCount) + } + f, err := os.OpenFile(seg.path, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + var replacement [8]byte + binary.LittleEndian.PutUint64(replacement[:], xxh3.HashString("trace-b")) + if _, err := f.WriteAt(replacement[:], int64(seg.indexOffset)); err != nil { + _ = f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + rows, err := store.Trace("trace-a") + if err != nil { + t.Fatal(err) + } + if len(rows) != 0 { + t.Fatalf("Trace returned %d rows from a stale resident index", len(rows)) + } +} + func TestStoreRejectsCorruptCommittedSegment(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "broken.fseg"), []byte("bad"), 0o644); err != nil { @@ -201,20 +235,20 @@ func TestOpenRejectsSegmentWithCorruptSectionOffsets(t *testing.T) { t.Fatal("Open succeeded with corrupt section offsets") } } - t.Run("rollup offset before index offset", func(t *testing.T) { + t.Run("index end before index offset", func(t *testing.T) { corrupt(t, func(header []byte) { indexOffset := binary.LittleEndian.Uint64(header[48:56]) binary.LittleEndian.PutUint64(header[56:64], indexOffset-1) }) }) - t.Run("rollup offset beyond file size", func(t *testing.T) { + t.Run("index end beyond file size", func(t *testing.T) { corrupt(t, func(header []byte) { binary.LittleEndian.PutUint64(header[56:64], 1<<40) }) }) } -func TestEndpointsCountCanonicalOTelErrorStatus(t *testing.T) { +func TestScanServiceCountsCanonicalOTelErrorStatus(t *testing.T) { dir := t.TempDir() base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() rows := []Span{ @@ -229,13 +263,6 @@ func TestEndpointsCountCanonicalOTelErrorStatus(t *testing.T) { if err := store.AppendID("seg-status", rows); err != nil { t.Fatal(err) } - endpoints := store.Endpoints("default", "api", base, base+int64(time.Minute), 10) - if len(endpoints) != 1 { - t.Fatalf("endpoints = %#v, want one route", endpoints) - } - if endpoints[0].Errors != 1 { - t.Fatalf("Errors = %d, want 1: OTLP ingest stores Status.Code.String() as STATUS_CODE_ERROR", endpoints[0].Errors) - } agg, err := store.ScanService("default", "api", base, base+int64(time.Minute)) if err != nil { t.Fatal(err) @@ -248,24 +275,25 @@ func TestEndpointsCountCanonicalOTelErrorStatus(t *testing.T) { func TestValidateSegmentSectionsRejectsOutOfBoundsDirectory(t *testing.T) { const size = 4096 tests := []struct { - name string - dirOffset, indexOffset, rollupOffset uint64 - blockCount uint32 + name string + dirOffset, indexOffset, indexEnd uint64 + blockCount, indexCount uint32 }{ - {"wrapping directory end", ^uint64(0) - uint64(0xFFFFFFFF)*blockDirSize + 1, 512, 1024, 0xFFFFFFFF}, - {"block count past index", headerSize, 512, 1024, 0xFFFFFFFF}, - {"directory before header", 0, 512, 1024, 1}, - {"sections out of order", headerSize, 2048, 1024, 1}, - {"rollups past end of file", headerSize, 512, size + 1, 1}, + {"wrapping directory end", ^uint64(0) - uint64(0xFFFFFFFF)*blockDirSize + 1, 512, 512, 0xFFFFFFFF, 0}, + {"block count past index", headerSize, 512, 512, 0xFFFFFFFF, 0}, + {"directory before header", 0, 512, 512, 1, 0}, + {"sections out of order", headerSize, 2048, 1024, 1, 0}, + {"index past end of file", headerSize, 512, size + 1, 1, 0}, + {"index count exceeds extent", headerSize, 512, 512, 1, 1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if err := validateSegmentSections(size, test.dirOffset, test.indexOffset, test.rollupOffset, test.blockCount); err == nil { + if err := validateSegmentSections(size, test.dirOffset, test.indexOffset, test.indexEnd, test.blockCount, test.indexCount); err == nil { t.Fatal("validateSegmentSections accepted a corrupt header") } }) } - if err := validateSegmentSections(size, headerSize, 512, 1024, 8); err != nil { + if err := validateSegmentSections(size, headerSize, size, size, 8, 0); err != nil { t.Fatalf("validateSegmentSections rejected a sound header: %v", err) } } @@ -332,7 +360,7 @@ func TestOpenRejectsSegmentWithCorruptBlockEntry(t *testing.T) { } } -func TestStoreAcceptsSpanWithOversizedRollupKey(t *testing.T) { +func TestStoreAcceptsSpanWithLargeRoute(t *testing.T) { dir := t.TempDir() base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() // A pathological route must not be a deterministic publish failure: the WAL @@ -344,7 +372,7 @@ func TestStoreAcceptsSpanWithOversizedRollupKey(t *testing.T) { t.Fatal(err) } if err := store.AppendID("seg-big-key", rows); err != nil { - t.Fatalf("AppendID error = %v, want an oversized rollup key to be publishable", err) + t.Fatalf("AppendID error = %v, want a large but bounded route to be publishable", err) } if err := store.Close(); err != nil { t.Fatal(err) @@ -354,9 +382,12 @@ func TestStoreAcceptsSpanWithOversizedRollupKey(t *testing.T) { t.Fatalf("Open error = %v, want the segment to round-trip", err) } defer reopened.Close() - endpoints := reopened.Endpoints("default", "api", base, base+int64(time.Minute), 10) - if len(endpoints) != 1 || endpoints[0].Route != route { - t.Fatalf("endpoints = %d entries, want the full route preserved", len(endpoints)) + trace, err := reopened.Trace("t") + if err != nil { + t.Fatal(err) + } + if len(trace) != 1 || trace[0].HTTPRoute != route { + t.Fatalf("trace = %#v, want the full route preserved", trace) } } @@ -385,21 +416,13 @@ func TestSegmentDecoderRejectsOversizedFrame(t *testing.T) { } } -func TestReadRollupRejectsLengthBeforeAllocating(t *testing.T) { - payload := make([]byte, 160, 170) - payload = binary.AppendUvarint(payload, 9) - if _, err := readRollupWithLimit(bufio.NewReader(bytes.NewReader(payload)), 8); err == nil { - t.Fatal("readRollup accepted a disk-controlled allocation above its budget") - } -} - -func TestValidateSpanRowsRejectsUnreopenableRollups(t *testing.T) { - rows := []Span{{HTTPRoute: "123456789"}} - if err := validateSpanRowsWithLimits(rows, 8, 1024); err == nil { - t.Fatal("validator accepted a rollup key larger than the reader budget") - } - rows = []Span{{HTTPRoute: "a"}, {HTTPRoute: "b"}} - if err := validateSpanRowsWithLimits(rows, 1024, 200); err == nil { - t.Fatal("validator accepted a rollup section larger than the decoder budget") +func TestValidateSpanRowsRejectsUnreopenableBlock(t *testing.T) { + rows := []Span{{AttributesJSON: []byte(strings.Repeat("x", 1024))}} + if err := validateSpanRowsWithLimit(rows, 512); err == nil || !strings.Contains(err.Error(), "column") { + t.Fatal("validator accepted a column larger than the decoder budget") + } + rows = []Span{{TraceID: "a", SpanID: "b", HTTPRoute: "c"}} + if err := validateSpanRowsWithLimit(rows, uint64(columnarHeaderSize+3)); err == nil { + t.Fatal("validator accepted a block larger than the decoder budget") } } diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index be9daec5..7e63f307 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -38,6 +38,12 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches } r.compactionMu.Lock() defer r.compactionMu.Unlock() + markerPath := filepath.Join(r.root, "COMPACTION.json") + if exists, err := pathExists(markerPath); err != nil { + return 0, fmt.Errorf("inspect pending compaction: %w", err) + } else if exists { + return 0, errors.New("pending Parquet compaction must recover before another can start") + } r.mu.RLock() selected := selectCompactionBatches(r.manifest.Batches, maxBatches) r.mu.RUnlock() @@ -102,7 +108,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if err != nil { return 0, err } - if err := writeDurableFile(filepath.Join(r.root, "COMPACTION.json"), data); err != nil { + if err := writeDurableFile(markerPath, data); err != nil { return 0, err } if err := syncDirectory(r.root); err != nil { diff --git a/internal/telemetry/store/publication_test.go b/internal/telemetry/store/publication_test.go index 18434704..557cef51 100644 --- a/internal/telemetry/store/publication_test.go +++ b/internal/telemetry/store/publication_test.go @@ -9,10 +9,11 @@ import ( ) type publicationInspectLock struct { - t *testing.T - repository *Repository - id string - locked bool + t *testing.T + repository *Repository + id string + locked bool + unlockedBeforeHot bool } func (l *publicationInspectLock) Lock() { @@ -28,7 +29,13 @@ func (l *publicationInspectLock) Lock() { } } -func (l *publicationInspectLock) Unlock() { l.locked = false } +func (l *publicationInspectLock) Unlock() { + if rows := l.repository.Spans.RowCount(); rows != 0 { + l.t.Fatalf("hot segment encoded while query publication gate was held: %d rows", rows) + } + l.unlockedBeforeHot = true + l.locked = false +} func TestCommitStagesOutsidePublicationLock(t *testing.T) { repository, err := Open(t.TempDir()) @@ -51,6 +58,9 @@ func TestCommitStagesOutsidePublicationLock(t *testing.T) { if lock.locked { t.Fatal("publication lock remained held after commit") } + if !lock.unlockedBeforeHot { + t.Fatal("publication lock was not released before hot-index encoding") + } for _, signal := range []string{"spans", "logs", "metrics"} { if _, err := os.Stat(filepath.Join(repository.Parquet.Dir(), signal, id+".parquet")); err != nil { t.Fatalf("%s final file: %v", signal, err) diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 683a764e..c8825aef 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -110,7 +110,7 @@ func Open(root string) (*Repository, error) { return nil, fmt.Errorf("rebuild hot telemetry tier: %w", err) } hotRebuilt = true - slog.Warn("corrupt hot telemetry tier quarantined and rebuilt from authoritative Parquet", "path", quarantine) + slog.Warn("corrupt hot telemetry tier quarantined and reset; authoritative Parquet preserved", "path", quarantine) } parquet, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { @@ -156,8 +156,6 @@ func (r *Repository) Close() error { // PruneHot removes acceleration segments older than cutoff. Parquet remains // authoritative for longer retention and SQL queries. func (r *Repository) PruneHot(cutoff int64) (int, error) { - r.commitMu.Lock() - defer r.commitMu.Unlock() r.hotMu.Lock() defer r.hotMu.Unlock() @@ -215,15 +213,12 @@ func (r *Repository) CompactHot(maxInputs int) (int, error) { // HotTrace returns the hot trace snapshot and the durable prune boundary that // was in force for that snapshot. -func (r *Repository) HotTrace(traceID string, scopeStartNanos int64) ([]telemetry.Span, int64, error) { +func (r *Repository) HotTrace(traceID string) ([]telemetry.Span, int64, error) { r.hotMu.RLock() defer r.hotMu.RUnlock() r.mu.RLock() cutoff := r.manifest.HotCutoffNanos r.mu.RUnlock() - if scopeStartNanos < cutoff { - return nil, cutoff, nil - } spans, err := r.Spans.Trace(traceID) return spans, cutoff, err } @@ -285,21 +280,35 @@ func (r *Repository) Commit(batch Batch) error { return err } // Parquet encoding and fsync happen before either the query publication gate - // or commit mutex is acquired. Only the final renames and manifest append are - // serialized with readers and maintenance. + // or commit mutex is acquired. The query gate covers only the final Parquet + // renames; the hot index, journal, and WAL cleanup cannot block readers. if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { return err } + publishLocked := false + unlockPublish := func() { + if publishLocked { + r.parquetPublish.Unlock() + publishLocked = false + } + } if r.parquetPublish != nil { r.parquetPublish.Lock() - defer r.parquetPublish.Unlock() + publishLocked = true + defer unlockPublish() } r.commitMu.Lock() defer r.commitMu.Unlock() if r.batchConsumedLocked(batch.ID) { + unlockPublish() return errors.Join(r.Parquet.DiscardBatch(batch.ID), r.removeWAL(batch.ID)) } - err = r.publish(batch) + _, err = r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0) + unlockPublish() + if err != nil { + return fmt.Errorf("publish parquet batch: %w", err) + } + err = r.publishHot(batch) if err == nil { r.mu.Lock() err = r.recordBatch(batch) @@ -350,15 +359,14 @@ func validateBatch(batch Batch) error { return nil } -func (r *Repository) publish(batch Batch) error { +func (r *Repository) publishHot(batch Batch) error { + // Parquet is authoritative and already queryable. If the disposable hot + // index fails, retain the WAL and retry/recover only that acceleration copy; + // a hot miss safely falls back to Parquet in the meantime. r.hotMu.Lock() defer r.hotMu.Unlock() - rollback, err := r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0) - if err != nil { - return fmt.Errorf("publish parquet batch: %w", err) - } if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { - return errors.Join(fmt.Errorf("commit span segment: %w", err), rollback()) + return fmt.Errorf("commit span segment: %w", err) } return nil } @@ -481,8 +489,11 @@ func (r *Repository) recover() error { if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { return fmt.Errorf("stage replay %s: %w", name, err) } - if err := r.publish(batch); err != nil { - return fmt.Errorf("replay %s: %w", name, err) + if _, err := r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0); err != nil { + return fmt.Errorf("publish replayed Parquet %s: %w", name, err) + } + if err := r.publishHot(batch); err != nil { + return fmt.Errorf("publish replayed hot index %s: %w", name, err) } if err := r.recordBatch(batch); err != nil { return fmt.Errorf("record replayed %s: %w", name, err) diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index d7d62283..a67b81b7 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -327,20 +327,13 @@ func TestRepositoryPersistsHotPruneBoundary(t *testing.T) { t.Fatal(err) } defer reopened.Close() - spans, cutoff, err := reopened.HotTrace("trace-boundary", 250) + spans, cutoff, err := reopened.HotTrace("trace-boundary") if err != nil { t.Fatal(err) } if cutoff != 250 || len(spans) != 1 { t.Fatalf("cutoff=%d spans=%d, want cutoff 250 and one retained boundary span", cutoff, len(spans)) } - skipped, cutoff, err := reopened.HotTrace("trace-boundary", 249) - if err != nil { - t.Fatal(err) - } - if cutoff != 250 || len(skipped) != 0 { - t.Fatalf("cross-boundary lookup cutoff=%d spans=%d, want Parquet handoff without a hot scan", cutoff, len(skipped)) - } } func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { @@ -679,6 +672,35 @@ func TestRepositoryCompactionRollsBackMidSwapFailure(t *testing.T) { } } +func TestRepositoryRefusesToOverwritePendingCompactionMarker(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + markerPath := filepath.Join(dir, "COMPACTION.json") + original := []byte(`{"id":"compact-pending"}`) + if err := writeDurableFile(markerPath, original); err != nil { + t.Fatal(err) + } + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := repository.CompactParquet(context.Background(), db, 64, nil); err == nil || !strings.Contains(err.Error(), "must recover") { + t.Fatalf("CompactParquet error = %v, want pending-marker refusal", err) + } + got, err := os.ReadFile(markerPath) + if err != nil { + t.Fatal(err) + } + if string(got) != string(original) { + t.Fatalf("pending marker was overwritten: %q", got) + } +} + func testBatchAt(timestamp int64) Batch { batch := testBatch() batch.Spans[0].StartUnixNanos = timestamp diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 964a7a56..38579794 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -119,7 +119,7 @@ func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDon oversized := requests[0] requests = requests[1:] chunks := splitBatch(oversized.batch, limit) - staged := chunks[:0] + staged := make([]Batch, 0, len(chunks)) var stageErr error for _, chunk := range chunks { chunk.ID = uuid.NewString() diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index ac14db82..8aa2b252 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -17,7 +17,7 @@ state—users, sessions, dashboards, alert rules, and agent history—lives in S | WAL | Acknowledges an OTLP request only after its complete bounded batch is durable | | Parquet | Authoritative retained spans, logs, and metrics | | DuckDB | SQL, log filtering, aggregation, and broad analytical reads | -| Hot span index | Fast trace lookup and service/endpoint rollups for recent spans | +| Hot span index | Fast trace lookup for recent spans | | Manifest journal | Constant-time commit ledger and crash-safe Parquet file lifecycle | | SQLite | Transactional application and identity state | @@ -43,9 +43,10 @@ Logs and metrics are written once, to Parquet. DuckDB applies filters, ordering, copies would add write latency and compaction work without serving a production query. -Spans additionally use a compact purpose-built index because trace lookup and -service rollups benefit from it. Parquet remains authoritative when the recent -span index is pruned or rebuilt. +Spans additionally use a compact purpose-built on-disk index because trace +lookup benefits from it. Parquet remains authoritative when the recent span +index is pruned or rebuilt. Dashboard rollups are rebuildable DuckDB caches, +not a second resident copy in the hot index. ## Small files are bounded by compaction From 2bf8822fcbb70d858ecb69e80b6f93c6e789f4aa Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 06:58:48 -0700 Subject: [PATCH 13/31] chore(bench): keep storage POC local Remove exploratory engine comparisons and benchmark notes from reviewable source. Local copies remain under repository-specific Git excludes. --- bench/storage/chdb/go.mod | 38 -- bench/storage/chdb/go.sum | 88 ----- bench/storage/chdb/main.go | 250 ------------- bench/storage/main.go | 525 --------------------------- docs/storage-architecture-options.md | 409 --------------------- docs/storage-benchmark.md | 107 ------ internal/storagebench/data.go | 45 --- 7 files changed, 1462 deletions(-) delete mode 100644 bench/storage/chdb/go.mod delete mode 100644 bench/storage/chdb/go.sum delete mode 100644 bench/storage/chdb/main.go delete mode 100644 bench/storage/main.go delete mode 100644 docs/storage-architecture-options.md delete mode 100644 docs/storage-benchmark.md delete mode 100644 internal/storagebench/data.go diff --git a/bench/storage/chdb/go.mod b/bench/storage/chdb/go.mod deleted file mode 100644 index ad066034..00000000 --- a/bench/storage/chdb/go.mod +++ /dev/null @@ -1,38 +0,0 @@ -module github.com/labstack/fanout/bench/storage/chdb - -go 1.27.0 - -require ( - github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 - github.com/chdb-io/chdb-go/v2 v2.1.0 - github.com/labstack/fanout v0.0.0 -) - -require ( - github.com/andybalholm/brotli v1.2.2 // indirect - github.com/apache/arrow-go/v18 v18.7.0 // indirect - github.com/apache/thrift v0.24.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 // indirect - github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 // indirect - github.com/ebitengine/purego v0.8.2 // indirect - github.com/goccy/go-json v0.10.6 // indirect - github.com/google/flatbuffers v25.12.19+incompatible // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.19.2 // indirect - github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/pierrec/lz4/v4 v4.1.29 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect - golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.12 // indirect -) - -replace github.com/labstack/fanout => ../../.. diff --git a/bench/storage/chdb/go.sum b/bench/storage/chdb/go.sum deleted file mode 100644 index 4ec371e6..00000000 --- a/bench/storage/chdb/go.sum +++ /dev/null @@ -1,88 +0,0 @@ -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= -github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= -github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= -github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1 h1:QiXQ1vZWcQCbRpciirWG/+F3KRXYnDLiFOVYkAJxCls= -github.com/chdb-io/chdb-go/lib/darwin-amd64 v0.260700.1/go.mod h1:jB7U0oct7fDV+SbrDzh3oorQFAQ8YOM5QFXf273ZEEs= -github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1 h1:qvkIS/fozvgJfd30MEPM1nVDSM1JMXQgqZfqp1Oh7aQ= -github.com/chdb-io/chdb-go/lib/darwin-arm64 v0.260700.1/go.mod h1:tebe6DiYx113PoHD0WjWCWVY74QDmaPcCdWA+aNpWPM= -github.com/chdb-io/chdb-go/lib/embedded v0.260700.1 h1:FBSXH0ChVm7fOEHUWpf3bNd2BKnp1cv9LhEn1eLcUdE= -github.com/chdb-io/chdb-go/lib/embedded v0.260700.1/go.mod h1:N9Dra/RfDuELfnT2TSMVBF+NzyDKL6AFiDqHwCWF7T0= -github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1 h1:w1q1whc7LlBpJtzoMkokgnSsdRrFLAG2kgRkb1tQ7jM= -github.com/chdb-io/chdb-go/lib/linux-amd64 v0.260700.1/go.mod h1:LK3ORN5rYtQDUYKswMZKf09RT1qIdwlCGk/3/vB7aHE= -github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1 h1:GoviqHKnVOJIfnfgiSYWiEqxciThy2XGmSHUdq1qvmI= -github.com/chdb-io/chdb-go/lib/linux-arm64 v0.260700.1/go.mod h1:vkAVOjzg+j6TwFWobfxvmU+pV8fD7dS9ibBdZc3Wf8Y= -github.com/chdb-io/chdb-go/v2 v2.1.0 h1:Nf/StmYfE90mePp0EzdWqCbqUv/TJUbVOrnea/x3PN0= -github.com/chdb-io/chdb-go/v2 v2.1.0/go.mod h1:tyiHoF8pWUfrD7ylseofFEnELnA+jocf/yElE3AHBaQ= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= -github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= -github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= -github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= -github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= -github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= -github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= -golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/bench/storage/chdb/main.go b/bench/storage/chdb/main.go deleted file mode 100644 index ed448c51..00000000 --- a/bench/storage/chdb/main.go +++ /dev/null @@ -1,250 +0,0 @@ -package main - -import ( - "bytes" - "encoding/csv" - "flag" - "fmt" - "os" - "path/filepath" - "runtime" - "sort" - "strconv" - "sync" - "time" - - _ "github.com/chdb-io/chdb-go/lib/embedded" - "github.com/chdb-io/chdb-go/v2/chdb" - "github.com/labstack/fanout/internal/storagebench" - "github.com/labstack/fanout/internal/telemetry/segment" -) - -func main() { - rows := flag.Int("rows", 1_000_000, "number of synthetic spans") - batch := flag.Int("batch", 50_000, "rows per insert") - repeats := flag.Int("repeats", 11, "query repetitions") - mixedRows := flag.Int("mixed-rows", 200_000, "additional rows written while trace reads run at 100 qps") - dir := flag.String("dir", "", "session directory; temporary when empty") - flag.Parse() - root := *dir - if root == "" { - var err error - root, err = os.MkdirTemp("", "fanout-chdb-poc-") - if err != nil { - fatal(err) - } - defer os.RemoveAll(root) - } else if err := os.MkdirAll(root, 0o755); err != nil { - fatal(err) - } - base := time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) - targetTrace := storagebench.TraceID(uint64(*rows / 2 / 5)) - initStart := time.Now() - session, err := chdb.NewSession(filepath.Join(root, "engine")) - if err != nil { - fatal(err) - } - initElapsed := time.Since(initStart) - defer session.Close() - for _, statement := range []string{ - "CREATE DATABASE fanout", spansDDL, endpointDDL, endpointMVDDL, - fmt.Sprintf("SET max_threads=%d", runtime.NumCPU()), "SET date_time_input_format='best_effort'", - } { - query(session, statement, "Null") - } - - writeStart := time.Now() - insertRange(session, 0, *rows, *rows, *batch, base.UnixNano()) - writeElapsed := time.Since(writeStart) - maintenanceStart := time.Now() - query(session, "OPTIMIZE TABLE fanout.spans FINAL", "Null") - query(session, "OPTIMIZE TABLE fanout.endpoint_rollup FINAL", "Null") - maintenanceElapsed := time.Since(maintenanceStart) - activeBytes, err := strconv.ParseInt(queryText(session, "SELECT sum(bytes_on_disk) FROM system.parts WHERE active AND database='fanout'"), 10, 64) - if err != nil { - fatal(fmt.Errorf("parse active bytes: %w", err)) - } - - startLiteral, endLiteral := "2026-08-25 00:00:00", "2026-08-26 00:00:00" - endpointSQL := fmt.Sprintf(`SELECT service_name,http_method,http_route,sum(calls),sum(errors),sum(duration_sum)/sum(calls),max(p95_ms) - FROM fanout.endpoint_rollup WHERE namespace='default' AND service_name='service-00' - AND bucket>=toDateTime64('%s',9,'UTC') AND bucket=toDateTime64('%s',9,'UTC') AND start_time= time.Second { - return fmt.Sprintf("%.2fs", value.Seconds()) - } - if value >= time.Millisecond { - return fmt.Sprintf("%.2fms", float64(value)/float64(time.Millisecond)) - } - return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) -} - -func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-bench-chdb:", err); os.Exit(1) } - -const spansDDL = `CREATE TABLE fanout.spans ( - namespace String,trace_id String,span_id String,parent_span_id String,service_name LowCardinality(String),name LowCardinality(String),kind LowCardinality(String), - start_time DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4),end_time DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4), - start_unix_nano Int64 CODEC(DoubleDelta,LZ4),end_unix_nano Int64 CODEC(DoubleDelta,LZ4),duration_ms Float64 CODEC(Gorilla,LZ4), - status_code LowCardinality(String),status_msg String CODEC(ZSTD(1)),resource_json String CODEC(ZSTD(1)),attributes_json String CODEC(ZSTD(1)), - events_json String CODEC(ZSTD(1)),links_json String CODEC(ZSTD(1)),trace_state String,flags UInt32,scope_name LowCardinality(String),scope_version String, - ingested_at DateTime64(9,'UTC') CODEC(DoubleDelta,LZ4),ingested_unix_nano Int64 CODEC(DoubleDelta,LZ4),http_method LowCardinality(String), - http_status_code LowCardinality(String),http_route LowCardinality(String),db_system LowCardinality(String),rpc_method LowCardinality(String), - rpc_service LowCardinality(String),peer_service LowCardinality(String),service_version LowCardinality(String),deployment_env LowCardinality(String), - exception_type LowCardinality(String),exception_message String CODEC(ZSTD(1)), - tenant_id LowCardinality(String) MATERIALIZED JSONExtractString(attributes_json,'tenant'), - PROJECTION by_trace INDEX trace_id TYPE basic,PROJECTION by_tenant INDEX tenant_id TYPE basic -) ENGINE=MergeTree PARTITION BY toDate(start_time) ORDER BY (namespace,start_time,service_name) SETTINGS old_parts_lifetime=0` - -const endpointDDL = `CREATE TABLE fanout.endpoint_rollup ( - namespace String,bucket DateTime64(9,'UTC'),service_name LowCardinality(String),http_method LowCardinality(String),http_route LowCardinality(String), - calls SimpleAggregateFunction(sum,UInt64),errors SimpleAggregateFunction(sum,UInt64),duration_sum SimpleAggregateFunction(sum,Float64), - p95_ms SimpleAggregateFunction(max,Float64) -) ENGINE=AggregatingMergeTree PARTITION BY toDate(bucket) ORDER BY (namespace,bucket,service_name,http_method,http_route) SETTINGS old_parts_lifetime=0` - -const endpointMVDDL = `CREATE MATERIALIZED VIEW fanout.endpoint_rollup_mv TO fanout.endpoint_rollup AS -SELECT namespace,toStartOfInterval(start_time,INTERVAL 5 MINUTE) AS bucket,service_name,http_method,http_route, - count() AS calls,countIf(status_code='ERROR') AS errors,sum(duration_ms) AS duration_sum,quantileTDigest(0.95)(duration_ms) AS p95_ms -FROM fanout.spans GROUP BY namespace,bucket,service_name,http_method,http_route` diff --git a/bench/storage/main.go b/bench/storage/main.go deleted file mode 100644 index f28a3f1f..00000000 --- a/bench/storage/main.go +++ /dev/null @@ -1,525 +0,0 @@ -// Command storage-bench compares Fanout's storage path with alternative formats -// with native DuckDB and Parquet on Fanout-shaped spans. It is an experiment, -// not a supported Fanout command. -package main - -import ( - "context" - "database/sql" - "database/sql/driver" - "errors" - "flag" - "fmt" - "os" - "path/filepath" - "runtime" - "sort" - "sync" - "time" - - duckdb "github.com/duckdb/duckdb-go/v2" - "github.com/labstack/fanout/internal/storagebench" - "github.com/labstack/fanout/internal/telemetry/segment" - telemetrystore "github.com/labstack/fanout/internal/telemetry/store" -) - -type result struct { - name string - writeRate float64 - rollupBuild time.Duration - maintenance time.Duration - diskBytes int64 - endpoint time.Duration - trace time.Duration - rawService time.Duration - recovery time.Duration - queryRowCount uint64 - mixedWrite float64 - mixedReadP95 time.Duration -} - -func main() { - rows := flag.Int("rows", 1_000_000, "number of synthetic spans") - batch := flag.Int("batch", 50_000, "rows per durable append") - repeats := flag.Int("repeats", 21, "query repetitions used for medians") - mixedRows := flag.Int("mixed-rows", 200_000, "additional rows written while trace reads run at 100 qps") - engine := flag.String("engine", "all", "engines to run: all, repository, custom, or duck") - keep := flag.String("keep", "", "keep artifacts in this directory instead of a temporary directory") - flag.Parse() - if *rows <= 0 || *batch <= 0 || *repeats <= 0 { - fmt.Fprintln(os.Stderr, "rows, batch, and repeats must be positive") - os.Exit(2) - } - - root := *keep - if root == "" { - var err error - root, err = os.MkdirTemp("", "fanout-storage-bench-") - if err != nil { - fatal(err) - } - defer os.RemoveAll(root) - } else if err := os.MkdirAll(root, 0o755); err != nil { - fatal(err) - } - - base := time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC).UnixNano() - targetTrace := storagebench.TraceID(uint64(*rows / 2 / 5)) - start := base - end := base + storagebench.DayNanos - fmt.Printf("Fanout storage benchmark: %d spans, %d-row commits, %s/%s, %d CPUs\n", *rows, *batch, runtime.GOOS, runtime.GOARCH, runtime.NumCPU()) - - var results []result - if *engine == "all" || *engine == "repository" { - repositoryResult, err := runRepository(filepath.Join(root, "repository"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) - if err != nil { - fatal(fmt.Errorf("production repository: %w", err)) - } - results = append(results, repositoryResult) - } - var custom result - if *engine == "all" || *engine == "custom" { - var err error - custom, err = runCustom(filepath.Join(root, "fanseg"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) - if err != nil { - fatal(fmt.Errorf("custom segments: %w", err)) - } - results = append(results, custom) - } - if *engine == "all" || *engine == "duck" { - duck, parquet, err := runDuck(filepath.Join(root, "duck.db"), filepath.Join(root, "parquet"), *rows, *batch, *repeats, *mixedRows, base, start, end, targetTrace) - if err != nil { - fatal(fmt.Errorf("duckdb: %w", err)) - } - results = append(results, duck, parquet) - } - if len(results) == 0 { - fatal(fmt.Errorf("unknown engine %q", *engine)) - } - - fmt.Println() - fmt.Printf("%-20s %14s %13s %12s %12s %12s %12s %12s\n", "storage / execution", "write rows/s", "rollup build", "maintenance", "disk MiB", "endpoint", "trace", "raw service") - for _, r := range results { - rollup := "live" - if r.rollupBuild > 0 { - rollup = formatDuration(r.rollupBuild) - } - fmt.Printf("%-20s %14.0f %13s %12s %12.1f %12s %12s %12s\n", r.name, r.writeRate, rollup, formatDuration(r.maintenance), float64(r.diskBytes)/(1<<20), formatOptionalDuration(r.endpoint), formatDuration(r.trace), formatDuration(r.rawService)) - } - fmt.Println("\nMixed load: committed writes plus full trace reads at 100 qps") - fmt.Printf("%-20s %14s %14s\n", "storage / execution", "write rows/s", "trace p95") - for _, r := range results { - if r.mixedWrite == 0 { - continue - } - fmt.Printf("%-20s %14.0f %14s\n", r.name, r.mixedWrite, formatDuration(r.mixedReadP95)) - } - if custom.name != "" { - fmt.Printf("\nfanseg reopen/recovery: %s; trace rows: %d\n", formatDuration(custom.recovery), custom.queryRowCount) - } - if *keep != "" { - fmt.Printf("artifacts: %s\n", root) - } -} - -func runRepository(dir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, error) { - repository, err := telemetrystore.Open(dir) - if err != nil { - return result{}, err - } - defer repository.Close() - writeStart := time.Now() - for offset := 0; offset < total; offset += batch { - rows := storagebench.Rows(offset, min(batch, total-offset), total, base) - if err := repository.Commit(telemetrystore.Batch{ID: fmt.Sprintf("initial-%08d", offset), Spans: rows}); err != nil { - return result{}, err - } - } - writeElapsed := time.Since(writeStart) - var traceSink []segment.Span - trace := median(repeats, func() (err error) { traceSink, err = repository.Spans.Trace(targetTrace); return err }) - var aggregateSink segment.Aggregate - raw := median(repeats, func() (err error) { - aggregateSink, err = repository.Spans.ScanService("default", "service-00", start, end) - return err - }) - if aggregateSink.Calls == 0 { - return result{}, errors.New("production repository queries returned no rows") - } - disk, err := directoryBytes(dir) - if err != nil { - return result{}, err - } - mixedWrite, mixedP95, err := mixedLoad(mixedRows, - func() error { _, err := repository.Spans.Trace(targetTrace); return err }, - func() error { - finalTotal := total + mixedRows - for offset := 0; offset < mixedRows; offset += batch { - rows := storagebench.Rows(total+offset, min(batch, mixedRows-offset), finalTotal, base) - if err := repository.Commit(telemetrystore.Batch{ID: fmt.Sprintf("mixed-%08d", offset), Spans: rows}); err != nil { - return err - } - } - return nil - }) - if err != nil { - return result{}, err - } - return result{name: "Fanout + Parquet", writeRate: float64(total) / writeElapsed.Seconds(), diskBytes: disk, trace: trace, rawService: raw, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil -} - -func directoryBytes(root string) (int64, error) { - var total int64 - err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if entry.Type().IsRegular() { - info, err := entry.Info() - if err != nil { - return err - } - total += info.Size() - } - return nil - }) - return total, err -} - -func runCustom(dir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, error) { - store, err := segment.Open(dir) - if err != nil { - return result{}, err - } - writeStart := time.Now() - for offset := 0; offset < total; offset += batch { - count := min(batch, total-offset) - if err := store.Append(storagebench.Rows(offset, count, total, base)); err != nil { - return result{}, err - } - } - writeElapsed := time.Since(writeStart) - maintenanceStart := time.Now() - if err := store.CompactOldest(store.SegmentCount()); err != nil { - return result{}, err - } - maintenance := time.Since(maintenanceStart) - disk, err := store.DiskBytes() - if err != nil { - return result{}, err - } - if err := store.Close(); err != nil { - return result{}, err - } - reopenStart := time.Now() - store, err = segment.Open(dir) - if err != nil { - return result{}, err - } - recovery := time.Since(reopenStart) - defer store.Close() - - var traceSink []segment.Span - trace := median(repeats, func() (err error) { traceSink, err = store.Trace(targetTrace); return err }) - var aggSink segment.Aggregate - raw := median(repeats, func() (err error) { aggSink, err = store.ScanService("default", "service-00", start, end); return err }) - if aggSink.Calls == 0 { - return result{}, fmt.Errorf("queries returned no rows") - } - mixedWrite, mixedP95, err := mixedCustom(store, total, mixedRows, batch, base, targetTrace) - if err != nil { - return result{}, err - } - return result{name: "fanseg + direct", writeRate: float64(total) / writeElapsed.Seconds(), maintenance: maintenance, diskBytes: disk, trace: trace, rawService: raw, recovery: recovery, queryRowCount: uint64(len(traceSink)), mixedWrite: mixedWrite, mixedReadP95: mixedP95}, nil -} - -func runDuck(dbPath, parquetDir string, total, batch, repeats, mixedRows int, base, start, end int64, targetTrace string) (result, result, error) { - connector, err := duckdb.NewConnector(dbPath, nil) - if err != nil { - return result{}, result{}, err - } - db := sql.OpenDB(connector) - db.SetMaxOpenConns(4) - defer db.Close() - if _, err := db.Exec(`CREATE TABLE spans ( - namespace VARCHAR, trace_id VARCHAR, span_id VARCHAR, parent_span_id VARCHAR, - service_name VARCHAR, name VARCHAR, kind VARCHAR, start_time TIMESTAMP_NS, - end_time TIMESTAMP_NS, start_ns BIGINT, end_ns BIGINT, duration_ms DOUBLE, - status_code VARCHAR, status_msg VARCHAR, resource JSON, attributes JSON, - events JSON, links JSON, trace_state VARCHAR, flags UINTEGER, - scope_name VARCHAR, scope_version VARCHAR, ingested_at TIMESTAMP_NS, - ingested_ns BIGINT, http_method VARCHAR, http_status_code VARCHAR, - http_route VARCHAR, db_system VARCHAR, rpc_method VARCHAR, rpc_service VARCHAR, - peer_service VARCHAR, service_version VARCHAR, deployment_env VARCHAR, - exception_type VARCHAR, exception_message VARCHAR - )`); err != nil { - return result{}, result{}, err - } - - writeStart := time.Now() - for offset := 0; offset < total; offset += batch { - rows := storagebench.Rows(offset, min(batch, total-offset), total, base) - if err := appendDuck(db, rows); err != nil { - return result{}, result{}, err - } - } - writeElapsed := time.Since(writeStart) - rollupStart := time.Now() - if _, err := db.Exec(`CREATE TABLE endpoint_rollup AS - SELECT start_ns - start_ns % 300000000000 AS bucket, namespace, service_name, http_method, http_route, - count(*)::UBIGINT AS calls, count(*) FILTER (WHERE status_code='ERROR')::UBIGINT AS errors, - sum(duration_ms) AS duration_ms, approx_quantile(duration_ms, 0.95) AS p95_ms - FROM spans GROUP BY ALL`); err != nil { - return result{}, result{}, err - } - rollupBuild := time.Since(rollupStart) - checkpointStart := time.Now() - if _, err := db.Exec("CHECKPOINT"); err != nil { - return result{}, result{}, err - } - checkpointElapsed := time.Since(checkpointStart) - info, err := os.Stat(dbPath) - if err != nil { - return result{}, result{}, err - } - - endpointQuery := `SELECT service_name, http_method, http_route, sum(calls) calls, sum(errors) errors, - sum(duration_ms) / sum(calls) average_ms, max(p95_ms) p95_ms - FROM endpoint_rollup WHERE namespace=? AND service_name=? AND bucket>=? AND bucket=? AND start_ns=? AND bucket=? AND start_ns= time.Second { - return fmt.Sprintf("%.2fs", value.Seconds()) - } - if value >= time.Millisecond { - return fmt.Sprintf("%.2fms", float64(value)/float64(time.Millisecond)) - } - return fmt.Sprintf("%.2fµs", float64(value)/float64(time.Microsecond)) -} - -func formatOptionalDuration(value time.Duration) string { - if value == 0 { - return "n/a" - } - return formatDuration(value) -} - -func fatal(err error) { fmt.Fprintln(os.Stderr, "storage-bench:", err); os.Exit(1) } diff --git a/docs/storage-architecture-options.md b/docs/storage-architecture-options.md deleted file mode 100644 index ce4b541d..00000000 --- a/docs/storage-architecture-options.md +++ /dev/null @@ -1,409 +0,0 @@ -# Fanout storage architecture options - -**Decision date:** August 2026 - -**Product constraint:** one distributable Fanout binary -**Workload:** high-volume OTLP spans, logs, and metrics with fast dashboards, -trace lookup, attribute filtering, retention, and ad-hoc SQL - -## Recommendation - -Use a hybrid architecture: - -1. **Fanout columnar segments** as a rebuildable recent-span trace index. -2. **Direct Fanout execution** for recent trace lookup. -3. **Parquet** as the authoritative durable telemetry format, written in the - same commit. -4. **DuckDB** for arbitrary SQL over Parquet. -5. **SQLite** for control-plane data only. -6. **Do not use DuckLake, Iceberg, or chDB initially.** - -```text -OTLP ingestion - │ - ▼ -Fanout hot columnar store (.fseg) - ├── disk-resident trace index - ├── direct recent-trace execution - └── atomic manifest + streaming compaction - │ same WAL-backed transaction - ▼ - Parquet files - │ - ▼ - DuckDB ad-hoc SQL - -SQLite: users, configuration, sessions, alerts, and other control data -``` - -This is still a single-binary product. Fanout owns the high-performance hot -path; embedded DuckDB supplies a mature SQL engine without owning ingestion or -table lifecycle. - -## First: separate the layers - -Several technologies under consideration solve different problems and are not -direct substitutes. - -| Layer | Purpose | Candidates | -|---|---|---| -| Physical format | Encodes column values in files | Fanout segments, Parquet, MergeTree parts, DuckDB native pages | -| Table management | Tracks files, commits, snapshots, schema, and deletion | Fanout manifest, DuckLake, Iceberg v3 | -| Query execution | Plans and executes filters, joins, and aggregations | Fanout direct execution, DuckDB, ClickHouse through chDB | -| Control database | Stores small transactional product state | SQLite | - -Important distinctions: - -- **Parquet is a file format**, not a database or query engine. -- **Iceberg uses Parquet in this proposal** and adds table metadata and commit - semantics above it. -- **DuckLake is a table-management layer for DuckDB and Parquet.** -- **DuckDB is a query engine and native database.** It can query plain Parquet - without DuckLake. -- **chDB embeds ClickHouse.** Its primary format is ClickHouse MergeTree parts; - it can also read and write Parquet. -- **SQLite does not overlap with the telemetry engines.** It remains the right - database for Fanout's low-volume control plane. - -## Measured result - -The normalized benchmark used one million complete Fanout-shaped spans, -50,000-row commits, complete trace reads, a raw service aggregation, and -another 200,000 rows under concurrent trace load at 100 queries per second. -General-purpose engines also ran an endpoint-rollup query. - -| Storage / execution | Write rows/s | Endpoint | Full trace | Raw scan | Mixed write | Mixed trace p95 | Active disk | -|---|---:|---:|---:|---:|---:|---:|---:| -| **Production repository: Fanout + Parquet** | 161,115 | n/a | 0.846 ms | 28.63 ms | 162,544/s | 1.79 ms | 56.7 MiB | -| **Fanout columnar + direct** | **548,714** | n/a | **0.663 ms** | 34.93 ms | **538,201/s** | **0.984 ms** | 34.6 MiB | -| DuckDB native | 98,307 | **0.880 ms** | 1.45 ms | **1.21 ms** | 72,420/s | 2.08 ms | 47.8 MiB | -| Zstd Parquet + DuckDB | 95,305 effective | 1.21 ms | 9.44 ms | 3.45 ms | n/a | n/a | **21.8 MiB** | -| chDB MergeTree | 125,388 | 2.91 ms | 7.10 ms | 6.10 ms | 121,775/s | 12.41 ms | 38.1 MiB | - -Maintenance measurements: - -| Operation | Time | -|---|---:| -| Fanout compressed-block and index compaction | **142 ms** | -| DuckDB endpoint-rollup build | 166 ms | -| DuckDB checkpoint | 4.59 ms | -| Parquet export | 320 ms | -| chDB forced optimization | 4.05 s | - -The production-repository row includes the real atomic WAL + hot-segment + -Parquet commit path and was rerun on 2026-08-26. The isolated rows measure each -engine separately. These are development measurements from an Apple M3 Max, -not published capacity claims. The detailed methodology and reproduction commands are in -[storage-benchmark.md](storage-benchmark.md). - -## Options at a glance - -| Option | Writes | Product reads | Ad-hoc SQL | Open data | Complexity | Verdict | -|---|---|---|---|---|---|---| -| Fanout hot index + Parquet + DuckDB | **Best** | **Best** | Strong | Yes | Medium | **Recommended** | -| DuckDB native | Medium | Strong | **Best** | Export required | Low | Good simpler alternative | -| DuckLake + DuckDB + Parquet | Medium | Strong | Strong | Yes | Medium-high | Remove from new design | -| Iceberg v3 + Parquet + DuckDB | Medium-low | Strong | Strong | **Best** | High | Add only for shared object storage | -| chDB + MergeTree | Strong | Good | Strong | Export required | Medium | Not selected | -| Fully custom database and SQL engine | Potentially best | Potentially best | Weak initially | No | **Extreme** | Do not build | - -## Option A: Fanout hot store + Parquet + DuckDB - -### Components - -- Fanout-owned immutable columnar hot segments. -- Atomic Fanout manifest and crash recovery. -- A fixed-width on-disk trace index, searched lazily without a retention-sized - resident map. -- Streaming compaction that copies compressed blocks without decoding rows. -- Parquet files committed alongside each hot segment. -- DuckDB for dashboards, broad scans, and ad-hoc SQL over Parquet. -- SQLite for control data. - -### Benefits - -- Highest measured ingestion throughput. -- Lowest measured indexed-query latency. -- No C++ call in the ingestion hot path. -- Fanout can optimize precisely for append-only telemetry and TTL retention. -- Parquet preserves interoperability for the complete retained dataset. -- DuckDB retains featureful SQL without controlling ingestion. - -### Costs and risks - -- Fanout owns file-format compatibility, checksums, recovery, retention, and - compaction correctness. -- Recent spans have a second, rebuildable physical representation. -- Trace queries that cross the hot-retention boundary fall back to the - authoritative Parquet view. -- The current benchmark's broad scan is much slower than DuckDB. -- Long-run compaction tuning remains workload-driven. - -### Decision - -**Recommended.** It wins the overall Fanout objective while delegating general -SQL and interoperable cold storage to established components. - -## Option B: DuckDB native tables - -### Components - -- DuckDB native database for raw telemetry and rollups. -- DuckDB for all reads and SQL. -- SQLite for control data. -- Optional Parquet export. - -### Benefits - -- Simplest analytical architecture. -- Excellent broad scans and small dashboard queries. -- Mature SQL, joins, window functions, extensions, and vectorized execution. -- No separate lakehouse catalog is required. - -### Costs and risks - -- Ingestion was approximately five times slower than the custom hot store in - the full-shape benchmark. -- Peak RSS was much higher in a previous isolated comparison; the current - normalized rerun did not repeat RSS measurement. -- Scheduled rollup work remains outside ingestion. -- Native files are not an interoperable telemetry format. -- Export is required for other engines to consume the data. - -### Decision - -**Best simpler replacement if owning a hot format becomes too expensive.** It -is preferable to a more complicated DuckLake or Iceberg deployment when -everything remains inside one Fanout process. It is an architecture choice, -not a runtime fallback path. - -## Option C: DuckLake + DuckDB + Parquet - -### Components - -- Parquet data files. -- DuckLake metadata and commits, currently backed by a SQLite catalog. -- DuckDB reads and writes. -- A separate SQLite database for Fanout control data. - -### Benefits - -- Transactional table semantics over Parquet. -- DuckDB-native integration. -- Schema evolution, snapshots, and managed file lifecycle. -- Parquet remains externally readable. - -### Costs and overlap - -- DuckLake and the proposed Fanout manifest both manage file commits, - compaction, retention, and visibility. -- Catalog writes require serialization in the current single-process design. -- More maintenance paths exist than with DuckDB native tables. -- It does not improve the measured hot-path advantage of custom segments. -- The SQLite DuckLake catalog is separate from Fanout's control SQLite and - should never be conflated with it. - -### Decision - -**Remove from the new architecture.** DuckLake makes sense when DuckDB owns the -authoritative Parquet table. In the recommended design, Fanout owns the hot -table lifecycle and DuckDB is a secondary SQL executor. - -## Option D: Iceberg v3 + Parquet + DuckDB - -### Components - -- Parquet data files. -- Iceberg v3 table metadata, manifests, snapshots, schema and partition - evolution, and row-level change mechanisms. -- An Iceberg catalog. -- DuckDB and potentially other engines as readers. -- `iceberg-go` for Fanout writes and metadata commits. - -### Benefits - -- Strongest open, multi-engine table contract. -- Appropriate for S3/R2 and large long-lived datasets. -- Supports snapshot history, time travel, schema evolution, and multiple - independent consumers. -- Avoids binding the cold table to DuckDB. - -### Costs and overlap - -- Iceberg does not replace Parquet or the query engine. -- Snapshot and manifest planning add work above direct Parquet reads. -- It introduces a catalog and a more complex commit protocol. -- Small-file management becomes a first-class operational responsibility. -- It solves multi-engine and object-storage coordination that a single-process - Fanout appliance does not initially have. - -### Add Iceberg when - -- S3 or R2 becomes primary durable storage. -- Multiple Fanout writers commit to the same table. -- Spark, Trino, Flink, or another external engine must share authoritative - tables. -- Snapshot history and time travel become product requirements. -- Cold data outlives individual Fanout installations. - -### Decision - -**Do not include initially.** Keep the Parquet layout compatible with a later -Iceberg adoption, but do not pay its catalog and metadata cost before those -requirements exist. - -## Option E: chDB + ClickHouse MergeTree - -### Components - -- Embedded ClickHouse through `chdb-go` bindings. -- MergeTree raw tables. -- Materialized views and AggregatingMergeTree rollups. -- Projections and data-skipping indexes. -- SQLite for control data. - -### Benefits - -- One analytical engine handles ingestion, raw storage, rollups, TTL, indexes, - and SQL. -- Strong ClickHouse feature set. -- Better measured ingestion and memory than DuckDB in some tests. -- No Fanout-owned analytical file format is required. - -### Costs and risks - -- It was substantially slower than the custom path for ingestion, endpoint - reads, full trace reads, mixed load, and maintenance. -- The Go package is a binding to a large C++ engine, not a native-Go database. -- The embedded library is extracted and dynamically loaded at runtime. -- Cold initialization and extracted-library size are meaningful appliance - concerns. -- The current Go result and bulk-ingestion APIs are less mature than DuckDB's - appender path. -- MergeTree files are engine-specific; Parquet export is required for open - storage. - -### Decision - -**Not selected.** It is a credible one-engine architecture, but the normalized -benchmark no longer justifies its footprint and binding complexity for Fanout. - -## Option F: fully custom database - -This would include a custom storage format, WAL, catalog, indexes, compaction, -query planner, vectorized execution engine, SQL parser, joins, memory manager, -and transaction system. - -### Potential benefit - -- Complete control and the theoretical maximum performance for Fanout-specific - operations. - -### Why not - -- The benchmark already demonstrates that custom **storage and fixed execution** - provide most of the useful advantage. -- Building general SQL would duplicate years of DuckDB work. -- Correct recovery, concurrency, query planning, joins, spilling, and schema - evolution would dominate product development. - -### Decision - -**Do not build a general database.** Build a Fanout storage engine and use -DuckDB where general SQL is valuable. - -## Why Parquet remains - -Parquet is the one lakehouse component retained in the initial architecture. - -It provides: - -- the smallest measured representation; -- an open, documented columnar format; -- direct DuckDB reads; -- compatibility with future Iceberg adoption; -- straightforward export and backup; -- independence from Fanout's hot-format evolution. - -Parquet is published in every durable repository commit. Background compaction -combines small files without changing the authoritative format. - -## Proposed data lifecycle - -```text -1. Receive OTLP batch -2. Normalize and promote indexed attributes once -3. Durably stage the WAL and authoritative Parquet -4. Atomically publish Parquet under the reader gate -5. Publish the hot segment and commit journal, then remove the WAL -6. Answer recent complete traces through the hot index -7. Query dashboards, broad scans, and ad-hoc SQL with DuckDB -8. Stream-compact small Parquet and hot-segment files -9. Delete expired whole files through manifest commits -``` - -## Query routing - -| Query | Recent path | Authoritative path | -|---|---|---| -| Trace by ID | Fanout trace index | DuckDB over Parquet | -| Service/endpoint dashboard | DuckDB rollup cache | DuckDB rollup cache | -| Promoted attribute filter | DuckDB predicate pushdown | DuckDB predicate pushdown | -| Log text search | DuckDB scan | DuckDB scan | -| Arbitrary SQL | DuckDB over Parquet | DuckDB over Parquet | -| Export | Parquet writer | Existing Parquet files | - -## Single-binary implications - -| Choice | Distribution consequence | -|---|---| -| Fanout hot store | Native Go code inside the existing binary | -| DuckDB | Embedded native dependency already used by Fanout | -| Parquet | Library code; no separate server | -| SQLite | Embedded control database already used by Fanout | -| chDB | Adds and extracts a large ClickHouse native library | -| Iceberg | Adds Go metadata/catalog logic but still needs storage and execution | - -No external database daemon is required by the recommended design. - -## Production gates - -The implementation should remain gated on these production checks: - -- [ ] Add per-block and per-file checksums. -- [ ] Test torn writes and corruption at every commit boundary. -- [ ] Run continuous kill/restart recovery tests. -- [ ] Prove retention and compaction are safe under active readers. -- [ ] Bound memory during multi-day compaction. -- [ ] Benchmark on the target Linux 4-vCPU/8-GB host. -- [ ] Run a long concurrent ingest/query/retention soak. -- [ ] Benchmark realistic high-cardinality attributes and large exception data. - -## Final decision table - -| Component | Initial decision | Revisit when | -|---|---|---| -| Fanout hot columnar format | **Use** | If ownership cost exceeds its measured advantage | -| Fanout direct trace path | **Use** | Always retain benchmarks against DuckDB | -| Parquet authoritative format | **Use** | No expected replacement | -| DuckDB query engine | **Use** | If another embedded engine wins normalized SQL tests materially | -| SQLite control database | **Use** | No overlap with telemetry storage | -| DuckDB native telemetry tables | Do not use | Reconsider only as a deliberate architecture replacement | -| DuckLake | **Remove** | If DuckDB again becomes authoritative over mutable Parquet tables | -| Iceberg v3 | Not initially | Shared object storage, multiple writers, or multi-engine tables | -| chDB | **Remove** | Only if its binding, footprint, and normalized results improve materially | -| Custom general SQL engine | **Do not build** | No planned revisit | - -## Bottom line - -Fanout does not need every lakehouse layer. - -The smallest architecture that satisfies the product is: - -```text -Fanout hot trace index + authoritative Parquet + DuckDB SQL + SQLite control -``` - -DuckLake and Iceberg overlap with lifecycle management that Fanout already must -own for the hot store. Iceberg remains a clean future option for shared object -storage; it is not a prerequisite for a fast, featureful single-binary Fanout. diff --git a/docs/storage-benchmark.md b/docs/storage-benchmark.md deleted file mode 100644 index 70ac6224..00000000 --- a/docs/storage-benchmark.md +++ /dev/null @@ -1,107 +0,0 @@ -# Fanout storage benchmark - -This benchmark asks whether a storage path designed only for Fanout's -telemetry workload can outperform embedded general-purpose databases while -remaining crash-safe and retaining a path to ad-hoc SQL. - -It is isolated from the product and does not define a migration format. - -## Workload - -The shared generator emits the complete typed span produced by Fanout's OTLP -parser: 32 source fields, including all resource, attribute, event, link, -scope, HTTP, RPC, database, peer, deployment, status, and exception values. -DuckDB and chDB additionally persist three derived timestamp representations, -matching Fanout's current 35-column analytical shape. Each run uses one million -spans, 50,000-row durable commits, 50 services, 20 routes, 200 tenants, five -spans per trace, and 24 hours of event time. - -Queries return complete 35-column traces and a raw service aggregation. The -general-purpose engines also run an endpoint rollup. The mixed test writes -another 200,000 committed rows while full trace reads run at 100 queries per -second. - -## Fanout segment design - -- immutable segments containing 2,048-row columnar blocks; -- every column compressed independently with Zstandard; -- block min/max event-time metadata; -- a fixed-width on-disk trace index, binary-searched lazily with full-ID - verification; -- atomic manifest replacement with file and directory `fsync` ordering; -- orphan detection after a crash between segment and manifest publication; -- streaming compaction that copies compressed blocks without materializing - rows and merges indexes with bounded memory, then atomically replaces the - input segments; -- direct execution for indexed Fanout operations. - -## Normalized result - -Collected on Darwin/arm64, Apple M3 Max, 14 logical CPUs. This is a development -comparison, not a published Fanout capacity claim. - -| Storage / execution | Write rows/s | Maintenance | Active disk | Endpoint | Full trace | Raw service | Mixed write | Mixed trace p95 | -|---|---:|---:|---:|---:|---:|---:|---:|---:| -| **Production Fanout + Parquet** | 161,115 | live | 56.7 MiB | n/a | 0.846 ms | 28.63 ms | 162,544/s | 1.79 ms | -| Fanout columnar experiment | **548,714** | **142 ms** | 34.6 MiB | n/a | **0.663 ms** | 34.93 ms | **538,201/s** | **0.984 ms** | -| DuckDB native | 98,307 | 166 ms rollup + 4.59 ms checkpoint | 47.8 MiB | **0.880 ms** | 1.45 ms | **1.21 ms** | 72,420/s | 2.08 ms | -| Zstd Parquet + DuckDB | 95,305 effective | 320 ms export | **21.8 MiB** | 1.21 ms | 9.44 ms | 3.45 ms | n/a | n/a | -| chDB MergeTree | 125,388 | 4.05 s optimize | 38.1 MiB active | 2.91 ms | 7.10 ms | 6.10 ms | 121,775/s | 12.41 ms | - -The chDB directory occupied 106.7 MiB after forced merges because inactive and -engine-internal files remain present; the table's active parts occupied 38.1 -MiB. Its embedded-engine initialization took 418 ms in the measured run. - -Endpoint is `n/a` for the Fanout rows because the span segment deliberately -contains only the production trace-lookup primitive. Production endpoint -dashboards use the rebuildable DuckDB `endpoint_rollup` cache; the DuckDB row -measures that query shape. - -Iceberg is not listed as an execution engine. Its data plane is Parquet; table -metadata, snapshots, manifests, deletion vectors, and planning would sit above -the Parquet/DuckDB result and add capabilities plus some overhead. - -## Interpretation - -The custom span experiment establishes the upper bound behind the earlier -roughly 500k rows/s figure. It is not the production write rate: it omits the -authoritative Parquet projection for logs and metrics and is intentionally not -a general SQL store. Its trace index stays on disk and is searched lazily, so -retention does not create a resident trace-ID or rollup map. - -The production design keeps the useful parts without taking on a home-grown -database: - -- Fanout owns request-level WAL durability, a compact commit journal, the - recent-span index, retention, and compaction; -- Parquet is the single authoritative format for spans, logs, and metrics; -- DuckDB executes SQL, filtering, ordering, and broad analytical scans; -- SQLite stores transactional control and identity state only. - -This trades some maximum write throughput for much lower implementation risk, -full telemetry coverage, standard files, and a featureful SQL engine. The -benchmark reports direct durable publication throughput; request acknowledgement -is decoupled through the WAL and should be measured separately under the target -collector concurrency and hardware before publishing a capacity claim. - -## Reproduction - -Custom, DuckDB, and Parquet: - -```sh -go run ./bench/storage \ - -rows 1000000 \ - -batch 50000 \ - -repeats 11 \ - -mixed-rows 200000 -``` - -Run one embedded engine in isolation with `-engine custom` or `-engine duck`. - -chDB is a nested experiment module so its embedded C++ library does not enter -Fanout's production dependency graph or binary: - -```sh -cd bench/storage/chdb -go run . -rows 1000000 -batch 50000 -repeats 11 -mixed-rows 200000 -``` diff --git a/internal/storagebench/data.go b/internal/storagebench/data.go deleted file mode 100644 index b33def1e..00000000 --- a/internal/storagebench/data.go +++ /dev/null @@ -1,45 +0,0 @@ -package storagebench - -import ( - "fmt" - "time" - - "github.com/labstack/fanout/internal/telemetry/segment" -) - -const DayNanos = int64(24 * time.Hour) - -func Rows(offset, count, total int, base int64) []segment.Span { - rows := make([]segment.Span, count) - methods := [...]string{"GET", "POST", "PUT", "DELETE"} - for j := range rows { - i := offset + j - service := fmt.Sprintf("service-%02d", i%50) - route := fmt.Sprintf("/api/v1/resource/%02d", i%20) - statusCode, statusMessage := "OK", "" - exceptionType, exceptionMessage := "", "" - if i%20 == 0 { - statusCode, statusMessage = "ERROR", "upstream request failed" - exceptionType, exceptionMessage = "TimeoutError", "deadline exceeded while calling dependency" - } - start := base + int64(i)*DayNanos/int64(total) - duration := float64(1+(i%5000)) / 10 - rows[j] = segment.Span{ - Namespace: "default", TraceID: TraceID(uint64(i / 5)), SpanID: fmt.Sprintf("%016x", i), - ParentSpanID: fmt.Sprintf("%016x", max(i-1, 0)), ServiceName: service, - Name: methods[i%len(methods)] + " " + route, Kind: "SERVER", - StartUnixNanos: start, EndUnixNanos: start + int64(duration*float64(time.Millisecond)), DurationMS: duration, - StatusCode: statusCode, StatusMsg: statusMessage, - ResourceJSON: []byte(fmt.Sprintf(`{"service.name":"%s","host.name":"node-%02d"}`, service, i%16)), - AttributesJSON: []byte(fmt.Sprintf(`{"tenant":"tenant-%03d","region":"us-west-2","http.request.method":"%s"}`, i%200, methods[i%len(methods)])), - EventsJSON: []byte(`[]`), LinksJSON: []byte(`[]`), TraceState: "vendor=opaque", Flags: 1, - ScopeName: "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp", ScopeVersion: "0.63.0", IngestedAt: start + int64(time.Second), - HTTPMethod: methods[i%len(methods)], HTTPStatusCode: fmt.Sprintf("%d", 200+(i%5)), HTTPRoute: route, - PeerService: fmt.Sprintf("dependency-%02d", i%10), ServiceVersion: "2026.8.0", DeploymentEnv: "production", - ExceptionType: exceptionType, ExceptionMessage: exceptionMessage, - } - } - return rows -} - -func TraceID(value uint64) string { return fmt.Sprintf("%016x%016x", value*0x9e3779b97f4a7c15, value) } From 526fd87c79bcbaccd75ca8159cee9ee7d66b536a Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 09:26:28 -0700 Subject: [PATCH 14/31] refactor(storage)!: make Parquet authoritative Remove DuckLake, WAL, and segment compatibility paths. Publish durable Parquet batches atomically, use on-disk trace indexes, and rebuild DuckDB rollups from Parquet. BREAKING CHANGE: existing DuckLake and segment data is not migrated. --- .gitignore | 1 + README.md | 12 +- THIRD_PARTY_NOTICES | 2714 ++++++++--------- cmd/bench/main.go | 13 +- cmd/bench/metrics_report.go | 14 +- cmd/fanout/main.go | 2 +- docs/diagrams/architecture.d2 | 4 +- docs/diagrams/architecture.svg | 172 +- docs/diagrams/persistence.d2 | 8 +- docs/diagrams/persistence.svg | 194 +- docs/operations.md | 6 +- fanout.example.yaml | 1 - go.mod | 10 +- go.sum | 32 +- internal/config/config.go | 16 +- internal/config/config_test.go | 10 - internal/observability/service_test.go | 75 +- internal/observability/trace.go | 109 +- internal/query/duck.go | 15 +- internal/query/duck_test.go | 70 + internal/query/edge_backlog_test.go | 2 +- internal/query/schema.go | 6 +- internal/query/views.go | 10 +- internal/telemetry/parquet.go | 985 ++++-- internal/telemetry/parquet_rows.go | 153 + internal/telemetry/parquet_test.go | 273 +- internal/telemetry/rows.go | 17 + internal/telemetry/segment/span_columnar.go | 316 -- internal/telemetry/segment/span_store.go | 1158 ------- internal/telemetry/segment/span_store_test.go | 428 --- internal/telemetry/store/compaction.go | 357 +-- internal/telemetry/store/publication_test.go | 69 - internal/telemetry/store/repository.go | 770 +---- internal/telemetry/store/repository_test.go | 888 +----- internal/telemetry/store/writer.go | 295 +- internal/telemetry/store/writer_test.go | 365 +-- internal/telemetry/trace_index.go | 250 ++ internal/telemetry/trace_index_test.go | 57 + .../docs/explanation/storage-model.mdx | 51 +- .../docs/guides/back-up-and-restore.mdx | 8 +- .../content/docs/guides/tune-retention.mdx | 38 +- .../content/docs/reference/data-layout.mdx | 19 +- .../docs/reference/settings/storage.mdx | 7 +- site/src/content/docs/start/first-boot.mdx | 1 - 44 files changed, 3672 insertions(+), 6329 deletions(-) create mode 100644 internal/telemetry/parquet_rows.go delete mode 100644 internal/telemetry/segment/span_columnar.go delete mode 100644 internal/telemetry/segment/span_store.go delete mode 100644 internal/telemetry/segment/span_store_test.go delete mode 100644 internal/telemetry/store/publication_test.go create mode 100644 internal/telemetry/trace_index.go create mode 100644 internal/telemetry/trace_index_test.go diff --git a/.gitignore b/.gitignore index 17d80c14..43165f98 100644 --- a/.gitignore +++ b/.gitignore @@ -53,5 +53,6 @@ cover.out .playwright-mcp/ # Ad-hoc output from `go build` without -o (named after the package). +/bench /fanout /fanout-docgen diff --git a/README.md b/README.md index f957d506..529e970c 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ executable, including the React client. ![Fanout architecture](docs/diagrams/architecture.svg) -Telemetry lands over OTLP/gRPC or OTLP/HTTP, is durably committed to indexed -hot segments and open Parquet files, -and is read back through a DuckDB query kernel that also maintains service, +Telemetry lands over OTLP/gRPC or OTLP/HTTP, is durably committed as atomic +Parquet batches with persistent trace indexes, and is read back through a +DuckDB query kernel that also maintains service, endpoint, and edge rollups. The browser client, an in-process agent, and any external MCP host all reach the same typed observability contract rather than issuing raw SQL. -Every write to the telemetry catalog — ingest flush, rollups, and background -maintenance alike — passes through a single write gate that holds one catalog -write in flight at a time: +Independent ingest batches encode in parallel. Rollup-cache writes are +serialized inside DuckDB, while retention and compaction atomically swap +immutable Parquet directories behind active readers: ![Fanout persistence](docs/diagrams/persistence.svg) diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 9a0532b8..be80c728 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -19,8 +19,6 @@ COMPONENT INVENTORY - Go: github.com/alexedwards/scs/v2 v2.9.0 - Go: github.com/andybalholm/brotli v1.2.2 - Go: github.com/antlr4-go/antlr/v4 v4.13.1 -- Go: github.com/apache/arrow-go/v18 v18.7.0 -- Go: github.com/apache/thrift v0.24.0 - Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 - Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 - Go: github.com/beorn7/perks v1.0.1 @@ -38,8 +36,6 @@ COMPONENT INVENTORY - Go: github.com/go-jose/go-jose/v4 v4.1.4 - Go: github.com/go-openapi/inflect v1.0.0 - Go: github.com/go-viper/mapstructure/v2 v2.5.0 -- Go: github.com/goccy/go-json v0.10.6 -- Go: github.com/google/flatbuffers v25.12.19+incompatible - Go: github.com/google/go-cmp v0.7.0 - Go: github.com/google/jsonschema-go v0.4.3 - Go: github.com/google/uuid v1.6.0 @@ -60,6 +56,9 @@ COMPONENT INVENTORY - Go: github.com/modelcontextprotocol/go-sdk v1.7.0 - Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - Go: github.com/ncruces/go-strftime v1.0.0 +- Go: github.com/parquet-go/bitpack v1.0.0 +- Go: github.com/parquet-go/jsonlite v1.0.0 +- Go: github.com/parquet-go/parquet-go v0.32.0 - Go: github.com/pierrec/lz4/v4 v4.1.29 - Go: github.com/prometheus/client_golang v1.24.1 - Go: github.com/prometheus/client_model v0.6.2 @@ -69,6 +68,7 @@ COMPONENT INVENTORY - Go: github.com/segmentio/asm v1.2.1 - Go: github.com/segmentio/encoding v0.5.4 - Go: github.com/sirupsen/logrus v1.10.2 +- Go: github.com/twpayne/go-geom v1.6.1 - Go: github.com/wneessen/go-mail v0.8.1 - Go: github.com/yosida95/uritemplate/v3 v3.0.2 - Go: github.com/zclconf/go-cty v1.19.0 @@ -361,7 +361,6 @@ LICENSE AND NOTICE TEXTS - Go: github.com/agext/levenshtein v1.2.3 / LICENSE - Go: github.com/go-jose/go-jose/v4 v4.1.4 / LICENSE - Go: github.com/go-openapi/inflect v1.0.0 / LICENSE -- Go: github.com/google/flatbuffers v25.12.19+incompatible / LICENSE - Go: github.com/prometheus/client_golang v1.24.1 / LICENSE - Go: github.com/prometheus/client_model v0.6.2 / LICENSE - Go: github.com/prometheus/common v0.70.1 / LICENSE @@ -937,7 +936,210 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- Applies to ------------------------------------------------------------- -- Go: github.com/apache/arrow-go/v18 v18.7.0 / LICENSE.txt +- Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2017 Martin Atkins + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--------- + +Unicode table generation programs are under a separate copyright and license: + +Copyright (c) 2014 Couchbase, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied. See the License for the specific language governing permissions +and limitations under the License. + +--------- + +Grapheme break data is provided as part of the Unicode character database, +copright 2016 Unicode, Inc, which is provided with the following license: + +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +Unicode Data Files do not include PDF online code charts under the +directory http://www.unicode.org/Public/. + +Software includes any source code published in the Unicode Standard +or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +NOTICE TO USER: Carefully read the following legal agreement. +BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S +DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), +YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. +IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE +THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2017 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2017 Martin Atkins + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/beorn7/perks v1.0.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (C) 2013 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/bmatcuk/doublestar v1.3.4 / LICENSE +---------------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Bob Matcuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/cespare/xxhash/v2 v2.3.0 / LICENSE.txt +---------------------------------------------------------------------------- + +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/coreos/go-oidc/v3 v3.20.0 / LICENSE +- Go: github.com/zclconf/go-cty-yaml v1.2.0 / LICENSE ---------------------------------------------------------------------------- Apache License @@ -1120,7 +1322,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -1128,7 +1330,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1142,405 +1344,202 @@ Apache License See the License for the specific language governing permissions and limitations under the License. --------------------------------------------------------------------------------- - -This project includes code from the Go project, BSD 3-clause license + PATENTS -weak patent termination clause -(https://github.com/golang/go/blob/master/PATENTS): +--- Applies to ------------------------------------------------------------- +- Go: github.com/coreos/go-oidc/v3 v3.20.0 / NOTICE +---------------------------------------------------------------------------- - * arrow/flight/cookie_middleware.go +CoreOS Project +Copyright 2014 CoreOS, Inc -Copyright (c) 2009 The Go Authors. All rights reserved. +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +--- Applies to ------------------------------------------------------------- +- Go: github.com/duckdb/duckdb-go-bindings v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 / LICENSE +- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 / LICENSE +---------------------------------------------------------------------------- - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +Copyright 2018-2026 Stichting DuckDB Foundation -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: --------------------------------------------------------------------------------- +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -This project includes code from the LLVM project: +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -* arrow/compute/internal/kernels/_lib/types.h +--- Applies to ------------------------------------------------------------- +- Go: github.com/duckdb/duckdb-go/v2 v2.10505.0 / LICENSE +---------------------------------------------------------------------------- -Apache License v2.0 with LLVM Exceptions. -See https://llvm.org/LICENSE.txt for license information. -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - --------------------------------------------------------------------------------- +Copyright 2019-2024 Marc Boeker +Copyright 2025-2026 Stichting DuckDB Foundation -This project includes code from the brotli project (https://github.com/google/brotli): +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -* parquet/compress/brotli.go +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -Copyright: 2013 Google Inc. All Rights Reserved -Distributed under MIT License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --- Applies to ------------------------------------------------------------- -- Go: github.com/apache/arrow-go/v18 v18.7.0 / NOTICE.txt +- Go: github.com/dustin/go-humanize v1.0.1 / LICENSE ---------------------------------------------------------------------------- -Apache Arrow Go -Copyright 2016-2025 The Apache Software Foundation +Copyright (c) 2005-2008 Dustin Sallings -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + --- Applies to ------------------------------------------------------------- -- Go: github.com/apache/thrift v0.24.0 / LICENSE +- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE ---------------------------------------------------------------------------- -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. - 1. Definitions. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +--- Applies to ------------------------------------------------------------- +- Go: github.com/go-openapi/inflect v1.0.0 / NOTICE +---------------------------------------------------------------------------- - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +Copyright 2015-2025 go-swagger maintainers - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +This software library, github.com/go-openapi/jsonpointer, includes software developed +by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this software except in compliance with the License. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +You may obtain a copy of the License at - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." + http://www.apache.org/licenses/LICENSE-2.0. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +This software is copied from, derived from, and inspired by other original software products. +It ships with copies of other software which license terms are recalled below. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +The original software was authored by Chris Farmiloe at https://bitbucket.org/pkg/inflect under a MIT License. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +ghttps://bitbucket.org/pkg/inflect +=========================== - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Copyright (c) 2011 Chris Farmiloe - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +--- Applies to ------------------------------------------------------------- +- Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE +- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE +---------------------------------------------------------------------------- - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +The MIT License (MIT) - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +Copyright (c) 2013 Mitchell Hashimoto - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +--- Applies to ------------------------------------------------------------- +- Go: github.com/google/go-cmp v0.7.0 / LICENSE +---------------------------------------------------------------------------- - END OF TERMS AND CONDITIONS +Copyright (c) 2017 The Go Authors. All rights reserved. - APPENDIX: How to apply the Apache License to your work. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------- -SOFTWARE DISTRIBUTED WITH THRIFT: - -The Apache Thrift software includes a number of subcomponents with -separate copyright notices and license terms. Your use of the source -code for the these subcomponents is subject to the terms and -conditions of the following licenses. - --------------------------------------------------- -Portions of the following files are licensed under the MIT License: - - lib/erl/src/Makefile.am - -Please see doc/otp-base-license.txt for the full terms of this license. - --------------------------------------------------- -For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: - -# Copyright (c) 2007 Thomas Porschberg -# -# Copying and distribution of this file, with or without -# modification, are permitted in any medium without royalty provided -# the copyright notice and this notice are preserved. - --------------------------------------------------- -For the lib/nodejs/lib/thrift/json_parse.js: - -/* - json_parse.js - 2015-05-02 - Public Domain. - NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. - -*/ -(By Douglas Crockford ) - --------------------------------------------------- -For lib/cpp/src/thrift/windows/SocketPair.cpp - -/* socketpair.c - * Copyright 2007 by Nathan C. Myers ; some rights reserved. - * This code is Free Software. It may be copied freely, in original or - * modified form, subject only to the restrictions that (1) the author is - * relieved from all responsibilities for any use for any purpose, and (2) - * this copyright notice must be retained, unchanged, in its entirety. If - * for any reason the author might be held responsible for any consequences - * of copying or use, license is withheld. - */ - - --------------------------------------------------- -For lib/py/compat/win32/stdint.h - -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2008 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. The name of the author may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - - --------------------------------------------------- -Codegen template in t_html_generator.h - -* Bootstrap v2.0.3 -* -* Copyright 2012 Twitter, Inc -* Licensed under the Apache License v2.0 -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Designed and built with all the love in the world @twitter by @mdo and @fat. - ---------------------------------------------------- -For t_cl_generator.cc - - * Copyright (c) 2008- Patrick Collison - * Copyright (c) 2006- Facebook - ---------------------------------------------------- - ---------------------------------------------------- -For compiler/cpp/src/thrift/generate/sha256.h - -SHA-256 implementation by Brad Conte (brad AT bradconte.com). -Source: https://github.com/B-Con/crypto-algorithms -The author has placed this code in the public domain (no copyright claimed). -No algorithmic changes were made; the file was adapted to a C++ header-only -form for inclusion in the Thrift compiler. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/apache/thrift v0.24.0 / NOTICE ----------------------------------------------------------------------------- - -Apache Thrift -Copyright (C) 2006 - 2019, The Apache Software Foundation + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- Applies to ------------------------------------------------------------- -- Go: github.com/apparentlymart/go-textseg/v15 v15.0.0 / LICENSE +- Go: github.com/google/jsonschema-go v0.4.3 / LICENSE ---------------------------------------------------------------------------- -Copyright (c) 2017 Martin Atkins +MIT License + +Copyright (c) 2025 JSON Schema Go Project Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -1560,637 +1559,436 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---------- +--- Applies to ------------------------------------------------------------- +- Go: github.com/google/uuid v1.6.0 / LICENSE +---------------------------------------------------------------------------- -Unicode table generation programs are under a separate copyright and license: +Copyright (c) 2009,2014 Google Inc. All rights reserved. -Copyright (c) 2014 Couchbase, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain a copy of the License at +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - http://www.apache.org/licenses/LICENSE-2.0 + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -Unless required by applicable law or agreed to in writing, software distributed under the -License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -either express or implied. See the License for the specific language governing permissions -and limitations under the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------- +--- Applies to ------------------------------------------------------------- +- Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 / LICENSE +---------------------------------------------------------------------------- -Grapheme break data is provided as part of the Unicode character database, -copright 2016 Unicode, Inc, which is provided with the following license: +Copyright (c) 2015, Gengo, Inc. +All rights reserved. -Unicode Data Files include all data files under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, -http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and -http://www.unicode.org/utility/trac/browser/. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: -Unicode Data Files do not include PDF online code charts under the -directory http://www.unicode.org/Public/. + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. -Software includes any source code published in the Unicode Standard -or under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, -http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and -http://www.unicode.org/utility/trac/browser/. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. -NOTICE TO USER: Carefully read the following legal agreement. -BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S -DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), -YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE -TERMS AND CONDITIONS OF THIS AGREEMENT. -IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE -THE DATA FILES OR SOFTWARE. + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. -COPYRIGHT AND PERMISSION NOTICE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Copyright © 1991-2017 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in http://www.unicode.org/copyright.html. +--- Applies to ------------------------------------------------------------- +- Go: github.com/hashicorp/hcl/v2 v2.24.0 / LICENSE +---------------------------------------------------------------------------- -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. +Copyright (c) 2014 HashiCorp, Inc. -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. +Mozilla Public License, version 2.0 -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. +1. Definitions ---- Applies to ------------------------------------------------------------- -- Go: github.com/apparentlymart/go-textseg/v17 v17.0.1 / LICENSE ----------------------------------------------------------------------------- +1.1. “Contributor” -Copyright (c) 2017 Martin Atkins + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +1.2. “Contributor Version” -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +1.3. “Contribution” ---- Applies to ------------------------------------------------------------- -- Go: github.com/beorn7/perks v1.0.1 / LICENSE ----------------------------------------------------------------------------- + means Covered Software of a particular Contributor. -Copyright (C) 2013 Blake Mizerany +1.4. “Covered Software” -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. +1.5. “Incompatible With Secondary Licenses” + means -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or ---- Applies to ------------------------------------------------------------- -- Go: github.com/bmatcuk/doublestar v1.3.4 / LICENSE ----------------------------------------------------------------------------- + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. -The MIT License (MIT) +1.6. “Executable Form” -Copyright (c) 2014 Bob Matcuk + means any form of the work other than Source Code Form. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +1.7. “Larger Work” -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +1.8. “License” ---- Applies to ------------------------------------------------------------- -- Go: github.com/cespare/xxhash/v2 v2.3.0 / LICENSE.txt ----------------------------------------------------------------------------- + means this document. -Copyright (c) 2016 Caleb Spare +1.9. “Licensable” -MIT License - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/coreos/go-oidc/v3 v3.20.0 / LICENSE -- Go: github.com/zclconf/go-cty-yaml v1.2.0 / LICENSE ----------------------------------------------------------------------------- - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +1.10. “Modifications” - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. + means any of the following: - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. + b. any new file in Source Code Form that contains any Covered Software. - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +1.11. “Patent Claims” of a Contributor - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +1.12. “Secondary License” - END OF TERMS AND CONDITIONS + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. - APPENDIX: How to apply the Apache License to your work. +1.13. “Source Code Form” - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. + means the form of the work preferred for making modifications. - Copyright {yyyy} {name of copyright owner} +1.14. “You” (or “Your”) - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +2. License Grants and Conditions ---- Applies to ------------------------------------------------------------- -- Go: github.com/coreos/go-oidc/v3 v3.20.0 / NOTICE ----------------------------------------------------------------------------- +2.1. Grants -CoreOS Project -Copyright 2014 CoreOS, Inc + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: -This product includes software developed at CoreOS, Inc. -(http://www.coreos.com/). + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and ---- Applies to ------------------------------------------------------------- -- Go: github.com/duckdb/duckdb-go-bindings v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 / LICENSE -- Go: github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 / LICENSE ----------------------------------------------------------------------------- + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. -Copyright 2018-2026 Stichting DuckDB Foundation +2.2. Effective Date -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +2.3. Limitations on Grant Scope -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: ---- Applies to ------------------------------------------------------------- -- Go: github.com/duckdb/duckdb-go/v2 v2.10505.0 / LICENSE ----------------------------------------------------------------------------- + a. for any code that a Contributor has removed from Covered Software; or -Copyright 2019-2024 Marc Boeker -Copyright 2025-2026 Stichting DuckDB Foundation + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +2.4. Subsequent Licenses ---- Applies to ------------------------------------------------------------- -- Go: github.com/dustin/go-humanize v1.0.1 / LICENSE ----------------------------------------------------------------------------- + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). -Copyright (c) 2005-2008 Dustin Sallings +2.5. Representation -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +2.6. Fair Use -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. - +2.7. Conditions ---- Applies to ------------------------------------------------------------- -- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE ----------------------------------------------------------------------------- + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. -Copyright © 2012 The Go Authors. All rights reserved. -Copyright © fsnotify Authors. All rights reserved. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +3. Responsibilities -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Google Inc. nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. +3.1. Distribution of Source Form -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. ---- Applies to ------------------------------------------------------------- -- Go: github.com/go-openapi/inflect v1.0.0 / NOTICE ----------------------------------------------------------------------------- +3.2. Distribution of Executable Form -Copyright 2015-2025 go-swagger maintainers + If You distribute Covered Software in Executable Form then: -// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers -// SPDX-License-Identifier: Apache-2.0 + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and -This software library, github.com/go-openapi/jsonpointer, includes software developed -by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this software except in compliance with the License. +3.3. Distribution of a Larger Work -You may obtain a copy of the License at + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). - http://www.apache.org/licenses/LICENSE-2.0. +3.4. Notices -This software is copied from, derived from, and inspired by other original software products. -It ships with copies of other software which license terms are recalled below. + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. -The original software was authored by Chris Farmiloe at https://bitbucket.org/pkg/inflect under a MIT License. +3.5. Application of Additional Terms -ghttps://bitbucket.org/pkg/inflect -=========================== + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. -Copyright (c) 2011 Chris Farmiloe +4. Inability to Comply Due to Statute or Regulation -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +5. Termination -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. ---- Applies to ------------------------------------------------------------- -- Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE -- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE ----------------------------------------------------------------------------- +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. -The MIT License (MIT) +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. -Copyright (c) 2013 Mitchell Hashimoto +6. Disclaimer of Warranty -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +7. Limitation of Liability -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. ---- Applies to ------------------------------------------------------------- -- Go: github.com/goccy/go-json v0.10.6 / LICENSE ----------------------------------------------------------------------------- +8. Litigation -MIT License + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. -Copyright (c) 2020 Masaaki Goshima +9. Miscellaneous -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +10. Versions of the License ---- Applies to ------------------------------------------------------------- -- Go: github.com/google/go-cmp v0.7.0 / LICENSE ----------------------------------------------------------------------------- +10.1. New Versions -Copyright (c) 2017 The Go Authors. All rights reserved. + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +10.2. Effect of New Versions - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +10.3. Modified Versions ---- Applies to ------------------------------------------------------------- -- Go: github.com/google/jsonschema-go v0.4.3 / LICENSE ----------------------------------------------------------------------------- + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). -MIT License +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. -Copyright (c) 2025 JSON Schema Go Project Authors +Exhibit A - Source Code Form License Notice -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. --- Applies to ------------------------------------------------------------- -- Go: github.com/google/uuid v1.6.0 / LICENSE +- Go: github.com/klauspost/compress v1.19.2 / LICENSE ---------------------------------------------------------------------------- -Copyright (c) 2009,2014 Google Inc. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2218,434 +2016,416 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2015, Gengo, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name of Gengo, Inc. nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/hashicorp/hcl/v2 v2.24.0 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2014 HashiCorp, Inc. - -Mozilla Public License, version 2.0 - -1. Definitions - -1.1. “Contributor” - - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. - -1.2. “Contributor Version” - - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” - - means Covered Software of a particular Contributor. - -1.4. “Covered Software” - - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. - -1.5. “Incompatible With Secondary Licenses” - means - - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or - - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. - -1.6. “Executable Form” - - means any form of the work other than Source Code Form. - -1.7. “Larger Work” - - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. - -1.8. “License” - - means this document. - -1.9. “Licensable” - - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. - -1.10. “Modifications” +------------------ - means any of the following: +Files: gzhttp/* - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - b. any new file in Source Code Form that contains any Covered Software. + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1.11. “Patent Claims” of a Contributor + 1. Definitions. - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -1.12. “Secondary License” + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -1.13. “Source Code Form” + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. - means the form of the work preferred for making modifications. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -1.14. “You” (or “Your”) + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. -2. License Grants and Conditions + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." -2.1. Grants + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: -2.2. Effective Date + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and -2.3. Limitations on Grant Scope + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. - a. for any code that a Contributor has removed from Covered Software; or + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. -2.4. Subsequent Licenses + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. -2.5. Representation + END OF TERMS AND CONDITIONS - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. + APPENDIX: How to apply the Apache License to your work. -2.6. Fair Use + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. + Copyright 2016-2017 The New York Times Company -2.7. Conditions + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -3. Responsibilities +------------------ -3.1. Distribution of Source Form +Files: s2/cmd/internal/readahead/* - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. +The MIT License (MIT) -3.2. Distribution of Executable Form +Copyright (c) 2015 Klaus Post - If You distribute Covered Software in Executable Form then: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -3.3. Distribution of a Larger Work +--------------------- +Files: snappy/* +Files: internal/snapref/* - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. -3.4. Notices +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -3.5. Application of Additional Terms +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. +----------------- -4. Inability to Comply Due to Statute or Regulation +Files: s2/cmd/internal/filepathx/* - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. +Copyright 2016 The filepathx Authors -5. Termination +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. +--- Applies to ------------------------------------------------------------- +- Go: github.com/klauspost/cpuid/v2 v2.4.0 / LICENSE +---------------------------------------------------------------------------- -6. Disclaimer of Warranty +The MIT License (MIT) - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. +Copyright (c) 2015 Klaus Post -7. Limitation of Liability +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -8. Litigation +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. +--- Applies to ------------------------------------------------------------- +- Go: github.com/knadh/koanf/maps v0.1.3 / LICENSE +- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 / LICENSE +- Go: github.com/knadh/koanf/providers/confmap v1.0.1 / LICENSE +- Go: github.com/knadh/koanf/providers/file v1.2.1 / LICENSE +- Go: github.com/knadh/koanf/v2 v2.3.6 / LICENSE +---------------------------------------------------------------------------- -9. Miscellaneous +The MIT License - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. +Copyright (c) 2019, Kailash Nadh. https://github.com/knadh +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -10. Versions of the License +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -10.1. New Versions +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. +--- Applies to ------------------------------------------------------------- +- Go: github.com/labstack/echo/v5 v5.3.1 / LICENSE +---------------------------------------------------------------------------- -10.2. Effect of New Versions +The MIT License (MIT) - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. +Copyright (c) 2022 LabStack -10.3. Modified Versions +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -Exhibit A - Source Code Form License Notice +--- Applies to ------------------------------------------------------------- +- Go: github.com/mattn/go-isatty v0.0.24 / LICENSE +---------------------------------------------------------------------------- - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. +Copyright (c) Yasuhiro MATSUMOTO -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +MIT License (Expat) -You may add additional accurate notices of copyright ownership. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -Exhibit B - “Incompatible With Secondary Licenses” Notice +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --- Applies to ------------------------------------------------------------- -- Go: github.com/klauspost/compress v1.19.2 / LICENSE +- Go: github.com/mitchellh/copystructure v1.2.0 / LICENSE +- Go: github.com/mitchellh/go-wordwrap v1.0.1 / LICENSE.md ---------------------------------------------------------------------------- -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2019 Klaus Post. All rights reserved. +The MIT License (MIT) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Copyright (c) 2014 Mitchell Hashimoto - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--- Applies to ------------------------------------------------------------- +- Go: github.com/modelcontextprotocol/go-sdk v1.7.0 / LICENSE +---------------------------------------------------------------------------- ------------------- +The MCP project is undergoing a licensing transition from the MIT License to the Apache License, Version 2.0 ("Apache-2.0"). All new code and specification contributions to the project are licensed under Apache-2.0. Documentation contributions (excluding specifications) are licensed under CC-BY-4.0. -Files: gzhttp/* +Contributions for which relicensing consent has been obtained are licensed under Apache-2.0. Contributions made by authors who originally licensed their work under the MIT License and who have not yet granted explicit permission to relicense remain licensed under the MIT License. + +No rights beyond those granted by the applicable original license are conveyed for such contributions. + +--- Apache License Version 2.0, January 2004 @@ -2697,9 +2477,9 @@ Files: gzhttp/* "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" + submitted to Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on behalf + of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, @@ -2824,38 +2604,11 @@ Files: gzhttp/* END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016-2017 The New York Times Company - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------- - -Files: s2/cmd/internal/readahead/* +--- -The MIT License (MIT) +MIT License -Copyright (c) 2015 Klaus Post +Copyright (c) 2024-2025 Model Context Protocol a Series of LF Projects, LLC. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -2875,31 +2628,43 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---------------------- -Files: snappy/* -Files: internal/snapref/* +--- -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. +Creative Commons Attribution 4.0 International (CC-BY-4.0) + +Documentation in this project (excluding specifications) is licensed under +CC-BY-4.0. See https://creativecommons.org/licenses/by/4.0/legalcode for +the full license text. + +--- Applies to ------------------------------------------------------------- +- Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY @@ -2907,25 +2672,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------ - -Files: s2/cmd/internal/filepathx/* - -Copyright 2016 The filepathx Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - --- Applies to ------------------------------------------------------------- -- Go: github.com/klauspost/cpuid/v2 v2.4.0 / LICENSE +- Go: github.com/ncruces/go-strftime v1.0.0 / LICENSE ---------------------------------------------------------------------------- -The MIT License (MIT) +MIT License -Copyright (c) 2015 Klaus Post +Copyright (c) 2022 Nuno Cruces Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -2946,83 +2699,218 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --- Applies to ------------------------------------------------------------- -- Go: github.com/knadh/koanf/maps v0.1.3 / LICENSE -- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 / LICENSE -- Go: github.com/knadh/koanf/providers/confmap v1.0.1 / LICENSE -- Go: github.com/knadh/koanf/providers/file v1.2.1 / LICENSE -- Go: github.com/knadh/koanf/v2 v2.3.6 / LICENSE +- Go: github.com/parquet-go/bitpack v1.0.0 / LICENSE ---------------------------------------------------------------------------- -The MIT License +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Copyright (c) 2019, Kailash Nadh. https://github.com/knadh + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 1. Definitions. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. ---- Applies to ------------------------------------------------------------- -- Go: github.com/labstack/echo/v5 v5.3.1 / LICENSE ----------------------------------------------------------------------------- + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. -The MIT License (MIT) + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. -Copyright (c) 2022 LabStack + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. ---- Applies to ------------------------------------------------------------- -- Go: github.com/mattn/go-isatty v0.0.24 / LICENSE ----------------------------------------------------------------------------- + END OF TERMS AND CONDITIONS -Copyright (c) Yasuhiro MATSUMOTO + APPENDIX: How to apply the Apache License to your work. -MIT License (Expat) + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + Copyright 2025 Achille Roussel, Filip Petkovski -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. --- Applies to ------------------------------------------------------------- -- Go: github.com/mitchellh/copystructure v1.2.0 / LICENSE -- Go: github.com/mitchellh/go-wordwrap v1.0.1 / LICENSE.md +- Go: github.com/parquet-go/jsonlite v1.0.0 / LICENSE ---------------------------------------------------------------------------- -The MIT License (MIT) +MIT License -Copyright (c) 2014 Mitchell Hashimoto +Copyright (c) 2025 parquet-go Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -3031,30 +2919,22 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. --- Applies to ------------------------------------------------------------- -- Go: github.com/modelcontextprotocol/go-sdk v1.7.0 / LICENSE +- Go: github.com/parquet-go/parquet-go v0.32.0 / LICENSE ---------------------------------------------------------------------------- -The MCP project is undergoing a licensing transition from the MIT License to the Apache License, Version 2.0 ("Apache-2.0"). All new code and specification contributions to the project are licensed under Apache-2.0. Documentation contributions (excluding specifications) are licensed under CC-BY-4.0. - -Contributions for which relicensing consent has been obtained are licensed under Apache-2.0. Contributions made by authors who originally licensed their work under the MIT License and who have not yet granted explicit permission to relicense remain licensed under the MIT License. - -No rights beyond those granted by the applicable original license are conveyed for such contributions. - ---- - - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -3104,9 +2984,9 @@ No rights beyond those granted by the applicable original license are conveyed f "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright - owner or by an individual or Legal Entity authorized to submit on behalf - of the copyright owner. For the purposes of this definition, "submitted" + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, @@ -3231,99 +3111,41 @@ No rights beyond those granted by the applicable original license are conveyed f END OF TERMS AND CONDITIONS ---- - -MIT License - -Copyright (c) 2024-2025 Model Context Protocol a Series of LF Projects, LLC. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ---- - -Creative Commons Attribution 4.0 International (CC-BY-4.0) - -Documentation in this project (excluding specifications) is licensed under -CC-BY-4.0. See https://creativecommons.org/licenses/by/4.0/legalcode for -the full license text. - ---- Applies to ------------------------------------------------------------- -- Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 / LICENSE ----------------------------------------------------------------------------- - -Copyright (c) 2011, Open Knowledge Foundation Ltd. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. + APPENDIX: How to apply the Apache License to your work. - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - Neither the name of the Open Knowledge Foundation Ltd. nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. + Copyright 2023 Twilio, Inc. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at ---- Applies to ------------------------------------------------------------- -- Go: github.com/ncruces/go-strftime v1.0.0 / LICENSE ----------------------------------------------------------------------------- + http://www.apache.org/licenses/LICENSE-2.0 -MIT License + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -Copyright (c) 2022 Nuno Cruces +-------------------------------------------------------------------------------- -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +This product includes code from Apache Parquet. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +* deprecated/parquet.go is based on Apache Parquet's thrift file +* format/parquet.go is based on Apache Parquet's thrift file -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Copyright: 2014 The Apache Software Foundation. +Home page: https://github.com/apache/parquet-format +License: http://www.apache.org/licenses/LICENSE-2.0 --- Applies to ------------------------------------------------------------- - Go: github.com/pierrec/lz4/v4 v4.1.29 / LICENSE @@ -3517,6 +3339,34 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- Applies to ------------------------------------------------------------- +- Go: github.com/twpayne/go-geom v1.6.1 / LICENSE +---------------------------------------------------------------------------- + +Copyright (c) 2013, Tom Payne +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + --- Applies to ------------------------------------------------------------- - Go: github.com/wneessen/go-mail v0.8.1 / LICENSE ---------------------------------------------------------------------------- diff --git a/cmd/bench/main.go b/cmd/bench/main.go index 1869f408..fbf9e538 100644 --- a/cmd/bench/main.go +++ b/cmd/bench/main.go @@ -81,8 +81,7 @@ type config struct { maxQueryP95 float64 // backfillHours, when >0, spreads each event's timestamp uniformly over the // last N hours (instead of "now"). Used to PRE-SEED a multi-hour dataset so - // Parquet spans several hour partitions — required to exercise within-day - // (hour-partition) pruning, which a same-hour run can't. + // time-window queries and rollups cover more than the current hour. backfillHours float64 // seed makes the synthetic workload reproducible: same seed, same services, // endpoints, attributes, and error placement. Two runs are only comparable if @@ -557,7 +556,7 @@ func (g *generator) outCtx(ctx context.Context) context.Context { // eventTime returns the timestamp for an emitted event: now(), or — when // backfillHours>0 — a time spread uniformly over the last N hours so a pre-seed -// run populates multiple hour partitions (to exercise within-day pruning). +// run exercises multi-hour time-window queries and rollups. func (g *generator) eventTime(rng *rand.Rand) time.Time { if g.cfg.backfillHours <= 0 { return time.Now() @@ -919,9 +918,9 @@ type serverReport struct { RowsDroppedStart float64 `json:"rows_dropped_start"` RowsDroppedEnd float64 `json:"rows_dropped_end"` RowsDroppedDelta float64 `json:"rows_dropped_delta"` - ParquetFilesStart float64 `json:"parquet_partitions_start"` - ParquetFiles float64 `json:"parquet_partitions"` - ParquetFilesDelta float64 `json:"parquet_partitions_delta"` + ParquetFilesStart float64 `json:"parquet_files_start"` + ParquetFiles float64 `json:"parquet_files"` + ParquetFilesDelta float64 `json:"parquet_files_delta"` ParquetSizeBytesStart float64 `json:"parquet_size_bytes_start"` ParquetSizeBytes float64 `json:"parquet_size_bytes"` ParquetSizeBytesDelta float64 `json:"parquet_size_bytes_delta"` @@ -982,7 +981,7 @@ func printReport(r report) { s := r.Server fmt.Printf("server (Δ over run):\n") fmt.Printf(" rows accepted=%.0f dropped=%.0f\n", s.IngestRowsDelta, s.RowsDroppedDelta) - fmt.Printf(" parquet_partitions=%.0f parquet_size=%.1fMB ingest_queue_depth=%.0f\n", + fmt.Printf(" parquet_files=%.0f parquet_size=%.1fMB ingest_queue_depth=%.0f\n", s.ParquetFiles, s.ParquetSizeBytes/(1<<20), s.IngestQueueDepth) fmt.Printf(" avg rollup=%.1fms flush=%.1fms query=%.1fms\n", s.AvgRollupMs, s.AvgFlushMs, s.AvgQueryMs) fmt.Printf(" cpu=%.2f core(s) rss=%.1fMB alloc=%.1fMB/s parquet_growth=%.1fMB\n", diff --git a/cmd/bench/metrics_report.go b/cmd/bench/metrics_report.go index de3aa05d..936ecae1 100644 --- a/cmd/bench/metrics_report.go +++ b/cmd/bench/metrics_report.go @@ -292,8 +292,8 @@ func serverDelta(base, final *metricSnapshot, durationSeconds float64) *serverRe } return round2(value / durationSeconds) } - lakePartitionsStart := base.total("fanout_parquet_files") - lakeSizeStart := base.total("fanout_parquet_size_bytes") + parquetFilesStart := base.total("fanout_parquet_files") + parquetSizeStart := base.total("fanout_parquet_size_bytes") cpuSeconds := delta("process_cpu_seconds_total") allocBytes := delta("go_memstats_alloc_bytes_total") // process_start_time_seconds is constant for the life of a process, so a @@ -311,13 +311,13 @@ func serverDelta(base, final *metricSnapshot, durationSeconds float64) *serverRe RowsDroppedStart: base.total("fanout_rows_dropped_total"), RowsDroppedEnd: final.total("fanout_rows_dropped_total"), RowsDroppedDelta: delta("fanout_rows_dropped_total"), - ParquetFilesStart: lakePartitionsStart, + ParquetFilesStart: parquetFilesStart, ParquetFiles: final.total("fanout_parquet_files"), - ParquetFilesDelta: final.total("fanout_parquet_files") - lakePartitionsStart, - ParquetSizeBytesStart: lakeSizeStart, + ParquetFilesDelta: final.total("fanout_parquet_files") - parquetFilesStart, + ParquetSizeBytesStart: parquetSizeStart, ParquetSizeBytes: final.total("fanout_parquet_size_bytes"), - ParquetSizeBytesDelta: final.total("fanout_parquet_size_bytes") - lakeSizeStart, - ParquetGrowthBytesPerSec: rate(final.total("fanout_parquet_size_bytes") - lakeSizeStart), + ParquetSizeBytesDelta: final.total("fanout_parquet_size_bytes") - parquetSizeStart, + ParquetGrowthBytesPerSec: rate(final.total("fanout_parquet_size_bytes") - parquetSizeStart), IngestQueueDepth: final.total("fanout_ingest_queue_depth"), AvgRollupMs: averageDurationMs(base, final, "fanout_rollup_duration_seconds"), AvgFlushMs: averageDurationMs(base, final, "fanout_flush_duration_seconds"), diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 7bae6b5e..4d8ecf19 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -111,7 +111,7 @@ func main() { } defer repository.Close() - // DuckDB is the SQL engine over open Parquet; hot indexed reads use the same + // DuckDB is the SQL engine over open Parquet; indexed trace reads use the same // repository directly through the typed observability kernel. q, err := query.NewDuck(ctx, cfg, repository) if err != nil { diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 index 01acf2a4..52383e83 100644 --- a/docs/diagrams/architecture.d2 +++ b/docs/diagrams/architecture.d2 @@ -26,7 +26,7 @@ fanout: "fanout — one Go process" { agent: "Agent runtime\nmodel + tool loop" mcp: "MCP server\n5 typed + 4 dashboard tools" obs: "Typed observability contract" - commit: "Telemetry commit worker\nWAL to Parquet" + commit: "Telemetry commit workers\natomic Parquet batches" query: "Query kernel\nDuckDB + rollups" alert: "Alert engine\nrule evaluation + webhooks" @@ -44,7 +44,7 @@ fanout: "fanout — one Go process" { store: Storage { style.fill: transparent - telemetry: "WAL + manifest + Parquet\nstorage.data_dir/telemetry" {shape: cylinder} + telemetry: "Parquet batches + trace indexes\nstorage.data_dir/telemetry" {shape: cylinder} qstate: "Query catalog\nstorage.data_dir/query" {shape: cylinder} control: "Control SQLite\nstorage.data_dir/control" {shape: cylinder} } diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index cd4d4d86..0ed5235f 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workerWAL to ParquetQuery kernelDuckDB + rollupsAlert enginerule evaluation + webhooksWAL + manifest + Parquetstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsmerge and maintenanceOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + .d2-4225682530 .fill-N1{fill:#0A0F25;} + .d2-4225682530 .fill-N2{fill:#676C7E;} + .d2-4225682530 .fill-N3{fill:#9499AB;} + .d2-4225682530 .fill-N4{fill:#CFD2DD;} + .d2-4225682530 .fill-N5{fill:#DEE1EB;} + .d2-4225682530 .fill-N6{fill:#EEF1F8;} + .d2-4225682530 .fill-N7{fill:#FFFFFF;} + .d2-4225682530 .fill-B1{fill:#000536;} + .d2-4225682530 .fill-B2{fill:#0F66B7;} + .d2-4225682530 .fill-B3{fill:#4393DD;} + .d2-4225682530 .fill-B4{fill:#87BFF3;} + .d2-4225682530 .fill-B5{fill:#BCDDFB;} + .d2-4225682530 .fill-B6{fill:#E5F3FF;} + .d2-4225682530 .fill-AA2{fill:#7639C5;} + .d2-4225682530 .fill-AA4{fill:#C1A2F3;} + .d2-4225682530 .fill-AA5{fill:#DACEFB;} + .d2-4225682530 .fill-AB4{fill:#EA99C6;} + .d2-4225682530 .fill-AB5{fill:#FFDEF1;} + .d2-4225682530 .stroke-N1{stroke:#0A0F25;} + .d2-4225682530 .stroke-N2{stroke:#676C7E;} + .d2-4225682530 .stroke-N3{stroke:#9499AB;} + .d2-4225682530 .stroke-N4{stroke:#CFD2DD;} + .d2-4225682530 .stroke-N5{stroke:#DEE1EB;} + .d2-4225682530 .stroke-N6{stroke:#EEF1F8;} + .d2-4225682530 .stroke-N7{stroke:#FFFFFF;} + .d2-4225682530 .stroke-B1{stroke:#000536;} + .d2-4225682530 .stroke-B2{stroke:#0F66B7;} + .d2-4225682530 .stroke-B3{stroke:#4393DD;} + .d2-4225682530 .stroke-B4{stroke:#87BFF3;} + .d2-4225682530 .stroke-B5{stroke:#BCDDFB;} + .d2-4225682530 .stroke-B6{stroke:#E5F3FF;} + .d2-4225682530 .stroke-AA2{stroke:#7639C5;} + .d2-4225682530 .stroke-AA4{stroke:#C1A2F3;} + .d2-4225682530 .stroke-AA5{stroke:#DACEFB;} + .d2-4225682530 .stroke-AB4{stroke:#EA99C6;} + .d2-4225682530 .stroke-AB5{stroke:#FFDEF1;} + .d2-4225682530 .background-color-N1{background-color:#0A0F25;} + .d2-4225682530 .background-color-N2{background-color:#676C7E;} + .d2-4225682530 .background-color-N3{background-color:#9499AB;} + .d2-4225682530 .background-color-N4{background-color:#CFD2DD;} + .d2-4225682530 .background-color-N5{background-color:#DEE1EB;} + .d2-4225682530 .background-color-N6{background-color:#EEF1F8;} + .d2-4225682530 .background-color-N7{background-color:#FFFFFF;} + .d2-4225682530 .background-color-B1{background-color:#000536;} + .d2-4225682530 .background-color-B2{background-color:#0F66B7;} + .d2-4225682530 .background-color-B3{background-color:#4393DD;} + .d2-4225682530 .background-color-B4{background-color:#87BFF3;} + .d2-4225682530 .background-color-B5{background-color:#BCDDFB;} + .d2-4225682530 .background-color-B6{background-color:#E5F3FF;} + .d2-4225682530 .background-color-AA2{background-color:#7639C5;} + .d2-4225682530 .background-color-AA4{background-color:#C1A2F3;} + .d2-4225682530 .background-color-AA5{background-color:#DACEFB;} + .d2-4225682530 .background-color-AB4{background-color:#EA99C6;} + .d2-4225682530 .background-color-AB5{background-color:#FFDEF1;} + .d2-4225682530 .color-N1{color:#0A0F25;} + .d2-4225682530 .color-N2{color:#676C7E;} + .d2-4225682530 .color-N3{color:#9499AB;} + .d2-4225682530 .color-N4{color:#CFD2DD;} + .d2-4225682530 .color-N5{color:#DEE1EB;} + .d2-4225682530 .color-N6{color:#EEF1F8;} + .d2-4225682530 .color-N7{color:#FFFFFF;} + .d2-4225682530 .color-B1{color:#000536;} + .d2-4225682530 .color-B2{color:#0F66B7;} + .d2-4225682530 .color-B3{color:#4393DD;} + .d2-4225682530 .color-B4{color:#87BFF3;} + .d2-4225682530 .color-B5{color:#BCDDFB;} + .d2-4225682530 .color-B6{color:#E5F3FF;} + .d2-4225682530 .color-AA2{color:#7639C5;} + .d2-4225682530 .color-AA4{color:#C1A2F3;} + .d2-4225682530 .color-AA5{color:#DACEFB;} + .d2-4225682530 .color-AB4{color:#EA99C6;} + .d2-4225682530 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-4225682530);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-4225682530);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-4225682530);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-4225682530);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-4225682530);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-4225682530);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-4225682530);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-4225682530);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-4225682530);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-4225682530);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-4225682530);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-4225682530);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsmerge and maintenanceOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - + diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 index fb2bf477..850947f3 100644 --- a/docs/diagrams/persistence.d2 +++ b/docs/diagrams/persistence.d2 @@ -17,7 +17,7 @@ writers: Writers { maint: "Merge and maintenance" {shape: rectangle} } -wal: "Durable WAL + commit worker\nrequest acknowledged after fsync" { +commit: "Parallel atomic Parquet commits\nacknowledged after directory publication" { shape: rectangle style: { stroke-width: 2 @@ -25,7 +25,7 @@ wal: "Durable WAL + commit worker\nrequest acknowledged after fsync" { } } -telemetry: "Manifest + Parquet + hot span index\nstorage.data_dir/telemetry" { +telemetry: "Parquet batches + trace sidecars\nstorage.data_dir/telemetry" { shape: cylinder tooltip: Authoritative telemetry and its crash-recovery state. } @@ -45,8 +45,8 @@ control_tables: "users, user_identities, verifications, sessions, auth_audit_eve style.font-size: 13 } -writers.ingest -> wal -wal -> telemetry: "asynchronous publication" +writers.ingest -> commit +commit -> telemetry: "atomic directory rename" writers.rollup -> querystate writers.maint -> telemetry telemetry <- querystate: "DuckDB scans Parquet" diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 80ff6de4..17f7976a 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,27 +1,27 @@ -WritersDurable WAL + commit workerrequest acknowledged after fsyncManifest + Parquet + hot span indexstorage.data_dir/telemetryAuthoritative telemetry and its crash-recovery state.DuckDB query statestorage.data_dir/queryRebuildable rollups and temp spill. Not the telemetry itself.Control SQLitestorage.data_dir/control/fanout.sqliteApplication state. Never on the telemetry write path.users, user_identities, verifications, sessions, auth_audit_eventsoauth_clients, oauth_tokens, oauth_authorization_codesdashboards, dashboard_widgets, dashboard_stateagui_threads, agui_runs, alert_rules, alerts, settingsOTLP requestspans, logs, metricsRollupsservice, endpoint, edgeMerge and maintenance asynchronous publication DuckDB scans Parquet Authoritative telemetry and its crash-recovery state. - + .d2-1755061225 .fill-N1{fill:#0A0F25;} + .d2-1755061225 .fill-N2{fill:#676C7E;} + .d2-1755061225 .fill-N3{fill:#9499AB;} + .d2-1755061225 .fill-N4{fill:#CFD2DD;} + .d2-1755061225 .fill-N5{fill:#DEE1EB;} + .d2-1755061225 .fill-N6{fill:#EEF1F8;} + .d2-1755061225 .fill-N7{fill:#FFFFFF;} + .d2-1755061225 .fill-B1{fill:#000536;} + .d2-1755061225 .fill-B2{fill:#0F66B7;} + .d2-1755061225 .fill-B3{fill:#4393DD;} + .d2-1755061225 .fill-B4{fill:#87BFF3;} + .d2-1755061225 .fill-B5{fill:#BCDDFB;} + .d2-1755061225 .fill-B6{fill:#E5F3FF;} + .d2-1755061225 .fill-AA2{fill:#7639C5;} + .d2-1755061225 .fill-AA4{fill:#C1A2F3;} + .d2-1755061225 .fill-AA5{fill:#DACEFB;} + .d2-1755061225 .fill-AB4{fill:#EA99C6;} + .d2-1755061225 .fill-AB5{fill:#FFDEF1;} + .d2-1755061225 .stroke-N1{stroke:#0A0F25;} + .d2-1755061225 .stroke-N2{stroke:#676C7E;} + .d2-1755061225 .stroke-N3{stroke:#9499AB;} + .d2-1755061225 .stroke-N4{stroke:#CFD2DD;} + .d2-1755061225 .stroke-N5{stroke:#DEE1EB;} + .d2-1755061225 .stroke-N6{stroke:#EEF1F8;} + .d2-1755061225 .stroke-N7{stroke:#FFFFFF;} + .d2-1755061225 .stroke-B1{stroke:#000536;} + .d2-1755061225 .stroke-B2{stroke:#0F66B7;} + .d2-1755061225 .stroke-B3{stroke:#4393DD;} + .d2-1755061225 .stroke-B4{stroke:#87BFF3;} + .d2-1755061225 .stroke-B5{stroke:#BCDDFB;} + .d2-1755061225 .stroke-B6{stroke:#E5F3FF;} + .d2-1755061225 .stroke-AA2{stroke:#7639C5;} + .d2-1755061225 .stroke-AA4{stroke:#C1A2F3;} + .d2-1755061225 .stroke-AA5{stroke:#DACEFB;} + .d2-1755061225 .stroke-AB4{stroke:#EA99C6;} + .d2-1755061225 .stroke-AB5{stroke:#FFDEF1;} + .d2-1755061225 .background-color-N1{background-color:#0A0F25;} + .d2-1755061225 .background-color-N2{background-color:#676C7E;} + .d2-1755061225 .background-color-N3{background-color:#9499AB;} + .d2-1755061225 .background-color-N4{background-color:#CFD2DD;} + .d2-1755061225 .background-color-N5{background-color:#DEE1EB;} + .d2-1755061225 .background-color-N6{background-color:#EEF1F8;} + .d2-1755061225 .background-color-N7{background-color:#FFFFFF;} + .d2-1755061225 .background-color-B1{background-color:#000536;} + .d2-1755061225 .background-color-B2{background-color:#0F66B7;} + .d2-1755061225 .background-color-B3{background-color:#4393DD;} + .d2-1755061225 .background-color-B4{background-color:#87BFF3;} + .d2-1755061225 .background-color-B5{background-color:#BCDDFB;} + .d2-1755061225 .background-color-B6{background-color:#E5F3FF;} + .d2-1755061225 .background-color-AA2{background-color:#7639C5;} + .d2-1755061225 .background-color-AA4{background-color:#C1A2F3;} + .d2-1755061225 .background-color-AA5{background-color:#DACEFB;} + .d2-1755061225 .background-color-AB4{background-color:#EA99C6;} + .d2-1755061225 .background-color-AB5{background-color:#FFDEF1;} + .d2-1755061225 .color-N1{color:#0A0F25;} + .d2-1755061225 .color-N2{color:#676C7E;} + .d2-1755061225 .color-N3{color:#9499AB;} + .d2-1755061225 .color-N4{color:#CFD2DD;} + .d2-1755061225 .color-N5{color:#DEE1EB;} + .d2-1755061225 .color-N6{color:#EEF1F8;} + .d2-1755061225 .color-N7{color:#FFFFFF;} + .d2-1755061225 .color-B1{color:#000536;} + .d2-1755061225 .color-B2{color:#0F66B7;} + .d2-1755061225 .color-B3{color:#4393DD;} + .d2-1755061225 .color-B4{color:#87BFF3;} + .d2-1755061225 .color-B5{color:#BCDDFB;} + .d2-1755061225 .color-B6{color:#E5F3FF;} + .d2-1755061225 .color-AA2{color:#7639C5;} + .d2-1755061225 .color-AA4{color:#C1A2F3;} + .d2-1755061225 .color-AA5{color:#DACEFB;} + .d2-1755061225 .color-AB4{color:#EA99C6;} + .d2-1755061225 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-1755061225);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-1755061225);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-1755061225);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-1755061225);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-1755061225);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-1755061225);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-1755061225);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-1755061225);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-1755061225);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-1755061225);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-1755061225);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-1755061225);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>WritersParallel atomic Parquet commitsacknowledged after directory publicationParquet batches + trace sidecarsstorage.data_dir/telemetryAuthoritative telemetry and its crash-recovery state.DuckDB query statestorage.data_dir/queryRebuildable rollups and temp spill. Not the telemetry itself.Control SQLitestorage.data_dir/control/fanout.sqliteApplication state. Never on the telemetry write path.users, user_identities, verifications, sessions, auth_audit_eventsoauth_clients, oauth_tokens, oauth_authorization_codesdashboards, dashboard_widgets, dashboard_stateagui_threads, agui_runs, alert_rules, alerts, settingsOTLP requestspans, logs, metricsRollupsservice, endpoint, edgeMerge and maintenance atomic directory rename DuckDB scans Parquet Authoritative telemetry and its crash-recovery state. + - + -Rebuildable rollups and temp spill. Not the telemetry itself. - +Rebuildable rollups and temp spill. Not the telemetry itself. + - + -Application state. Never on the telemetry write path. - +Application state. Never on the telemetry write path. + - + - - - - + + + + diff --git a/docs/operations.md b/docs/operations.md index f508bd76..49599efd 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -67,14 +67,14 @@ its configured cycle rather than immediately when the setting changes. ## Backup The supported portable baseline is a **cold backup** of the complete -`FANOUT_DATA_DIR`. It contains the telemetry WAL, commit manifest, Parquet -files, query state, and the control SQLite database; copying only one +`FANOUT_DATA_DIR`. It contains atomic telemetry Parquet batches, query state, +and the control SQLite database; copying only one subdirectory does not produce a recoverable installation. 1. Record the running Fanout version and configuration, excluding secrets from ordinary logs or tickets. 2. Stop Fanout cleanly and wait for the process to exit. Shutdown stops both - OTLP listeners before draining the telemetry commit worker. + OTLP listeners before draining the telemetry commit workers. 3. Snapshot or copy the complete data directory with ownership and permissions preserved. 4. Restart Fanout and confirm `/readyz`. diff --git a/fanout.example.yaml b/fanout.example.yaml index 7eebd4e4..c8c6eb17 100644 --- a/fanout.example.yaml +++ b/fanout.example.yaml @@ -26,7 +26,6 @@ ingest: storage: data_dir: ./data # FANOUT_DATA_DIR retention_days: 30 # FANOUT_RETENTION_DAYS - hot_retention: 24h # FANOUT_HOT_RETENTION rollup_interval: 1m # FANOUT_ROLLUP_INTERVAL rollup_skip_to_latest: false # FANOUT_ROLLUP_SKIP_TO_LATEST maintenance_interval: 1h # FANOUT_MAINTENANCE_INTERVAL diff --git a/go.mod b/go.mod index 887dab9b..f43c144e 100644 --- a/go.mod +++ b/go.mod @@ -8,19 +8,18 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260826145851-49e71f2b2d21 github.com/alexedwards/scs/v2 v2.9.0 - github.com/apache/arrow-go/v18 v18.7.0 github.com/coreos/go-oidc/v3 v3.20.0 github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 - github.com/klauspost/compress v1.19.2 github.com/knadh/koanf/parsers/yaml v1.1.1 github.com/knadh/koanf/providers/confmap v1.0.1 github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/v2 v2.3.6 github.com/labstack/echo/v5 v5.3.1 github.com/modelcontextprotocol/go-sdk v1.7.0 + github.com/parquet-go/parquet-go v0.32.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/wneessen/go-mail v0.8.1 @@ -39,6 +38,7 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/andybalholm/brotli v1.2.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/apache/arrow-go/v18 v18.5.1 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect @@ -61,6 +61,7 @@ require ( github.com/google/jsonschema-go v0.4.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/hcl/v2 v2.24.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/knadh/koanf/maps v0.1.3 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -70,6 +71,8 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.29 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect @@ -77,6 +80,7 @@ require ( github.com/segmentio/asm v1.2.1 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/sirupsen/logrus v1.10.2 // indirect + github.com/twpayne/go-geom v1.6.1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zclconf/go-cty v1.19.0 // indirect github.com/zclconf/go-cty-yaml v1.2.0 // indirect @@ -86,8 +90,10 @@ require ( golang.org/x/mod v0.40.0 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect + golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/tools v0.49.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7e10e4e5..d10fac7c 100644 --- a/go.sum +++ b/go.sum @@ -10,14 +10,18 @@ github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260826145851-49e71f2b github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260826145851-49e71f2b2d21/go.mod h1:ERAMOexUee4AIuoxksuuGoEcHl3aqLwaazjGwlR9ZCI= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= +github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alexedwards/scs/v2 v2.9.0 h1:xa05mVpwTBm1iLeTMNFfAWpKUm4fXAW7CeAViqBVS90= github.com/alexedwards/scs/v2 v2.9.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= -github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= @@ -70,6 +74,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -86,7 +92,11 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= @@ -111,6 +121,10 @@ github.com/labstack/echo/v5 v5.3.1 h1:75maCxkQVGualckLc/5s/ihgpH1a1Dc6AuGWNVNs6b github.com/labstack/echo/v5 v5.3.1/go.mod h1:4iEGNQiPPZnkfYpNR/L6fINd3NLiGWUD5+eBotFALas= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -123,6 +137,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= +github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= @@ -141,10 +161,10 @@ github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfv github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/wneessen/go-mail v0.8.1 h1:tVcncj02/QySVFw3zr/kXOzZcuFQqBNT6K+Rbgm/pcM= github.com/wneessen/go-mail v0.8.1/go.mod h1:dWZ61zadzCIyvB4y1/YzC5O7MrbbzBfPkARmbosdf8w= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -195,12 +215,16 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 h1:izFU9hz7aeLI/Mi1J0991ae+xcwRLr7hTqWnB/9aIIU= diff --git a/internal/config/config.go b/internal/config/config.go index 0ec2c97b..78f27664 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,11 +33,8 @@ type Config struct { RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` - // HotRetention controls how long the custom indexed segments are retained. - // Older telemetry remains queryable in Parquet through DuckDB. - HotRetention time.Duration `koanf:"storage.hot_retention" env:"FANOUT_HOT_RETENTION" default:"24h"` - // MaintenanceInterval controls hot-segment pruning, Parquet retention and - // compaction, and query-cache checkpointing. + // MaintenanceInterval controls Parquet retention and compaction, and + // query-cache checkpointing. MaintenanceInterval time.Duration `koanf:"storage.maintenance_interval" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` // RollupSkipToLatest, set once at boot, advances every rollup watermark to the // current max ingested timestamp so existing data is treated as already-rolled-up @@ -54,9 +51,9 @@ type Config struct { // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing // resolves to a ~6.4 GB memory cap and 4 query threads (deterministic from - // 8 GB / 4 vCPU). For a current throughput figure run `just stress hetzner` - // rather than trusting a number here — as of 2026-06 it handled ~55k rows/s - // with 0 drops and ~0.4 GB RSS, but that will drift with the ingest path. + // 8 GB / 4 vCPU). Measure the target host with cmd/bench before setting + // production limits; throughput and memory use vary with CPU, disk, and the + // telemetry mix. // // Kept free-floating, separated from the field below by a blank line, so it // stays context for the group rather than becoming DuckDBMemory's own doc @@ -192,9 +189,6 @@ func (c Config) Validate() error { if c.RetentionDays < 0 { return fmt.Errorf("storage.retention_days must be >= 0, got %d", c.RetentionDays) } - if c.HotRetention <= 0 { - return fmt.Errorf("storage.hot_retention must be positive, got %s", c.HotRetention) - } if c.MaintenanceInterval < time.Second { return fmt.Errorf("storage.maintenance_interval must be at least 1s, got %s", c.MaintenanceInterval) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6872141d..bcd16947 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -611,7 +611,6 @@ func TestValidate(t *testing.T) { IngestBatchSize: 50000, RollupInterval: time.Minute, RetentionDays: 30, - HotRetention: 24 * time.Hour, MaintenanceInterval: time.Hour, DuckDBMaxConns: 4, AlertEvaluationInterval: 30 * time.Second, @@ -642,7 +641,6 @@ func TestValidate(t *testing.T) { {"RollupInterval=0", func(c *Config) { c.RollupInterval = 0 }}, {"RollupInterval=999ms", func(c *Config) { c.RollupInterval = 999 * time.Millisecond }}, {"RetentionDays=-1", func(c *Config) { c.RetentionDays = -1 }}, - {"HotRetention=0", func(c *Config) { c.HotRetention = 0 }}, {"HTTPAddr empty", func(c *Config) { c.HTTPAddr = "" }}, {"OTLPGRPCAddr empty", func(c *Config) { c.OTLPGRPCAddr = "" }}, {"OTLPHTTPAddr empty", func(c *Config) { c.OTLPHTTPAddr = "" }}, @@ -699,14 +697,6 @@ func TestValidate(t *testing.T) { } }) - t.Run("short hot retention valid", func(t *testing.T) { - c := valid - c.HotRetention = 15 * time.Minute - if err := c.Validate(); err != nil { - t.Errorf("HotRetention=15m should be valid: %v", err) - } - }) - t.Run("local mode allows absent SMTP and agent", func(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index e26d2f4d..5dc8f04e 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -330,9 +330,6 @@ func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { }}}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { - t.Fatal(err) - } mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", "error", "error", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). @@ -353,14 +350,13 @@ func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { } } -func TestLogsAreIndependentOfSpanHotPruneBoundary(t *testing.T) { +func TestLogsQueryParquetAcrossBatches(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) - cutoff := start.Add(250 * time.Millisecond) end := start.Add(time.Second) if err := svc.repository.Commit(telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.Add(150 * time.Millisecond).UnixNano(), Body: "newer-old", Severity: "INFO"}, - {Namespace: "prod", TimeUnixNanos: start.Add(300 * time.Millisecond).UnixNano(), Body: "newer-hot", Severity: "INFO"}, + {Namespace: "prod", TimeUnixNanos: start.Add(300 * time.Millisecond).UnixNano(), Body: "newer-batch", Severity: "INFO"}, }}); err != nil { t.Fatal(err) } @@ -370,13 +366,10 @@ func TestLogsAreIndependentOfSpanHotPruneBoundary(t *testing.T) { }}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { - t.Fatal(err) - } mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). WithArgs(start, end, "prod", "prod", "", "", "", "", "", "", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). - AddRow(start.Add(300*time.Millisecond), "INFO", "", "newer-hot", "", ""). + AddRow(start.Add(300*time.Millisecond), "INFO", "", "newer-batch", "", ""). AddRow(start.Add(200*time.Millisecond), "INFO", "", "late-boundary", "", ""). AddRow(start.Add(150*time.Millisecond), "INFO", "", "newer-old", "", ""). AddRow(start.Add(100*time.Millisecond), "INFO", "", "late-old", "", "")) @@ -391,7 +384,7 @@ func TestLogsAreIndependentOfSpanHotPruneBoundary(t *testing.T) { if len(result.Data.Entries) != 4 || result.Summary != "4 logs matched the selected telemetry window" { t.Fatalf("boundary logs = %#v", result) } - if result.Data.Entries[0].Body != "newer-hot" || result.Provenance.DataSource != "parquet" { + if result.Data.Entries[0].Body != "newer-batch" || result.Provenance.DataSource != "parquet" { t.Fatalf("boundary result = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { @@ -429,14 +422,16 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { } } -func TestTraceUsesParquetWhenHotSegmentsMiss(t *testing.T) { +func TestTraceUsesIndexedParquet(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). - WithArgs("parquet-trace", start, end, "prod", "prod", 10). - WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). - AddRow("root", "", "checkout", "pay", "SERVER", start, 25.0, "ERROR", "declined")) + if err := svc.repository.Commit(telemetrystore.Batch{ID: "indexed-trace", Spans: []telemetry.Span{{ + Namespace: "prod", TraceID: "parquet-trace", SpanID: "root", ServiceName: "checkout", Name: "pay", + Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 25, StatusCode: "ERROR", StatusMsg: "declined", + }}}); err != nil { + t.Fatal(err) + } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). WithArgs("parquet-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). @@ -446,17 +441,19 @@ func TestTraceUsesParquetWhenHotSegmentsMiss(t *testing.T) { t.Fatal(err) } if len(result.Data.Spans) != 1 || len(result.Data.Logs) != 1 || !result.Data.HasError { - t.Fatalf("Parquet trace detail = %#v", result.Data) + t.Fatalf("indexed Parquet trace detail = %#v", result.Data) + } + if result.Provenance.DataSource != "parquet_index" { + t.Fatalf("trace data source = %q", result.Provenance.DataSource) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) } } -func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { +func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) - cutoff := start.Add(30 * time.Minute) end := start.Add(time.Hour) if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "root", ServiceName: "frontend", @@ -464,20 +461,12 @@ func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { }}}); err != nil { t.Fatal(err) } - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-hot-child", Spans: []telemetry.Span{{ + if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-newer-child", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "child", ParentSpanID: "root", ServiceName: "backend", StartUnixNanos: start.Add(50 * time.Minute).UnixNano(), DurationMS: 25, StatusCode: "OK", }}}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { - t.Fatal(err) - } - mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). - WithArgs("split-trace", start, end, "prod", "prod", 10). - WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). - AddRow("root", "", "frontend", "request", "SERVER", start.Add(10*time.Minute), 100.0, "ERROR", "failed"). - AddRow("child", "root", "backend", "work", "CLIENT", start.Add(50*time.Minute), 25.0, "OK", "")) mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). WithArgs("split-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) @@ -485,15 +474,15 @@ func TestTraceUsesParquetWhenTraceStraddlesHotBoundary(t *testing.T) { if err != nil { t.Fatal(err) } - if len(result.Data.Spans) != 2 || !result.Data.HasError || len(result.Data.Services) != 2 || result.Provenance.DataSource != "parquet" { - t.Fatalf("straddling trace = %#v", result) + if len(result.Data.Spans) != 2 || !result.Data.HasError || len(result.Data.Services) != 2 || result.Provenance.DataSource != "parquet_index" { + t.Fatalf("multi-batch trace = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) } } -func TestTraceUsesRebuiltHotTierWhenRootIsAboveCutoff(t *testing.T) { +func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) @@ -504,9 +493,6 @@ func TestTraceUsesRebuiltHotTierWhenRootIsAboveCutoff(t *testing.T) { }}}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { - t.Fatal(err) - } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). WithArgs("new-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) @@ -514,15 +500,15 @@ func TestTraceUsesRebuiltHotTierWhenRootIsAboveCutoff(t *testing.T) { if err != nil { t.Fatal(err) } - if len(result.Data.Spans) != 1 || result.Data.Spans[0].SpanID != "root" || result.Provenance.DataSource != "fanout_segments" { - t.Fatalf("rebuilt hot trace = %#v", result) + if len(result.Data.Spans) != 1 || result.Data.Spans[0].SpanID != "root" || result.Provenance.DataSource != "parquet_index" { + t.Fatalf("indexed trace = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) } } -func TestTraceDoesNotUseRootFromAnotherNamespaceAsHotCoverage(t *testing.T) { +func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) @@ -533,14 +519,6 @@ func TestTraceDoesNotUseRootFromAnotherNamespaceAsHotCoverage(t *testing.T) { }}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(cutoff.UnixNano()); err != nil { - t.Fatal(err) - } - mock.ExpectQuery(regexp.QuoteMeta(traceSpansQuery)). - WithArgs("shared-trace", start, end, "prod", "prod", 10). - WillReturnRows(sqlmock.NewRows([]string{"span_id", "parent_span_id", "service", "operation", "kind", "start_time", "duration_ms", "status", "status_message"}). - AddRow("old-root", "", "frontend", "request", "SERVER", start.Add(10*time.Minute), 10.0, "OK", ""). - AddRow("child", "old-root", "backend", "work", "CLIENT", cutoff.Add(time.Minute), 5.0, "OK", "")) mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). WithArgs("shared-trace", start, end, "prod", "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) @@ -548,8 +526,8 @@ func TestTraceDoesNotUseRootFromAnotherNamespaceAsHotCoverage(t *testing.T) { if err != nil { t.Fatal(err) } - if len(result.Data.Spans) != 2 || result.Provenance.DataSource != "parquet" { - t.Fatalf("cross-namespace trace = %#v", result) + if len(result.Data.Spans) != 1 || result.Data.Spans[0].SpanID != "child" || result.Provenance.DataSource != "parquet_index" { + t.Fatalf("namespace-filtered trace = %#v", result) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) @@ -568,9 +546,6 @@ func TestLogsBoundsParquetQueryWithLimitAndAggregatedBuckets(t *testing.T) { }}}); err != nil { t.Fatal(err) } - if _, err := svc.repository.PruneHot(end.Add(time.Hour).UnixNano()); err != nil { - t.Fatal(err) - } // The sample query must carry the row limit so a wide window cannot stream // the whole Parquet history through the driver. mock.ExpectQuery(regexp.QuoteMeta(logEntriesQuery)). diff --git a/internal/observability/trace.go b/internal/observability/trace.go index e08b7abb..01e9f16b 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -6,6 +6,8 @@ import ( "sort" "strings" "time" + + "github.com/labstack/fanout/internal/telemetry" ) const recentTraceQuery = ` @@ -17,14 +19,6 @@ ORDER BY MAX(CASE WHEN upper(status) IN ('ERROR', 'STATUS_CODE_ERROR') THEN 1 EL MAX(end_time) - MIN(start_time) DESC LIMIT 1` -const traceSpansQuery = ` -SELECT span_id, coalesce(parent_span_id, ''), service, operation, kind, start_time, - duration_ms, status, coalesce(status_message, '') -FROM spans -WHERE trace_id = ? AND start_time >= ? AND start_time < ? AND (? = '' OR namespace = ?) -ORDER BY start_time ASC, duration_ms DESC -LIMIT ?` - const traceLogsQuery = ` SELECT time, severity, coalesce(service, ''), body, coalesce(trace_id, ''), coalesce(span_id, '') FROM logs @@ -60,38 +54,19 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin rows.Close() } - dataSource := "fanout_segments" + dataSource := "parquet_index" data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} - hotCutoff := int64(0) - hotHasRoot := false if traceID != "" { - storedSpans, spanCutoff, readErr := s.repository.HotTrace(traceID) + storedSpans, readErr := s.repository.Trace(ctx, telemetry.TraceQuery{ + TraceID: traceID, Namespace: scope.Namespace, + StartNanos: scope.Start.UnixNano(), EndNanos: scope.End.UnixNano(), Limit: limit, + }) if readErr != nil { - return Result[TraceDetail]{}, fmt.Errorf("read trace segments: %w", readErr) + return Result[TraceDetail]{}, fmt.Errorf("read indexed Parquet trace: %w", readErr) } - hotCutoff = spanCutoff - startNanos, endNanos := scope.Start.UnixNano(), scope.End.UnixNano() for _, row := range storedSpans { - matchesNamespace := scope.Namespace == "" || row.Namespace == scope.Namespace - if row.ParentSpanID == "" && row.StartUnixNanos >= hotCutoff && - row.StartUnixNanos < endNanos && matchesNamespace { - hotHasRoot = true - } - if row.StartUnixNanos < startNanos || row.StartUnixNanos >= endNanos || - !matchesNamespace { - continue - } data.Spans = append(data.Spans, TraceSpan{SpanID: row.SpanID, ParentSpanID: row.ParentSpanID, Service: row.ServiceName, Operation: row.Name, Kind: row.Kind, Start: time.Unix(0, row.StartUnixNanos).UTC(), DurationMS: row.DurationMS, Status: row.StatusCode, StatusMessage: row.StatusMsg}) } - sort.Slice(data.Spans, func(i, j int) bool { - if data.Spans[i].Start.Equal(data.Spans[j].Start) { - return data.Spans[i].DurationMS > data.Spans[j].DurationMS - } - return data.Spans[i].Start.Before(data.Spans[j].Start) - }) - if len(data.Spans) > limit { - data.Spans = data.Spans[:limit] - } serviceSet := make(map[string]struct{}) var first, last time.Time @@ -116,23 +91,6 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data.Services = append(data.Services, name) } sort.Strings(data.Services) - - } - // A root retained at or above the durable cutoff proves the hot index contains - // the beginning of this trace, including immediately after a tier rebuild. A - // crossing scope without that root may contain only a suffix and must use the - // authoritative Parquet copy. - hotComplete := scope.Start.UnixNano() >= hotCutoff || hotHasRoot - if traceID != "" && (len(data.Spans) == 0 || !hotComplete) { - data, err = s.traceFromParquet(ctx, scope, traceID, limit) - if err != nil { - return Result[TraceDetail]{}, err - } - dataSource = "parquet" - } else if traceID != "" { - // Parquet is authoritative and published before the disposable hot index. - // DuckDB can apply trace_id and LIMIT to the associated logs without a - // full Go decode while preserving clock-skewed trace events. data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) if err != nil { return Result[TraceDetail]{}, err @@ -146,57 +104,6 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{Schema: TraceSchema, Summary: summary, Data: data, Provenance: s.provenanceFor(scope, dataSource)}, nil } -func (s *Service) traceFromParquet(ctx context.Context, scope Scope, traceID string, limit int) (TraceDetail, error) { - data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} - rows, err := s.db.QueryContext(ctx, traceSpansQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) - if err != nil { - return TraceDetail{}, fmt.Errorf("query trace parquet spans: %w", err) - } - for rows.Next() { - var span TraceSpan - if err := rows.Scan(&span.SpanID, &span.ParentSpanID, &span.Service, &span.Operation, &span.Kind, &span.Start, &span.DurationMS, &span.Status, &span.StatusMessage); err != nil { - rows.Close() - return TraceDetail{}, fmt.Errorf("scan trace parquet span: %w", err) - } - data.Spans = append(data.Spans, span) - } - if err := rows.Err(); err != nil { - rows.Close() - return TraceDetail{}, fmt.Errorf("iterate trace parquet spans: %w", err) - } - rows.Close() - - serviceSet := make(map[string]struct{}) - var first, last time.Time - for _, span := range data.Spans { - if first.IsZero() || span.Start.Before(first) { - first = span.Start - } - if end := span.Start.Add(time.Duration(span.DurationMS * float64(time.Millisecond))); end.After(last) { - last = end - } - if strings.Contains(strings.ToUpper(span.Status), "ERROR") { - data.HasError = true - } - if span.Service != "" { - serviceSet[span.Service] = struct{}{} - } - } - if !first.IsZero() { - data.DurationMS = last.Sub(first).Seconds() * 1000 - } - for service := range serviceSet { - data.Services = append(data.Services, service) - } - sort.Strings(data.Services) - - data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) - if err != nil { - return TraceDetail{}, err - } - return data, nil -} - func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID string, limit int) ([]LogEntry, error) { rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) if err != nil { diff --git a/internal/query/duck.go b/internal/query/duck.go index eff5cbe2..3630fc05 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -71,7 +71,7 @@ const ( ) // rollupPublicationSafetyLag covers the maximum public SQL hold, publication -// grace, bounded commit retries, and queued segment encoding with headroom for +// grace, bounded commit retries, and queued Parquet encoding with headroom for // a busy disk. Rows stamped at request receipt remain inside the recomputed // tail until their Parquet commit becomes visible. const rollupPublicationSafetyLag = 5 * time.Minute @@ -206,7 +206,6 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. } d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} - repository.SetParquetPublishLock(&d.parquetMu) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -289,6 +288,10 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { } func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, error) { + // Every pooled connection must use UTC. DuckDB otherwise inherits the host + // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock + // TIMESTAMP values, while API windows arrive in UTC. + // // temp_directory is an instance-global setting: re-setting it after the temp // dir has already been used fails with "Cannot switch temporary directory // after the current one has been used". The boot hook runs once per pooled @@ -298,6 +301,9 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB var tempDirErr error connector, err := duckdb.NewConnector(dsn, func(execer driver.ExecerContext) error { + if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil { + return fmt.Errorf("set timezone: %w", err) + } tempDirOnce.Do(func() { _, tempDirErr = execer.ExecContext(ctx, "SET temp_directory="+sqlLiteral(tempDir), nil) }) @@ -439,11 +445,8 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() - cutoff := time.Now().Add(-d.cfg.HotRetention).UnixNano() var pruneErr error if d.repository != nil { - _, pruneErr = d.repository.PruneHot(cutoff) - _, hotCompactErr := d.repository.CompactHot(64) var parquetErr error if d.cfg.RetentionDays > 0 { d.parquetMu.Lock() @@ -459,7 +462,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { compactResult = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) - pruneErr = errors.Join(pruneErr, hotCompactErr, parquetErr, compactErr) + pruneErr = errors.Join(pruneErr, parquetErr, compactErr) } var cacheErr error unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 83e01b7a..98839af3 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -2,6 +2,7 @@ package query import ( "context" + "database/sql" "errors" "testing" "time" @@ -10,6 +11,7 @@ import ( "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/metrics" "github.com/labstack/fanout/internal/query/writegate" + "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -149,6 +151,74 @@ func TestNewDuckUsesSingleConnectionPool(t *testing.T) { } } +func TestNewDuckUsesUTCForEveryConnection(t *testing.T) { + t.Setenv("TZ", "America/Los_Angeles") + ctx := context.Background() + cfg := config.Config{ + DataDir: t.TempDir(), + RollupInterval: time.Minute, + DuckDBMemory: "128MB", + DuckDBMaxConns: 4, + DuckDBThreads: 2, + MaintenanceInterval: time.Hour, + } + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck() error = %v", err) + } + defer d.Close() + eventTime := time.Date(2026, 8, 27, 16, 0, 30, 0, time.UTC) + if err := repository.Commit(telemetrystore.Batch{ID: "timezone-window", Spans: []telemetry.Span{{ + Namespace: "default", TraceID: "trace-timezone", SpanID: "span-timezone", + ServiceName: "checkout", StartUnixNanos: eventTime.UnixNano(), DurationMS: 5, + StatusCode: "STATUS_CODE_OK", IngestedAt: eventTime.UnixNano(), + }}}); err != nil { + t.Fatalf("commit timezone fixture: %v", err) + } + d.rollupLagNanos = 0 + if _, err := d.rollupOnce(ctx); err != nil { + t.Fatalf("roll up timezone fixture: %v", err) + } + var spans int64 + if err := d.DB.QueryRowContext(ctx, ` +SELECT COALESCE(SUM(spans), 0)::BIGINT +FROM service_rollup +WHERE bucket >= ? AND bucket < ?`, eventTime.Add(-time.Minute), eventTime.Add(time.Minute)).Scan(&spans); err != nil { + t.Fatalf("query UTC rollup window: %v", err) + } + if spans != 1 { + t.Fatalf("UTC rollup window spans = %d, want 1", spans) + } + + connections := make([]*sql.Conn, 0, cfg.DuckDBMaxConns) + defer func() { + for _, connection := range connections { + _ = connection.Close() + } + }() + for range cfg.DuckDBMaxConns { + connection, err := d.DB.Conn(ctx) + if err != nil { + t.Fatalf("open pooled connection: %v", err) + } + connections = append(connections, connection) + } + for i, connection := range connections { + var zone string + if err := connection.QueryRowContext(ctx, "SELECT current_setting('TimeZone')").Scan(&zone); err != nil { + t.Fatalf("connection %d timezone query: %v", i, err) + } + if zone != "UTC" { + t.Fatalf("connection %d timezone = %q, want UTC", i, zone) + } + } +} + func TestQueryContextHoldsParquetLockUntilRowsFinish(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index 9d54076f..de4363e1 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -144,7 +144,7 @@ FROM telemetry.spans`).Scan(&spanBuckets); err != nil { func TestSkipRollupToLatest(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "2GB", RetentionDays: 30, HotRetention: 24 * time.Hour} + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "2GB", RetentionDays: 30} repository, err := telemetrystore.Open(cfg.TelemetryDir()) if err != nil { t.Fatalf("open telemetry repository: %v", err) diff --git a/internal/query/schema.go b/internal/query/schema.go index d2b5b0fd..308a5576 100644 --- a/internal/query/schema.go +++ b/internal/query/schema.go @@ -10,9 +10,9 @@ func GetSchema(dataDir string) string { const schemaTemplate = ` ## Fanout Data Schema -Fanout stores telemetry in indexed hot segments and open Parquet files. DuckDB -exposes the Parquet files through the read-only telemetry schema. The rebuildable -query cache and product state live under {DATA_DIR}. +Fanout stores telemetry in atomic Parquet batches with persistent trace indexes. +DuckDB exposes the Parquet files through the read-only telemetry schema. The +rebuildable query cache and product state live under {DATA_DIR}. Primary query surfaces: - spans view: clean span columns for most queries diff --git a/internal/query/views.go b/internal/query/views.go index 5665f07e..eee6f4ba 100644 --- a/internal/query/views.go +++ b/internal/query/views.go @@ -267,7 +267,7 @@ func CreateTables(db *sql.DB) error { } // CreateCacheTables creates only DuckDB's rebuildable query accelerators. The -// production telemetry rows themselves live in immutable segments and Parquet. +// production telemetry rows themselves live in immutable Parquet batches. func CreateCacheTables(db *sql.DB) error { if err := ensureCacheTable(db, "service_rollup", createServiceRollupTable, "namespace", "bucket", "service", "spans", "p50_ms", "p95_ms", "error_rate", "log_count", "metric_count"); err != nil { @@ -295,8 +295,12 @@ func CreateParquetViews(db *sql.DB, parquetDir string) error { return err } for _, signal := range []string{"spans", "logs", "metrics"} { - pattern := filepath.ToSlash(filepath.Join(parquetDir, signal, "*.parquet")) - stmt := fmt.Sprintf(`CREATE OR REPLACE VIEW telemetry.%s AS SELECT * FROM read_parquet(%s, union_by_name=true)`, signal, sqlLiteral(pattern)) + pattern := filepath.ToSlash(filepath.Join(parquetDir, "batches", "*.batch", signal+".parquet")) + projection := "*" + if signal == "spans" { + projection = "* EXCLUDE (_trace_hash)" + } + stmt := fmt.Sprintf(`CREATE OR REPLACE VIEW telemetry.%s AS SELECT %s FROM read_parquet(%s, union_by_name=true)`, signal, projection, sqlLiteral(pattern)) if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("create parquet view telemetry.%s: %w", signal, err) } diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index cbd1a26e..64903001 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -1,28 +1,65 @@ package telemetry import ( + "container/heap" + "context" + "encoding/json" "errors" "fmt" + "io" + "math" "os" "path/filepath" + "sort" + "strings" + "sync" - "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/arrow-go/v18/arrow/array" - "github.com/apache/arrow-go/v18/arrow/memory" - "github.com/apache/arrow-go/v18/parquet" - "github.com/apache/arrow-go/v18/parquet/compress" - "github.com/apache/arrow-go/v18/parquet/pqarrow" + "github.com/parquet-go/parquet-go" + "github.com/parquet-go/parquet-go/compress/zstd" + "github.com/zeebo/xxh3" ) -type parquetColumn[T any] struct { - name string - typeInfo arrow.DataType - nullable bool - value func(T) any +const ( + BatchSuffix = ".batch" + SchemaBatch = "_schema" + BatchSuffix + batchMetadataVersion = 1 + parquetPageSize = 64 << 10 + parquetRowGroupRows = 50_000 + maxTraceQueryResults = 500 +) + +type BatchMetadata struct { + Version uint32 `json:"version"` + ID string `json:"id"` + MinIngestedNanos int64 `json:"min_ingested_nanos"` + MaxIngestedNanos int64 `json:"max_ingested_nanos"` + Generation uint32 `json:"generation"` + Spans int `json:"spans"` + Logs int `json:"logs"` + Metrics int `json:"metrics"` +} + +type TraceQuery struct { + TraceID string + Namespace string + StartNanos int64 + EndNanos int64 + Limit int +} + +type storedBatch struct { + metadata BatchMetadata + dir string + traces traceIndex } type ParquetStore struct { - dir string + dir string + batchesDir string + stagingDir string + mu sync.RWMutex + publishMu sync.Mutex + batches map[string]*storedBatch } type ParquetStats struct { @@ -31,183 +68,642 @@ type ParquetStats struct { } func OpenParquetStore(dir string) (*ParquetStore, error) { - for _, signal := range []string{"spans", "logs", "metrics"} { - if err := os.MkdirAll(filepath.Join(dir, signal), 0o755); err != nil { - return nil, fmt.Errorf("create parquet %s directory: %w", signal, err) + p := &ParquetStore{ + dir: dir, batchesDir: filepath.Join(dir, "batches"), stagingDir: filepath.Join(dir, "staging"), + batches: make(map[string]*storedBatch), + } + for _, path := range []string{p.dir, p.batchesDir} { + if err := os.MkdirAll(path, 0o755); err != nil { + return nil, err } } - store := &ParquetStore{dir: dir} - if err := writeParquet(filepath.Join(dir, "spans", "_schema.parquet"), spanParquetColumns(), []Span{}); err != nil { - return nil, fmt.Errorf("create span parquet schema: %w", err) + // Staging is never acknowledged or queried. Removing it is the complete + // recovery protocol for writes interrupted before atomic publication. + if err := os.RemoveAll(p.stagingDir); err != nil { + return nil, fmt.Errorf("clear incomplete Parquet batches: %w", err) } - if err := writeParquet(filepath.Join(dir, "logs", "_schema.parquet"), logParquetColumns(), []Log{}); err != nil { - return nil, fmt.Errorf("create log parquet schema: %w", err) + if err := os.Mkdir(p.stagingDir, 0o755); err != nil { + return nil, err } - if err := writeParquet(filepath.Join(dir, "metrics", "_schema.parquet"), metricParquetColumns(), []Metric{}); err != nil { - return nil, fmt.Errorf("create metric parquet schema: %w", err) + if err := p.ensureSchemaBatch(); err != nil { + return nil, err } - return store, nil + if err := p.loadBatches(); err != nil { + return nil, err + } + return p, nil } -func (p *ParquetStore) Dir() string { return p.dir } +func (p *ParquetStore) Close() error { return nil } +func (p *ParquetStore) Dir() string { return p.dir } +func (p *ParquetStore) BatchesDir() string { return p.batchesDir } -// Stats reports the current immutable-file footprint for each signal. -func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { - stats := make(map[string]ParquetStats, 3) - for _, signal := range []string{"spans", "logs", "metrics"} { - entries, err := os.ReadDir(filepath.Join(p.dir, signal)) +func (p *ParquetStore) Pattern(signal string) string { + return filepath.ToSlash(filepath.Join(p.batchesDir, "*"+BatchSuffix, signal+".parquet")) +} + +func (p *ParquetStore) BatchPath(id string) string { + return filepath.Join(p.batchesDir, id+BatchSuffix) +} + +func (p *ParquetStore) StagingPath(id string) string { + return filepath.Join(p.stagingDir, id) +} + +func (p *ParquetStore) BatchMetadata() []BatchMetadata { + p.mu.RLock() + defer p.mu.RUnlock() + out := make([]BatchMetadata, 0, len(p.batches)) + for _, batch := range p.batches { + out = append(out, batch.metadata) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Generation != out[j].Generation { + return out[i].Generation < out[j].Generation + } + if out[i].MinIngestedNanos != out[j].MinIngestedNanos { + return out[i].MinIngestedNanos < out[j].MinIngestedNanos + } + return out[i].ID < out[j].ID + }) + return out +} + +func (p *ParquetStore) RowCount() uint64 { + p.mu.RLock() + defer p.mu.RUnlock() + var count uint64 + for _, batch := range p.batches { + count += uint64(batch.metadata.Spans + batch.metadata.Logs + batch.metadata.Metrics) + } + return count +} + +func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error { + if err := validateBatchID(metadata.ID); err != nil { + return err + } + metadata.Version = batchMetadataVersion + metadata.Spans, metadata.Logs, metadata.Metrics = len(spans), len(logs), len(metrics) + final := p.BatchPath(metadata.ID) + if p.hasBatch(metadata.ID) { + return nil + } + if info, err := os.Stat(final); err == nil && info.IsDir() { + if err := syncDirectory(p.batchesDir); err != nil { + return err + } + return p.registerBatch(final) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + stage := filepath.Join(p.stagingDir, metadata.ID) + if err := os.RemoveAll(stage); err != nil { + return err + } + if err := os.Mkdir(stage, 0o755); err != nil { + return err + } + complete := false + defer func() { + if !complete { + _ = os.RemoveAll(stage) + } + }() + + if len(spans) > 0 { + rows := make([]spanParquetRow, len(spans)) + for i := range spans { + rows[i] = makeSpanParquetRow(spans[i]) + rows[i].TraceHash = xxh3.HashString(rows[i].TraceID) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].TraceHash != rows[j].TraceHash { + return rows[i].TraceHash < rows[j].TraceHash + } + if rows[i].StartUnixNano != rows[j].StartUnixNano { + return rows[i].StartUnixNano < rows[j].StartUnixNano + } + return rows[i].SpanID < rows[j].SpanID + }) + if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), rows, parquetPageSize); err != nil { + return fmt.Errorf("write span Parquet: %w", err) + } + if err := writeTraceIndex(filepath.Join(stage, "trace.fidx"), rows); err != nil { + return fmt.Errorf("write trace index: %w", err) + } + } + if len(logs) > 0 { + rows := make([]logParquetRow, len(logs)) + for i := range logs { + rows[i] = makeLogParquetRow(logs[i]) + } + if err := writeTypedParquet(filepath.Join(stage, "logs.parquet"), rows, parquetPageSize); err != nil { + return fmt.Errorf("write log Parquet: %w", err) + } + } + if len(metrics) > 0 { + rows := make([]metricParquetRow, len(metrics)) + for i := range metrics { + rows[i] = makeMetricParquetRow(metrics[i]) + } + if err := writeTypedParquet(filepath.Join(stage, "metrics.parquet"), rows, parquetPageSize); err != nil { + return fmt.Errorf("write metric Parquet: %w", err) + } + } + if err := writeJSONFile(filepath.Join(stage, "metadata.json"), metadata); err != nil { + return err + } + if err := syncDirectory(stage); err != nil { + return err + } + + p.publishMu.Lock() + defer p.publishMu.Unlock() + if p.hasBatch(metadata.ID) { + complete = true + return os.RemoveAll(stage) + } + if err := os.Rename(stage, final); err != nil { + if info, statErr := os.Stat(final); statErr == nil && info.IsDir() { + complete = true + if err := syncDirectory(p.batchesDir); err != nil { + return err + } + return p.registerBatch(final) + } + return fmt.Errorf("publish Parquet batch: %w", err) + } + complete = true + if err := syncDirectory(p.batchesDir); err != nil { + return err + } + return p.registerBatch(final) +} + +// CleanupRetired removes inputs hidden by a completed retention or compaction +// publication. It is called only after compaction recovery has consumed its +// durable marker, so no rollback can still need these directories. +func (p *ParquetStore) CleanupRetired() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() + entries, err := os.ReadDir(p.batchesDir) + if err != nil { + return err + } + removed := false + var cleanupErr error + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() || strings.HasSuffix(name, BatchSuffix) || !strings.Contains(name, ".retired") { + continue + } + if err := os.RemoveAll(filepath.Join(p.batchesDir, name)); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } else { + removed = true + } + } + if removed { + cleanupErr = errors.Join(cleanupErr, syncDirectory(p.batchesDir)) + } + return cleanupErr +} + +// Trace reads only ranges selected by the persistent hash index. Scope filters +// and the limit are applied while decoding so a pathological trace cannot grow +// request memory without bound. +func (p *ParquetStore) Trace(ctx context.Context, query TraceQuery) ([]IndexedSpan, error) { + if query.TraceID == "" { + return nil, nil + } + if query.Limit <= 0 { + return nil, errors.New("trace query limit must be positive") + } + if query.Limit > maxTraceQueryResults { + return nil, fmt.Errorf("trace query limit exceeds %d", maxTraceQueryResults) + } + if query.StartNanos >= query.EndNanos { + return nil, errors.New("trace query time range must be positive") + } + hash := xxh3.HashString(query.TraceID) + p.mu.RLock() + defer p.mu.RUnlock() + selected := make(indexedSpanHeap, 0, query.Limit) + for _, batch := range p.batches { + if err := ctx.Err(); err != nil { + return nil, err + } + match, found, err := batch.traces.Lookup(hash) if err != nil { return nil, err } - var signalStats ParquetStats - for _, entry := range entries { - if entry.IsDir() || filepath.Ext(entry.Name()) != ".parquet" || entry.Name() == "_schema.parquet" { + if !found { + continue + } + if err := readIndexedTrace(ctx, batch, match, query, &selected); err != nil { + return nil, err + } + } + out := []IndexedSpan(selected) + sort.Slice(out, func(i, j int) bool { + return indexedSpanEarlier(out[i], out[j]) + }) + return out, nil +} + +func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, query TraceQuery, selected *indexedSpanHeap) (err error) { + if match.row > uint64(math.MaxInt64) { + return errors.New("trace index row exceeds Parquet reader limit") + } + file, err := os.Open(filepath.Join(batch.dir, "spans.parquet")) + if err != nil { + return err + } + reader := parquet.NewGenericReader[indexedSpanParquetRow](file) + defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() + if err := reader.SeekToRow(int64(match.row)); err != nil { + return err + } + remaining := match.count + buffer := make([]indexedSpanParquetRow, min(uint64(8192), remaining)) + for remaining > 0 { + if err := ctx.Err(); err != nil { + return err + } + want := min(uint64(len(buffer)), remaining) + n, readErr := reader.Read(buffer[:int(want)]) + if readErr != nil && !errors.Is(readErr, io.EOF) { + return readErr + } + if n == 0 { + return io.ErrUnexpectedEOF + } + for i := range n { + row := buffer[i] + if row.StartUnixNano >= query.EndNanos { + return nil + } + if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || + (query.Namespace != "" && row.Namespace != query.Namespace) { + continue + } + selected.Add(row.span(), query.Limit) + } + remaining -= uint64(n) + } + return nil +} + +type indexedSpanHeap []IndexedSpan + +func (h indexedSpanHeap) Len() int { return len(h) } +func (h indexedSpanHeap) Less(i, j int) bool { + return indexedSpanEarlier(h[j], h[i]) +} +func (h indexedSpanHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *indexedSpanHeap) Push(value any) { + *h = append(*h, value.(IndexedSpan)) +} +func (h *indexedSpanHeap) Pop() any { + old := *h + value := old[len(old)-1] + *h = old[:len(old)-1] + return value +} + +func (h *indexedSpanHeap) Add(span IndexedSpan, limit int) { + if len(*h) < limit { + heap.Push(h, span) + return + } + if indexedSpanEarlier(span, (*h)[0]) { + (*h)[0] = span + heap.Fix(h, 0) + } +} + +func indexedSpanEarlier(left, right IndexedSpan) bool { + if left.StartUnixNanos != right.StartUnixNanos { + return left.StartUnixNanos < right.StartUnixNanos + } + if left.DurationMS != right.DurationMS { + return left.DurationMS > right.DurationMS + } + return left.SpanID < right.SpanID +} + +// PruneBefore hides complete batches before deleting them. The caller pins +// DuckDB readers while this method runs. +func (p *ParquetStore) PruneBefore(cutoff int64) (int, error) { + p.publishMu.Lock() + defer p.publishMu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() + var retired []string + var pruneErr error + for id, batch := range p.batches { + if batch.metadata.MaxIngestedNanos <= 0 || batch.metadata.MaxIngestedNanos >= cutoff { + continue + } + path := filepath.Join(p.batchesDir, id+".retired") + if err := os.Rename(batch.dir, path); err != nil { + pruneErr = errors.Join(pruneErr, err) + continue + } + delete(p.batches, id) + retired = append(retired, path) + } + if len(retired) > 0 { + pruneErr = errors.Join(pruneErr, syncDirectory(p.batchesDir)) + } + for _, path := range retired { + pruneErr = errors.Join(pruneErr, os.RemoveAll(path)) + } + if len(retired) > 0 { + pruneErr = errors.Join(pruneErr, syncDirectory(p.batchesDir)) + } + return len(retired), pruneErr +} + +func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { + p.mu.RLock() + defer p.mu.RUnlock() + stats := map[string]ParquetStats{"spans": {}, "logs": {}, "metrics": {}} + for _, batch := range p.batches { + for signal := range stats { + info, err := os.Stat(filepath.Join(batch.dir, signal+".parquet")) + if errors.Is(err, os.ErrNotExist) { continue } - info, err := entry.Info() if err != nil { return nil, err } - signalStats.Files++ - signalStats.Bytes += info.Size() + value := stats[signal] + value.Files++ + value.Bytes += info.Size() + stats[signal] = value } - stats[signal] = signalStats } return stats, nil } -// StageBatch writes durable files that DuckDB's *.parquet views cannot see. -// Publication is a separate, rename-only step so encoding and fsync never hold -// the query read gate. -func (p *ParquetStore) StageBatch(id string, spans []Span, logs []Log, metrics []Metric) error { - for _, item := range []struct { - signal string - write func(string) error - }{ - {"spans", func(path string) error { return writeParquet(path, spanParquetColumns(), spans) }}, - {"logs", func(path string) error { return writeParquet(path, logParquetColumns(), logs) }}, - {"metrics", func(path string) error { return writeParquet(path, metricParquetColumns(), metrics) }}, - } { - rows := len(spans) - if item.signal == "logs" { - rows = len(logs) - } else if item.signal == "metrics" { - rows = len(metrics) - } - if rows == 0 { - continue +// PrepareReplacement completes metadata and the trace sidecar for Parquet +// files produced by the compactor. +func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) error { + metadata.Version = batchMetadataVersion + if metadata.Spans > 0 { + f, err := os.Open(filepath.Join(dir, "spans.parquet")) + if err != nil { + return err } - final := filepath.Join(p.dir, item.signal, id+".parquet") - if _, err := os.Stat(final); err == nil { - continue - } else if !errors.Is(err, os.ErrNotExist) { + reader := parquet.NewGenericReader[traceParquetRow](f) + index, err := newTraceIndexWriter(filepath.Join(dir, "trace.fidx")) + if err != nil { + _ = reader.Close() + _ = f.Close() + return err + } + rows := 0 + buffer := make([]traceParquetRow, min(8192, metadata.Spans)) + for { + n, readErr := reader.Read(buffer) + for i := range n { + if err := index.Append(buffer[i].TraceHash); err != nil { + index.Abort() + _ = reader.Close() + _ = f.Close() + return err + } + } + rows += n + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + index.Abort() + _ = reader.Close() + _ = f.Close() + return readErr + } + if n == 0 { + index.Abort() + _ = reader.Close() + _ = f.Close() + return io.ErrNoProgress + } + } + if err := errors.Join(reader.Close(), f.Close()); err != nil { + index.Abort() return err } - if err := item.write(final + ".pending"); err != nil { - return fmt.Errorf("stage %s parquet: %w", item.signal, err) + if rows != metadata.Spans { + index.Abort() + return fmt.Errorf("compacted span count: got %d want %d", rows, metadata.Spans) + } + if err := index.Close(); err != nil { + return err } } - return nil + if err := writeJSONFile(filepath.Join(dir, "metadata.json"), metadata); err != nil { + return err + } + return syncDirectory(dir) } -// PublishBatch atomically exposes all present signal files to in-process -// readers when the caller holds the Parquet publication gate. The returned -// rollback is used if the hot span projection cannot be published afterward. -func (p *ParquetStore) PublishBatch(id string, hasSpans, hasLogs, hasMetrics bool) (func() error, error) { - present := []struct { - signal string - has bool - }{{"spans", hasSpans}, {"logs", hasLogs}, {"metrics", hasMetrics}} - renamed := make([]string, 0, len(present)) +// PublishReplacement swaps a prepared compacted batch for its inputs while +// holding the store lock, so native trace readers cannot observe removed files. +func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string) error { + p.publishMu.Lock() + defer p.publishMu.Unlock() + final := p.BatchPath(metadata.ID) + source := stage + if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { + source = final + } else if err != nil { + return err + } + replacement, err := loadStoredBatch(source) + if err != nil { + return err + } + replacement.dir = final + if replacement.metadata.Spans > 0 { + replacement.traces.path = filepath.Join(final, "trace.fidx") + } + + p.mu.Lock() + defer p.mu.Unlock() + retired := make([][2]string, 0, len(inputs)) rollback := func() error { var rollbackErr error - for i := len(renamed) - 1; i >= 0; i-- { - final := filepath.Join(p.dir, renamed[i], id+".parquet") - if err := os.Rename(final, final+".pending"); err != nil && !errors.Is(err, os.ErrNotExist) { - rollbackErr = errors.Join(rollbackErr, err) + if source == final { + if _, statErr := os.Stat(stage); errors.Is(statErr, os.ErrNotExist) { + rollbackErr = errors.Join(rollbackErr, os.Rename(final, stage)) } } - if len(renamed) > 0 { - rollbackErr = errors.Join(rollbackErr, syncParquetSignalDirectories(p.dir, renamed)) - } - return rollbackErr - } - for _, item := range present { - if !item.has { - continue + for i := len(retired) - 1; i >= 0; i-- { + rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) } - final := filepath.Join(p.dir, item.signal, id+".parquet") - if _, err := os.Stat(final); err == nil { - continue - } else if !errors.Is(err, os.ErrNotExist) { - return rollback, errors.Join(err, rollback()) + return errors.Join(rollbackErr, syncDirectory(p.batchesDir)) + } + for _, id := range inputs { + active := p.BatchPath(id) + retiredPath := filepath.Join(p.batchesDir, id+".retired-"+metadata.ID) + if _, statErr := os.Stat(active); statErr == nil { + if err := os.Rename(active, retiredPath); err != nil { + return errors.Join(err, rollback()) + } + retired = append(retired, [2]string{active, retiredPath}) + } else if !errors.Is(statErr, os.ErrNotExist) { + return errors.Join(statErr, rollback()) + } else if _, retiredErr := os.Stat(retiredPath); retiredErr == nil { + // Recovery can resume after inputs were retired but before the + // replacement or directory sync completed. + retired = append(retired, [2]string{active, retiredPath}) + } else if !errors.Is(retiredErr, os.ErrNotExist) { + return errors.Join(retiredErr, rollback()) } - if err := os.Rename(final+".pending", final); err != nil { - return rollback, errors.Join(err, rollback()) + } + if source != final { + if err := os.Rename(stage, final); err != nil { + return errors.Join(err, rollback()) } - renamed = append(renamed, item.signal) + source = final } - if err := syncParquetSignalDirectories(p.dir, renamed); err != nil { - return rollback, errors.Join(err, rollback()) + if err := syncDirectory(p.batchesDir); err != nil { + return errors.Join(err, rollback()) } - return rollback, nil + for _, id := range inputs { + delete(p.batches, id) + } + p.batches[metadata.ID] = replacement + var removeErr error + for _, pair := range retired { + removeErr = errors.Join(removeErr, os.RemoveAll(pair[1])) + } + return errors.Join(removeErr, syncDirectory(p.batchesDir)) } -// DiscardBatch removes invisible staging files for a batch that another commit -// or compaction has already consumed. -func (p *ParquetStore) DiscardBatch(id string) error { - var discardErr error - changed := make([]string, 0, 3) - for _, signal := range []string{"spans", "logs", "metrics"} { - path := filepath.Join(p.dir, signal, id+".parquet.pending") - if err := os.Remove(path); err == nil { - changed = append(changed, signal) - } else if !errors.Is(err, os.ErrNotExist) { - discardErr = errors.Join(discardErr, err) - } +func (p *ParquetStore) ensureSchemaBatch() error { + final := filepath.Join(p.batchesDir, SchemaBatch) + if info, err := os.Stat(final); err == nil && info.IsDir() { + return nil + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + stage := filepath.Join(p.stagingDir, "_schema") + if err := os.Mkdir(stage, 0o755); err != nil { + return err + } + if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), []spanParquetRow{}, parquetPageSize); err != nil { + return err + } + if err := writeTypedParquet(filepath.Join(stage, "logs.parquet"), []logParquetRow{}, parquetPageSize); err != nil { + return err + } + if err := writeTypedParquet(filepath.Join(stage, "metrics.parquet"), []metricParquetRow{}, parquetPageSize); err != nil { + return err + } + if err := syncDirectory(stage); err != nil { + return err + } + if err := os.Rename(stage, final); err != nil { + return err } - return errors.Join(discardErr, syncParquetSignalDirectories(p.dir, changed)) + return syncDirectory(p.batchesDir) } -func syncParquetSignalDirectories(root string, signals []string) error { - seen := make(map[string]struct{}, len(signals)) - var syncErr error - for _, signal := range signals { - if _, ok := seen[signal]; ok { +func (p *ParquetStore) loadBatches() error { + entries, err := os.ReadDir(p.batchesDir) + if err != nil { + return err + } + for _, entry := range entries { + if !entry.IsDir() || entry.Name() == SchemaBatch || !strings.HasSuffix(entry.Name(), BatchSuffix) { continue } - seen[signal] = struct{}{} - syncErr = errors.Join(syncErr, syncDirectory(filepath.Join(root, signal))) + if err := p.registerBatch(filepath.Join(p.batchesDir, entry.Name())); err != nil { + return err + } } - return syncErr + return nil } -func writeParquet[T any](path string, columns []parquetColumn[T], rows []T) error { - if _, err := os.Stat(path); err == nil { - return nil - } else if !errors.Is(err, os.ErrNotExist) { +func (p *ParquetStore) registerBatch(dir string) error { + batch, err := loadStoredBatch(dir) + if err != nil { return err } - fields := make([]arrow.Field, len(columns)) - for i, column := range columns { - fields[i] = arrow.Field{Name: column.name, Type: column.typeInfo, Nullable: column.nullable} + if filepath.Base(dir) != batch.metadata.ID+BatchSuffix { + return fmt.Errorf("parquet batch directory %q does not match metadata ID %q", filepath.Base(dir), batch.metadata.ID) } - schema := arrow.NewSchema(fields, nil) - builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) - defer builder.Release() - for _, row := range rows { - for i, column := range columns { - if err := appendArrowValue(builder.Field(i), column.value(row)); err != nil { - return fmt.Errorf("append parquet column %s: %w", column.name, err) - } + p.mu.Lock() + p.batches[batch.metadata.ID] = batch + p.mu.Unlock() + return nil +} + +func loadStoredBatch(dir string) (*storedBatch, error) { + metadata, err := readBatchMetadata(filepath.Join(dir, "metadata.json")) + if err != nil { + return nil, err + } + signals := [...]struct { + name string + count int + }{ + {name: "spans", count: metadata.Spans}, + {name: "logs", count: metadata.Logs}, + {name: "metrics", count: metadata.Metrics}, + } + for _, signal := range signals { + if signal.count == 0 { + continue + } + path := filepath.Join(dir, signal.name+".parquet") + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s Parquet is not a regular file", signal.name) + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + parquetFile, openErr := parquet.OpenFile(file, info.Size()) + closeErr := file.Close() + if err := errors.Join(openErr, closeErr); err != nil { + return nil, fmt.Errorf("open %s Parquet: %w", signal.name, err) + } + if rows := parquetFile.NumRows(); rows != int64(signal.count) { + return nil, fmt.Errorf("%s Parquet has %d rows; metadata declares %d", signal.name, rows, signal.count) + } + } + var traces traceIndex + if metadata.Spans > 0 { + traces, err = loadTraceIndex(filepath.Join(dir, "trace.fidx"), metadata.Spans) + if err != nil { + return nil, err } } - record := builder.NewRecordBatch() - defer record.Release() + return &storedBatch{metadata: metadata, dir: dir, traces: traces}, nil +} + +func (p *ParquetStore) hasBatch(id string) bool { + p.mu.RLock() + _, ok := p.batches[id] + p.mu.RUnlock() + return ok +} - tmp := path + ".tmp" - _ = os.Remove(tmp) - f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) +func writeTypedParquet[T any](path string, rows []T, pageSize int) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) if err != nil { return err } @@ -215,65 +711,75 @@ func writeParquet[T any](path string, columns []parquetColumn[T], rows []T) erro defer func() { _ = f.Close() if !ok { - _ = os.Remove(tmp) + _ = os.Remove(path) } }() - writer, err := pqarrow.NewFileWriter( - schema, - f, - parquet.NewWriterProperties(parquet.WithCompression(compress.Codecs.Zstd)), - pqarrow.NewArrowWriterProperties(pqarrow.WithStoreSchema()), - ) - if err != nil { - return err - } - if err := writer.Write(record); err != nil { + writer := parquet.NewGenericWriter[T](f, + parquet.Compression(&zstd.Codec{Level: zstd.SpeedFastest, Concurrency: 1}), + parquet.MaxRowsPerRowGroup(parquetRowGroupRows), parquet.PageBufferSize(pageSize)) + if _, err := writer.Write(rows); err != nil { _ = writer.Close() return err } if err := writer.Close(); err != nil { return err } - _ = f.Close() // pqarrow may already have closed the sink. - syncFile, err := os.OpenFile(tmp, os.O_RDWR, 0) - if err != nil { + if err := f.Sync(); err != nil { return err } - if err := syncFile.Sync(); err != nil { - _ = syncFile.Close() + if err := f.Close(); err != nil { return err } - if err := syncFile.Close(); err != nil { - return err + ok = true + return nil +} + +func readBatchMetadata(path string) (BatchMetadata, error) { + data, err := os.ReadFile(path) + if err != nil { + return BatchMetadata{}, err } - if err := os.Rename(tmp, path); err != nil { - return err + var metadata BatchMetadata + if err := json.Unmarshal(data, &metadata); err != nil { + return BatchMetadata{}, err + } + if metadata.Version != batchMetadataVersion { + return BatchMetadata{}, fmt.Errorf("unsupported batch metadata version %d", metadata.Version) } - if err := syncDirectory(filepath.Dir(path)); err != nil { + if err := validateBatchID(metadata.ID); err != nil { + return BatchMetadata{}, err + } + if metadata.Spans < 0 || metadata.Logs < 0 || metadata.Metrics < 0 { + return BatchMetadata{}, errors.New("parquet batch metadata has a negative row count") + } + if metadata.MinIngestedNanos > 0 && metadata.MaxIngestedNanos > 0 && metadata.MinIngestedNanos > metadata.MaxIngestedNanos { + return BatchMetadata{}, errors.New("parquet batch metadata has an inverted time range") + } + return metadata, nil +} + +func writeJSONFile(path string, value any) error { + data, err := json.Marshal(value) + if err != nil { return err } - ok = true - return nil + return writeBytesFile(path, data) } -func appendArrowValue(builder array.Builder, value any) error { - if value == nil { - builder.AppendNull() - return nil +func writeBytesFile(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err } - switch b := builder.(type) { - case *array.StringBuilder: - b.Append(value.(string)) - case *array.Int64Builder: - b.Append(value.(int64)) - case *array.Float64Builder: - b.Append(value.(float64)) - case *array.TimestampBuilder: - b.Append(arrow.Timestamp(value.(int64))) - default: - return fmt.Errorf("unsupported Arrow builder %T", builder) + if err := f.Sync(); err != nil { + _ = f.Close() + return err } - return nil + return f.Close() } func syncDirectory(dir string) error { @@ -285,131 +791,18 @@ func syncDirectory(dir string) error { return f.Sync() } -func text(v string) any { - if v == "" { - return nil +func validateBatchID(id string) error { + if id == "" || len(id) > 128 || id[0] == '.' || strings.ContainsAny(id, `/\\`) { + return fmt.Errorf("invalid telemetry batch ID %q", id) } - return v -} - -func jsonText(v []byte) any { - if len(v) == 0 { - return nil - } - return string(v) -} - -func nanos(primary, secondary, ingested int64) any { - for _, value := range []int64{primary, secondary, ingested} { - if value > 0 { - return value + for _, r := range id { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') { + return fmt.Errorf("invalid telemetry batch ID %q", id) } } return nil } -func optionalNanos(value int64) any { - if value <= 0 { - return nil - } - return value -} - -func optionalInt(value int64) any { - if value == 0 { - return nil - } - return value -} - -func spanParquetColumns() []parquetColumn[Span] { - s, i, f, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Float64, arrow.FixedWidthTypes.Timestamp_ns - return []parquetColumn[Span]{ - {"namespace", s, false, func(r Span) any { return r.Namespace }}, - {"trace_id", s, false, func(r Span) any { return r.TraceID }}, - {"span_id", s, false, func(r Span) any { return r.SpanID }}, - {"parent_span_id", s, true, func(r Span) any { return text(r.ParentSpanID) }}, - {"service", s, false, func(r Span) any { return r.ServiceName }}, - {"operation", s, false, func(r Span) any { return r.Name }}, - {"kind", s, false, func(r Span) any { return r.Kind }}, - {"start_time", ts, true, func(r Span) any { return nanos(r.StartUnixNanos, 0, r.IngestedAt) }}, - {"end_time", ts, true, func(r Span) any { return optionalNanos(r.EndUnixNanos) }}, - {"start_unix_nano", i, false, func(r Span) any { return r.StartUnixNanos }}, - {"end_unix_nano", i, false, func(r Span) any { return r.EndUnixNanos }}, - {"duration_ms", f, false, func(r Span) any { return r.DurationMS }}, - {"status", s, false, func(r Span) any { return r.StatusCode }}, - {"status_message", s, true, func(r Span) any { return text(r.StatusMsg) }}, - {"resource_json", s, true, func(r Span) any { return jsonText(r.ResourceJSON) }}, - {"attributes_json", s, true, func(r Span) any { return jsonText(r.AttributesJSON) }}, - {"events_json", s, true, func(r Span) any { return jsonText(r.EventsJSON) }}, - {"links_json", s, true, func(r Span) any { return jsonText(r.LinksJSON) }}, - {"trace_state", s, true, func(r Span) any { return text(r.TraceState) }}, - {"flags", i, false, func(r Span) any { return int64(r.Flags) }}, - {"scope_name", s, true, func(r Span) any { return text(r.ScopeName) }}, - {"scope_version", s, true, func(r Span) any { return text(r.ScopeVersion) }}, - {"ingested_at", ts, true, func(r Span) any { return optionalNanos(r.IngestedAt) }}, - {"ingested_unix_nano", i, false, func(r Span) any { return r.IngestedAt }}, - {"http_method", s, true, func(r Span) any { return text(r.HTTPMethod) }}, - {"http_status_code", s, true, func(r Span) any { return text(r.HTTPStatusCode) }}, - {"http_route", s, true, func(r Span) any { return text(r.HTTPRoute) }}, - {"db_system", s, true, func(r Span) any { return text(r.DBSystem) }}, - {"rpc_method", s, true, func(r Span) any { return text(r.RPCMethod) }}, - {"rpc_service", s, true, func(r Span) any { return text(r.RPCService) }}, - {"peer_service", s, true, func(r Span) any { return text(r.PeerService) }}, - {"service_version", s, true, func(r Span) any { return text(r.ServiceVersion) }}, - {"deployment_env", s, true, func(r Span) any { return text(r.DeploymentEnv) }}, - {"exception_type", s, true, func(r Span) any { return text(r.ExceptionType) }}, - {"exception_message", s, true, func(r Span) any { return text(r.ExceptionMessage) }}, - } -} - -func logParquetColumns() []parquetColumn[Log] { - s, i, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.FixedWidthTypes.Timestamp_ns - return []parquetColumn[Log]{ - {"namespace", s, false, func(r Log) any { return r.Namespace }}, - {"log_time", ts, true, func(r Log) any { return nanos(r.TimeUnixNanos, r.ObservedTimeNanos, r.IngestedAt) }}, - {"observed_time", ts, true, func(r Log) any { return nanos(r.ObservedTimeNanos, r.TimeUnixNanos, r.IngestedAt) }}, - {"time_unix_nano", i, false, func(r Log) any { return r.TimeUnixNanos }}, - {"observed_time_unix_nano", i, true, func(r Log) any { return optionalInt(r.ObservedTimeNanos) }}, - {"severity", s, false, func(r Log) any { return r.Severity }}, - {"severity_number", i, false, func(r Log) any { return int64(r.SeverityNumber) }}, - {"body", s, false, func(r Log) any { return r.Body }}, - {"service", s, true, func(r Log) any { return text(r.ServiceName) }}, - {"trace_id", s, true, func(r Log) any { return text(r.TraceID) }}, - {"span_id", s, true, func(r Log) any { return text(r.SpanID) }}, - {"flags", i, false, func(r Log) any { return int64(r.Flags) }}, - {"resource_json", s, true, func(r Log) any { return jsonText(r.ResourceJSON) }}, - {"attributes_json", s, true, func(r Log) any { return jsonText(r.AttributesJSON) }}, - {"scope_name", s, true, func(r Log) any { return text(r.ScopeName) }}, - {"scope_version", s, true, func(r Log) any { return text(r.ScopeVersion) }}, - {"ingested_at", ts, true, func(r Log) any { return optionalNanos(r.IngestedAt) }}, - {"ingested_unix_nano", i, false, func(r Log) any { return r.IngestedAt }}, - {"body_template", s, true, func(r Log) any { return text(r.BodyTemplate) }}, - } -} - -func metricParquetColumns() []parquetColumn[Metric] { - s, i, f, ts := arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Float64, arrow.FixedWidthTypes.Timestamp_ns - return []parquetColumn[Metric]{ - {"namespace", s, false, func(r Metric) any { return r.Namespace }}, - {"metric_time", ts, true, func(r Metric) any { return nanos(r.TimeUnixNanos, 0, r.IngestedAt) }}, - {"time_unix_nano", i, false, func(r Metric) any { return r.TimeUnixNanos }}, - {"name", s, false, func(r Metric) any { return r.Name }}, - {"description", s, true, func(r Metric) any { return text(r.Description) }}, - {"unit", s, true, func(r Metric) any { return text(r.Unit) }}, - {"metric_type", s, false, func(r Metric) any { return r.Type }}, - {"service", s, true, func(r Metric) any { return text(r.ServiceName) }}, - {"value", f, false, func(r Metric) any { return r.Value }}, - {"hist_bounds_json", s, true, func(r Metric) any { return jsonText(r.HistBoundsJSON) }}, - {"hist_counts_json", s, true, func(r Metric) any { return jsonText(r.HistCountsJSON) }}, - {"hist_count", i, true, func(r Metric) any { return optionalInt(r.HistCount) }}, - {"hist_sum", f, false, func(r Metric) any { return r.HistSum }}, - {"exemplars_json", s, true, func(r Metric) any { return jsonText(r.ExemplarsJSON) }}, - {"attributes_json", s, true, func(r Metric) any { return jsonText(r.AttributesJSON) }}, - {"resource_json", s, true, func(r Metric) any { return jsonText(r.ResourceJSON) }}, - {"scope_name", s, true, func(r Metric) any { return text(r.ScopeName) }}, - {"scope_version", s, true, func(r Metric) any { return text(r.ScopeVersion) }}, - {"ingested_at", ts, true, func(r Metric) any { return optionalNanos(r.IngestedAt) }}, - {"ingested_unix_nano", i, false, func(r Metric) any { return r.IngestedAt }}, - } +type traceParquetRow struct { + TraceHash uint64 `parquet:"_trace_hash"` } diff --git a/internal/telemetry/parquet_rows.go b/internal/telemetry/parquet_rows.go new file mode 100644 index 00000000..fe922756 --- /dev/null +++ b/internal/telemetry/parquet_rows.go @@ -0,0 +1,153 @@ +package telemetry + +type spanParquetRow struct { + Namespace string `parquet:"namespace"` + TraceID string `parquet:"trace_id"` + TraceHash uint64 `parquet:"_trace_hash"` + SpanID string `parquet:"span_id"` + ParentSpanID string `parquet:"parent_span_id"` + Service string `parquet:"service"` + Operation string `parquet:"operation"` + Kind string `parquet:"kind"` + StartTime int64 `parquet:"start_time,timestamp(nanosecond)"` + EndTime int64 `parquet:"end_time,timestamp(nanosecond)"` + StartUnixNano int64 `parquet:"start_unix_nano"` + EndUnixNano int64 `parquet:"end_unix_nano"` + DurationMS float64 `parquet:"duration_ms"` + Status string `parquet:"status"` + StatusMessage string `parquet:"status_message"` + ResourceJSON string `parquet:"resource_json,dict"` + AttributesJSON string `parquet:"attributes_json,dict"` + EventsJSON string `parquet:"events_json,dict"` + LinksJSON string `parquet:"links_json,dict"` + TraceState string `parquet:"trace_state,dict"` + Flags int64 `parquet:"flags"` + ScopeName string `parquet:"scope_name,dict"` + ScopeVersion string `parquet:"scope_version,dict"` + IngestedAt int64 `parquet:"ingested_at,timestamp(nanosecond)"` + IngestedUnixNano int64 `parquet:"ingested_unix_nano"` + HTTPMethod string `parquet:"http_method,dict"` + HTTPStatusCode string `parquet:"http_status_code,dict"` + HTTPRoute string `parquet:"http_route,dict"` + DBSystem string `parquet:"db_system,dict"` + RPCMethod string `parquet:"rpc_method,dict"` + RPCService string `parquet:"rpc_service,dict"` + PeerService string `parquet:"peer_service,dict"` + ServiceVersion string `parquet:"service_version,dict"` + DeploymentEnv string `parquet:"deployment_env,dict"` + ExceptionType string `parquet:"exception_type,dict"` + ExceptionMessage string `parquet:"exception_message,dict"` +} + +func makeSpanParquetRow(r Span) spanParquetRow { + return spanParquetRow{ + Namespace: r.Namespace, TraceID: r.TraceID, SpanID: r.SpanID, ParentSpanID: r.ParentSpanID, + Service: r.ServiceName, Operation: r.Name, Kind: r.Kind, StartTime: r.StartUnixNanos, + EndTime: r.EndUnixNanos, StartUnixNano: r.StartUnixNanos, EndUnixNano: r.EndUnixNanos, + DurationMS: r.DurationMS, Status: r.StatusCode, StatusMessage: r.StatusMsg, + ResourceJSON: string(r.ResourceJSON), AttributesJSON: string(r.AttributesJSON), EventsJSON: string(r.EventsJSON), LinksJSON: string(r.LinksJSON), + TraceState: r.TraceState, Flags: int64(r.Flags), ScopeName: r.ScopeName, ScopeVersion: r.ScopeVersion, + IngestedAt: r.IngestedAt, IngestedUnixNano: r.IngestedAt, HTTPMethod: r.HTTPMethod, + HTTPStatusCode: r.HTTPStatusCode, HTTPRoute: r.HTTPRoute, DBSystem: r.DBSystem, RPCMethod: r.RPCMethod, + RPCService: r.RPCService, PeerService: r.PeerService, ServiceVersion: r.ServiceVersion, + DeploymentEnv: r.DeploymentEnv, ExceptionType: r.ExceptionType, ExceptionMessage: r.ExceptionMessage, + } +} + +type indexedSpanParquetRow struct { + Namespace string `parquet:"namespace"` + TraceID string `parquet:"trace_id"` + SpanID string `parquet:"span_id"` + ParentSpanID string `parquet:"parent_span_id"` + Service string `parquet:"service"` + Operation string `parquet:"operation"` + Kind string `parquet:"kind"` + StartUnixNano int64 `parquet:"start_unix_nano"` + DurationMS float64 `parquet:"duration_ms"` + Status string `parquet:"status"` + StatusMessage string `parquet:"status_message"` +} + +func (r indexedSpanParquetRow) span() IndexedSpan { + return IndexedSpan{ + Namespace: r.Namespace, TraceID: r.TraceID, SpanID: r.SpanID, ParentSpanID: r.ParentSpanID, + ServiceName: r.Service, Name: r.Operation, Kind: r.Kind, StartUnixNanos: r.StartUnixNano, + DurationMS: r.DurationMS, StatusCode: r.Status, StatusMsg: r.StatusMessage, + } +} + +type logParquetRow struct { + Namespace string `parquet:"namespace,dict"` + LogTime int64 `parquet:"log_time,timestamp(nanosecond)"` + ObservedTime int64 `parquet:"observed_time,timestamp(nanosecond)"` + TimeUnixNano int64 `parquet:"time_unix_nano"` + ObservedTimeUnixNano int64 `parquet:"observed_time_unix_nano"` + Severity string `parquet:"severity,dict"` + SeverityNumber int64 `parquet:"severity_number"` + Body string `parquet:"body"` + Service string `parquet:"service,dict"` + TraceID string `parquet:"trace_id"` + SpanID string `parquet:"span_id"` + Flags int64 `parquet:"flags"` + ResourceJSON string `parquet:"resource_json,dict"` + AttributesJSON string `parquet:"attributes_json,dict"` + ScopeName string `parquet:"scope_name,dict"` + ScopeVersion string `parquet:"scope_version,dict"` + IngestedAt int64 `parquet:"ingested_at,timestamp(nanosecond)"` + IngestedUnixNano int64 `parquet:"ingested_unix_nano"` + BodyTemplate string `parquet:"body_template,dict"` +} + +func makeLogParquetRow(r Log) logParquetRow { + return logParquetRow{ + Namespace: r.Namespace, LogTime: firstPositive(r.TimeUnixNanos, r.ObservedTimeNanos, r.IngestedAt), + ObservedTime: firstPositive(r.ObservedTimeNanos, r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, + ObservedTimeUnixNano: r.ObservedTimeNanos, Severity: r.Severity, SeverityNumber: int64(r.SeverityNumber), + Body: r.Body, Service: r.ServiceName, TraceID: r.TraceID, SpanID: r.SpanID, Flags: int64(r.Flags), + ResourceJSON: string(r.ResourceJSON), AttributesJSON: string(r.AttributesJSON), ScopeName: r.ScopeName, + ScopeVersion: r.ScopeVersion, IngestedAt: r.IngestedAt, IngestedUnixNano: r.IngestedAt, BodyTemplate: r.BodyTemplate, + } +} + +type metricParquetRow struct { + Namespace string `parquet:"namespace,dict"` + MetricTime int64 `parquet:"metric_time,timestamp(nanosecond)"` + TimeUnixNano int64 `parquet:"time_unix_nano"` + Name string `parquet:"name,dict"` + Description string `parquet:"description,dict"` + Unit string `parquet:"unit,dict"` + MetricType string `parquet:"metric_type,dict"` + Service string `parquet:"service,dict"` + Value float64 `parquet:"value"` + HistBoundsJSON string `parquet:"hist_bounds_json,dict"` + HistCountsJSON string `parquet:"hist_counts_json,dict"` + HistCount int64 `parquet:"hist_count"` + HistSum float64 `parquet:"hist_sum"` + ExemplarsJSON string `parquet:"exemplars_json,dict"` + AttributesJSON string `parquet:"attributes_json,dict"` + ResourceJSON string `parquet:"resource_json,dict"` + ScopeName string `parquet:"scope_name,dict"` + ScopeVersion string `parquet:"scope_version,dict"` + IngestedAt int64 `parquet:"ingested_at,timestamp(nanosecond)"` + IngestedUnixNano int64 `parquet:"ingested_unix_nano"` +} + +func makeMetricParquetRow(r Metric) metricParquetRow { + return metricParquetRow{ + Namespace: r.Namespace, MetricTime: firstPositive(r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, + Name: r.Name, Description: r.Description, Unit: r.Unit, MetricType: r.Type, Service: r.ServiceName, + Value: r.Value, HistBoundsJSON: string(r.HistBoundsJSON), HistCountsJSON: string(r.HistCountsJSON), + HistCount: r.HistCount, HistSum: r.HistSum, ExemplarsJSON: string(r.ExemplarsJSON), + AttributesJSON: string(r.AttributesJSON), ResourceJSON: string(r.ResourceJSON), ScopeName: r.ScopeName, + ScopeVersion: r.ScopeVersion, IngestedAt: r.IngestedAt, IngestedUnixNano: r.IngestedAt, + } +} + +func firstPositive(values ...int64) int64 { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index 1306139d..aa6edf2e 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -1,56 +1,271 @@ package telemetry import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" "os" "path/filepath" + "reflect" "testing" + + "github.com/parquet-go/parquet-go" ) -func TestParquetBatchStagingIsInvisibleUntilPublish(t *testing.T) { - store, err := OpenParquetStore(t.TempDir()) +func TestParquetStorePublishesCompleteBatchAndRecovers(t *testing.T) { + dir := t.TempDir() + store, err := OpenParquetStore(dir) if err != nil { t.Fatal(err) } - const id = "batch-1" - if err := store.StageBatch(id, []Span{{Namespace: "default", TraceID: "trace", SpanID: "span"}}, []Log{{Namespace: "default", Body: "body"}}, []Metric{{Namespace: "default", Name: "metric"}}); err != nil { + span := completeTestSpan() + metadata := BatchMetadata{ID: "batch-1", MinIngestedNanos: span.IngestedAt, MaxIngestedNanos: span.IngestedAt} + if err := store.CommitBatch(metadata, []Span{span}, []Log{{Namespace: "tenant", Body: "ready", TimeUnixNanos: 12}}, []Metric{{Namespace: "tenant", Name: "requests", TimeUnixNanos: 13}}); err != nil { t.Fatal(err) } - for _, signal := range []string{"spans", "logs", "metrics"} { - final := filepath.Join(store.Dir(), signal, id+".parquet") - if _, err := os.Stat(final); !os.IsNotExist(err) { - t.Fatalf("%s final file visible before publish: %v", signal, err) - } - if _, err := os.Stat(final + ".pending"); err != nil { - t.Fatalf("%s pending file: %v", signal, err) + + batchDir := store.BatchPath(metadata.ID) + for _, name := range []string{"spans.parquet", "logs.parquet", "metrics.parquet", "trace.fidx", "metadata.json"} { + if _, err := os.Stat(filepath.Join(batchDir, name)); err != nil { + t.Fatalf("published batch is missing %s: %v", name, err) } } - rollback, err := store.PublishBatch(id, true, true, true) + if got := store.RowCount(); got != 3 { + t.Fatalf("row count = %d, want 3", got) + } + if err := store.CommitBatch(metadata, []Span{span}, nil, nil); err != nil { + t.Fatalf("idempotent commit: %v", err) + } + if got := store.RowCount(); got != 3 { + t.Fatalf("idempotent row count = %d, want 3", got) + } + + // An interrupted, unpublished directory is neither cataloged nor retained. + orphan := store.StagingPath("interrupted") + if err := os.Mkdir(orphan, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(orphan, "partial"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := OpenParquetStore(dir) if err != nil { t.Fatal(err) } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(store.Dir(), signal, id+".parquet")); err != nil { - t.Fatalf("%s final file: %v", signal, err) - } + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Fatalf("unpublished staging directory survived restart: %v", err) } - if err := rollback(); err != nil { + spans, err := traceAll(reopened, span.TraceID) + if err != nil { t.Fatal(err) } - for _, signal := range []string{"spans", "logs", "metrics"} { - final := filepath.Join(store.Dir(), signal, id+".parquet") - if _, err := os.Stat(final); !os.IsNotExist(err) { - t.Fatalf("%s final file visible after rollback: %v", signal, err) - } - if _, err := os.Stat(final + ".pending"); err != nil { - t.Fatalf("%s restored pending file: %v", signal, err) - } + wantIndexed := IndexedSpan{ + Namespace: span.Namespace, TraceID: span.TraceID, SpanID: span.SpanID, ParentSpanID: span.ParentSpanID, + ServiceName: span.ServiceName, Name: span.Name, Kind: span.Kind, StartUnixNanos: span.StartUnixNanos, + DurationMS: span.DurationMS, StatusCode: span.StatusCode, StatusMsg: span.StatusMsg, + } + if !reflect.DeepEqual(spans, []IndexedSpan{wantIndexed}) { + t.Fatalf("trace projection mismatch\n got: %#v\nwant: %#v", spans, []IndexedSpan{wantIndexed}) + } + gotSpan := readOneParquetRow[spanParquetRow](t, filepath.Join(batchDir, "spans.parquet")) + wantSpan := makeSpanParquetRow(span) + wantSpan.TraceHash = gotSpan.TraceHash + if !reflect.DeepEqual(gotSpan, wantSpan) { + t.Fatalf("complete span row mismatch\n got: %#v\nwant: %#v", gotSpan, wantSpan) + } +} + +func TestParquetStoreTraceUsesExactIDAndEventOrder(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + spans := []Span{ + {TraceID: "wanted", SpanID: "later", StartUnixNanos: 30}, + {TraceID: "other", SpanID: "other", StartUnixNanos: 20}, + {TraceID: "wanted", SpanID: "earlier", StartUnixNanos: 10}, + } + if err := store.CommitBatch(BatchMetadata{ID: "batch"}, spans, nil, nil); err != nil { + t.Fatal(err) + } + got, err := traceAll(store, "wanted") + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].SpanID != "earlier" || got[1].SpanID != "later" { + t.Fatalf("ordered trace = %#v", got) + } + missing, err := traceAll(store, "missing") + if err != nil || len(missing) != 0 { + t.Fatalf("missing trace = %#v, %v", missing, err) + } +} + +func TestParquetStoreTraceFiltersAndBoundsResultsDuringRead(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + spans := make([]Span, 0, 102) + for i := 100; i >= 1; i-- { + spans = append(spans, Span{ + Namespace: "prod", TraceID: "large-trace", SpanID: fmt.Sprintf("span-%03d", i), + StartUnixNanos: int64(i), + }) + } + spans = append(spans, + Span{Namespace: "staging", TraceID: "large-trace", SpanID: "wrong-namespace", StartUnixNanos: 11}, + Span{Namespace: "prod", TraceID: "large-trace", SpanID: "outside-window", StartUnixNanos: 200}, + ) + if err := store.CommitBatch(BatchMetadata{ID: "large"}, spans, nil, nil); err != nil { + t.Fatal(err) + } + + got, err := store.Trace(context.Background(), TraceQuery{ + TraceID: "large-trace", Namespace: "prod", StartNanos: 10, EndNanos: 90, Limit: 3, + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0].SpanID != "span-010" || got[1].SpanID != "span-011" || got[2].SpanID != "span-012" { + t.Fatalf("bounded trace = %#v", got) + } +} + +func TestParquetStorePreservesCompleteLogAndMetricRows(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + logRow := Log{ + Namespace: "tenant", EventUnixNanos: 11, TimeUnixNanos: 12, ObservedTimeNanos: 13, + Severity: "ERROR", SeverityNumber: 17, Body: "declined", ServiceName: "checkout", + TraceID: "trace", SpanID: "span", Flags: 1, ResourceJSON: []byte(`{"host":"one"}`), + AttributesJSON: []byte(`{"attempt":2}`), ScopeName: "scope", ScopeVersion: "1.2.3", + IngestedAt: 14, BodyTemplate: "declined: {reason}", + } + metricRow := Metric{ + Namespace: "tenant", EventUnixNanos: 21, TimeUnixNanos: 22, Name: "request.duration", + Description: "request latency", Unit: "ms", Type: "histogram", ServiceName: "checkout", Value: 23.5, + HistBoundsJSON: []byte(`[1,5,10]`), HistCountsJSON: []byte(`[2,3,4,5]`), HistCount: 14, HistSum: 47, + ExemplarsJSON: []byte(`[{"trace_id":"trace"}]`), AttributesJSON: []byte(`{"route":"/pay"}`), + ResourceJSON: []byte(`{"host":"one"}`), ScopeName: "scope", ScopeVersion: "1.2.3", IngestedAt: 24, + } + if err := store.CommitBatch(BatchMetadata{ID: "complete-signals"}, nil, []Log{logRow}, []Metric{metricRow}); err != nil { + t.Fatal(err) + } + batchDir := store.BatchPath("complete-signals") + gotLog := readOneParquetRow[logParquetRow](t, filepath.Join(batchDir, "logs.parquet")) + if want := makeLogParquetRow(logRow); !reflect.DeepEqual(gotLog, want) { + t.Fatalf("log row mismatch\n got: %#v\nwant: %#v", gotLog, want) } - if err := store.DiscardBatch(id); err != nil { + gotMetric := readOneParquetRow[metricParquetRow](t, filepath.Join(batchDir, "metrics.parquet")) + if want := makeMetricParquetRow(metricRow); !reflect.DeepEqual(gotMetric, want) { + t.Fatalf("metric row mismatch\n got: %#v\nwant: %#v", gotMetric, want) + } +} + +func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { t.Fatal(err) } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(store.Dir(), signal, id+".parquet.pending")); !os.IsNotExist(err) { - t.Fatalf("%s pending file remained after discard: %v", signal, err) + for _, id := range []string{"", ".hidden", "../escape", "has space"} { + if err := store.CommitBatch(BatchMetadata{ID: id}, []Span{{TraceID: "t"}}, nil, nil); err == nil { + t.Fatalf("CommitBatch accepted unsafe ID %q", id) } } } + +func TestParquetStoreCleansOnlyRetiredDirectories(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + retired := filepath.Join(store.BatchesDir(), "old.retired-compacted") + if err := os.Mkdir(retired, 0o755); err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(BatchMetadata{ID: "contains.retired"}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + t.Fatal(err) + } + if err := store.CleanupRetired(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(retired); !os.IsNotExist(err) { + t.Fatalf("retired directory remains: %v", err) + } + if _, err := os.Stat(store.BatchPath("contains.retired")); err != nil { + t.Fatalf("published batch with retired in its ID was removed: %v", err) + } +} + +func TestParquetStoreRejectsMetadataRowCountMismatchOnOpen(t *testing.T) { + dir := t.TempDir() + store, err := OpenParquetStore(dir) + if err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(BatchMetadata{ID: "mismatch"}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + t.Fatal(err) + } + metadataPath := filepath.Join(store.BatchPath("mismatch"), "metadata.json") + metadata, err := readBatchMetadata(metadataPath) + if err != nil { + t.Fatal(err) + } + metadata.Spans++ + data, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(metadataPath, data, 0o644); err != nil { + t.Fatal(err) + } + if _, err := OpenParquetStore(dir); err == nil { + t.Fatal("OpenParquetStore accepted metadata with the wrong row count") + } +} + +func completeTestSpan() Span { + return Span{ + Namespace: "tenant", TraceID: "0123456789abcdef0123456789abcdef", SpanID: "0123456789abcdef", + ParentSpanID: "fedcba9876543210", ServiceName: "checkout", Name: "POST /orders", Kind: "SERVER", + StartUnixNanos: 10, EndUnixNanos: 20, DurationMS: 0.00001, StatusCode: "ERROR", StatusMsg: "declined", + ResourceJSON: []byte(`{"host":"one"}`), AttributesJSON: []byte(`{"http.request.method":"POST"}`), + EventsJSON: []byte(`[{"name":"exception"}]`), LinksJSON: []byte(`[{"trace_id":"linked"}]`), + TraceState: "vendor=value", Flags: 1, ScopeName: "scope", ScopeVersion: "1.2.3", IngestedAt: 30, + HTTPMethod: "POST", HTTPStatusCode: "500", HTTPRoute: "/orders", DBSystem: "postgresql", + RPCMethod: "Create", RPCService: "orders.v1.Orders", PeerService: "payments", ServiceVersion: "4.5.6", + DeploymentEnv: "production", ExceptionType: "CardDeclined", ExceptionMessage: "declined", + } +} + +func traceAll(store *ParquetStore, traceID string) ([]IndexedSpan, error) { + return store.Trace(context.Background(), TraceQuery{ + TraceID: traceID, StartNanos: math.MinInt64, EndNanos: math.MaxInt64, Limit: maxTraceQueryResults, + }) +} + +func readOneParquetRow[T any](t *testing.T, path string) T { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + reader := parquet.NewGenericReader[T](file) + defer reader.Close() + var rows [1]T + if n, err := reader.Read(rows[:]); err != nil && !errors.Is(err, io.EOF) || n != 1 { + t.Fatalf("read %s: rows=%d err=%v", path, n, err) + } + return rows[0] +} diff --git a/internal/telemetry/rows.go b/internal/telemetry/rows.go index 606503f5..1b403f66 100644 --- a/internal/telemetry/rows.go +++ b/internal/telemetry/rows.go @@ -37,6 +37,23 @@ type Span struct { ExceptionMessage string } +// IndexedSpan is the narrow projection needed by trace-detail queries. The +// complete authoritative span remains in Parquet and is available through SQL; +// keeping this narrow projection small avoids decoding large JSON columns. +type IndexedSpan struct { + Namespace string + TraceID string + SpanID string + ParentSpanID string + ServiceName string + Name string + Kind string + StartUnixNanos int64 + DurationMS float64 + StatusCode string + StatusMsg string +} + type Log struct { Namespace string EventUnixNanos int64 diff --git a/internal/telemetry/segment/span_columnar.go b/internal/telemetry/segment/span_columnar.go deleted file mode 100644 index 9ae82b09..00000000 --- a/internal/telemetry/segment/span_columnar.go +++ /dev/null @@ -1,316 +0,0 @@ -// Column codecs are deliberately private to the versioned segment format. -package segment - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "math" - - "github.com/klauspost/compress/zstd" -) - -const ( - colNamespace = iota - colTraceID - colSpanID - colParentSpanID - colServiceName - colName - colKind - colStartUnixNanos - colEndUnixNanos - colDurationMS - colStatusCode - colStatusMsg - colResourceJSON - colAttributesJSON - colEventsJSON - colLinksJSON - colTraceState - colFlags - colScopeName - colScopeVersion - colIngestedAt - colHTTPMethod - colHTTPStatusCode - colHTTPRoute - colDBSystem - colRPCMethod - colRPCService - colPeerService - colServiceVersion - colDeploymentEnv - colExceptionType - colExceptionMessage - columnCount -) - -const columnarHeaderSize = 4 + columnCount*8 - -var allColumns = func() []int { - out := make([]int, columnCount) - for i := range out { - out[i] = i - } - return out -}() - -func encodeColumnarBlock(enc *zstd.Encoder, rows []Span) ([]byte, error) { - columns := make([][]byte, columnCount) - for _, row := range rows { - columns[colNamespace] = appendString(columns[colNamespace], row.Namespace) - columns[colTraceID] = appendString(columns[colTraceID], row.TraceID) - columns[colSpanID] = appendString(columns[colSpanID], row.SpanID) - columns[colParentSpanID] = appendString(columns[colParentSpanID], row.ParentSpanID) - columns[colServiceName] = appendString(columns[colServiceName], row.ServiceName) - columns[colName] = appendString(columns[colName], row.Name) - columns[colKind] = appendString(columns[colKind], row.Kind) - columns[colStartUnixNanos] = binary.LittleEndian.AppendUint64(columns[colStartUnixNanos], uint64(row.StartUnixNanos)) - columns[colEndUnixNanos] = binary.LittleEndian.AppendUint64(columns[colEndUnixNanos], uint64(row.EndUnixNanos)) - columns[colDurationMS] = binary.LittleEndian.AppendUint64(columns[colDurationMS], math.Float64bits(row.DurationMS)) - columns[colStatusCode] = appendString(columns[colStatusCode], row.StatusCode) - columns[colStatusMsg] = appendString(columns[colStatusMsg], row.StatusMsg) - columns[colResourceJSON] = appendBytes(columns[colResourceJSON], row.ResourceJSON) - columns[colAttributesJSON] = appendBytes(columns[colAttributesJSON], row.AttributesJSON) - columns[colEventsJSON] = appendBytes(columns[colEventsJSON], row.EventsJSON) - columns[colLinksJSON] = appendBytes(columns[colLinksJSON], row.LinksJSON) - columns[colTraceState] = appendString(columns[colTraceState], row.TraceState) - columns[colFlags] = binary.LittleEndian.AppendUint32(columns[colFlags], row.Flags) - columns[colScopeName] = appendString(columns[colScopeName], row.ScopeName) - columns[colScopeVersion] = appendString(columns[colScopeVersion], row.ScopeVersion) - columns[colIngestedAt] = binary.LittleEndian.AppendUint64(columns[colIngestedAt], uint64(row.IngestedAt)) - columns[colHTTPMethod] = appendString(columns[colHTTPMethod], row.HTTPMethod) - columns[colHTTPStatusCode] = appendString(columns[colHTTPStatusCode], row.HTTPStatusCode) - columns[colHTTPRoute] = appendString(columns[colHTTPRoute], row.HTTPRoute) - columns[colDBSystem] = appendString(columns[colDBSystem], row.DBSystem) - columns[colRPCMethod] = appendString(columns[colRPCMethod], row.RPCMethod) - columns[colRPCService] = appendString(columns[colRPCService], row.RPCService) - columns[colPeerService] = appendString(columns[colPeerService], row.PeerService) - columns[colServiceVersion] = appendString(columns[colServiceVersion], row.ServiceVersion) - columns[colDeploymentEnv] = appendString(columns[colDeploymentEnv], row.DeploymentEnv) - columns[colExceptionType] = appendString(columns[colExceptionType], row.ExceptionType) - columns[colExceptionMessage] = appendString(columns[colExceptionMessage], row.ExceptionMessage) - } - - out := make([]byte, columnarHeaderSize) - binary.LittleEndian.PutUint32(out[0:4], columnCount) - offset := columnarHeaderSize - for id, plain := range columns { - if len(plain) > segmentDecoderMaxMemory { - return nil, fmt.Errorf("column %d exceeds decoder memory limit", id) - } - compressed := enc.EncodeAll(plain, nil) - if len(compressed) > math.MaxUint32 || offset > math.MaxUint32-len(compressed) { - return nil, fmt.Errorf("column %d exceeds segment extent limit", id) - } - entry := out[4+id*8:] - binary.LittleEndian.PutUint32(entry[0:4], uint32(offset)) - binary.LittleEndian.PutUint32(entry[4:8], uint32(len(compressed))) - out = append(out, compressed...) - offset += len(compressed) - } - if len(out) > segmentMaxCompressedBytes { - return nil, fmt.Errorf("encoded block uses %d bytes; maximum is %d", len(out), segmentMaxCompressedBytes) - } - return out, nil -} - -// spanColumnPlainSizes mirrors encodeColumnarBlock without allocating. The WAL -// validator uses it to guarantee that every acknowledged block fits the same -// per-column and aggregate budgets enforced by the decoder. -func spanColumnPlainSizes(rows []Span) [columnCount]uint64 { - var sizes [columnCount]uint64 - framed := func(valueLen int) uint64 { - var scratch [binary.MaxVarintLen64]byte - return uint64(binary.PutUvarint(scratch[:], uint64(valueLen)) + valueLen) - } - for _, row := range rows { - for id, value := range []string{ - row.Namespace, row.TraceID, row.SpanID, row.ParentSpanID, - row.ServiceName, row.Name, row.Kind, - } { - sizes[id] += framed(len(value)) - } - sizes[colStartUnixNanos] += 8 - sizes[colEndUnixNanos] += 8 - sizes[colDurationMS] += 8 - for offset, value := range []string{row.StatusCode, row.StatusMsg} { - sizes[colStatusCode+offset] += framed(len(value)) - } - for offset, value := range [][]byte{row.ResourceJSON, row.AttributesJSON, row.EventsJSON, row.LinksJSON} { - sizes[colResourceJSON+offset] += framed(len(value)) - } - sizes[colTraceState] += framed(len(row.TraceState)) - sizes[colFlags] += 4 - sizes[colScopeName] += framed(len(row.ScopeName)) - sizes[colScopeVersion] += framed(len(row.ScopeVersion)) - sizes[colIngestedAt] += 8 - for offset, value := range []string{ - row.HTTPMethod, row.HTTPStatusCode, row.HTTPRoute, row.DBSystem, - row.RPCMethod, row.RPCService, row.PeerService, row.ServiceVersion, - row.DeploymentEnv, row.ExceptionType, row.ExceptionMessage, - } { - sizes[colHTTPMethod+offset] += framed(len(value)) - } - } - return sizes -} - -func decodeColumns(dec *zstd.Decoder, block []byte, wanted []int) (map[int][]byte, error) { - if len(block) < columnarHeaderSize { - return nil, io.ErrUnexpectedEOF - } - if got := binary.LittleEndian.Uint32(block[0:4]); got != columnCount { - return nil, fmt.Errorf("column count: got %d want %d", got, columnCount) - } - out := make(map[int][]byte, len(wanted)) - for _, id := range wanted { - if id < 0 || id >= columnCount { - return nil, fmt.Errorf("column %d out of range", id) - } - entry := block[4+id*8:] - offset := int(binary.LittleEndian.Uint32(entry[0:4])) - length := int(binary.LittleEndian.Uint32(entry[4:8])) - if offset < columnarHeaderSize || length < 0 || offset > len(block)-length { - return nil, fmt.Errorf("column %d has invalid extent %d+%d", id, offset, length) - } - plain, err := dec.DecodeAll(block[offset:offset+length], nil) - if err != nil { - return nil, fmt.Errorf("decompress column %d: %w", id, err) - } - out[id] = plain - } - return out, nil -} - -func decodeSelectedBlock(columns map[int][]byte, count int, selected []int) ([]Span, error) { - if len(selected) == 0 { - return nil, nil - } - for i, row := range selected { - if row < 0 || row >= count || (i > 0 && selected[i-1] >= row) { - return nil, errors.New("selected rows must be sorted, unique, and in range") - } - } - stringColumns := []int{ - colNamespace, colTraceID, colSpanID, colParentSpanID, colServiceName, - colName, colKind, colStatusCode, colStatusMsg, colTraceState, - colScopeName, colScopeVersion, colHTTPMethod, colHTTPStatusCode, - colHTTPRoute, colDBSystem, colRPCMethod, colRPCService, colPeerService, - colServiceVersion, colDeploymentEnv, colExceptionType, colExceptionMessage, - } - stringsByColumn := make(map[int][]string, len(stringColumns)) - for _, id := range stringColumns { - values, err := selectStrings(columns[id], count, selected) - if err != nil { - return nil, fmt.Errorf("select string column %d: %w", id, err) - } - stringsByColumn[id] = values - } - bytesByColumn := make(map[int][][]byte, 4) - for _, id := range []int{colResourceJSON, colAttributesJSON, colEventsJSON, colLinksJSON} { - values, err := selectBytes(columns[id], count, selected) - if err != nil { - return nil, fmt.Errorf("select bytes column %d: %w", id, err) - } - bytesByColumn[id] = values - } - for _, fixed := range []struct{ id, width int }{{colStartUnixNanos, 8}, {colEndUnixNanos, 8}, {colDurationMS, 8}, {colFlags, 4}, {colIngestedAt, 8}} { - if err := requireFixed(columns[fixed.id], count, fixed.width); err != nil { - return nil, err - } - } - rows := make([]Span, len(selected)) - for i, sourceRow := range selected { - rows[i] = Span{ - Namespace: stringsByColumn[colNamespace][i], TraceID: stringsByColumn[colTraceID][i], SpanID: stringsByColumn[colSpanID][i], - ParentSpanID: stringsByColumn[colParentSpanID][i], ServiceName: stringsByColumn[colServiceName][i], Name: stringsByColumn[colName][i], Kind: stringsByColumn[colKind][i], - StartUnixNanos: int64At(columns[colStartUnixNanos], sourceRow), EndUnixNanos: int64At(columns[colEndUnixNanos], sourceRow), DurationMS: float64At(columns[colDurationMS], sourceRow), - StatusCode: stringsByColumn[colStatusCode][i], StatusMsg: stringsByColumn[colStatusMsg][i], ResourceJSON: bytesByColumn[colResourceJSON][i], - AttributesJSON: bytesByColumn[colAttributesJSON][i], EventsJSON: bytesByColumn[colEventsJSON][i], LinksJSON: bytesByColumn[colLinksJSON][i], - TraceState: stringsByColumn[colTraceState][i], Flags: uint32At(columns[colFlags], sourceRow), ScopeName: stringsByColumn[colScopeName][i], - ScopeVersion: stringsByColumn[colScopeVersion][i], IngestedAt: int64At(columns[colIngestedAt], sourceRow), HTTPMethod: stringsByColumn[colHTTPMethod][i], - HTTPStatusCode: stringsByColumn[colHTTPStatusCode][i], HTTPRoute: stringsByColumn[colHTTPRoute][i], DBSystem: stringsByColumn[colDBSystem][i], - RPCMethod: stringsByColumn[colRPCMethod][i], RPCService: stringsByColumn[colRPCService][i], PeerService: stringsByColumn[colPeerService][i], - ServiceVersion: stringsByColumn[colServiceVersion][i], DeploymentEnv: stringsByColumn[colDeploymentEnv][i], ExceptionType: stringsByColumn[colExceptionType][i], - ExceptionMessage: stringsByColumn[colExceptionMessage][i], - } - } - return rows, nil -} - -func selectStrings(src []byte, count int, selected []int) ([]string, error) { - out := make([]string, len(selected)) - target := 0 - for row := 0; row < count && target < len(selected); row++ { - value, rest, err := consumeByteView(src) - if err != nil { - return nil, err - } - src = rest - if row == selected[target] { - out[target] = string(value) - target++ - } - } - if target != len(selected) { - return nil, io.ErrUnexpectedEOF - } - return out, nil -} - -func selectBytes(src []byte, count int, selected []int) ([][]byte, error) { - out := make([][]byte, len(selected)) - target := 0 - for row := 0; row < count && target < len(selected); row++ { - value, rest, err := consumeByteView(src) - if err != nil { - return nil, err - } - src = rest - if row == selected[target] { - out[target] = append([]byte(nil), value...) - target++ - } - } - if target != len(selected) { - return nil, io.ErrUnexpectedEOF - } - return out, nil -} - -func matchingStringRows(src []byte, count int, target []byte) ([]int, error) { - var out []int - for row := 0; row < count; row++ { - value, rest, err := consumeByteView(src) - if err != nil { - return nil, err - } - src = rest - if bytes.Equal(value, target) { - out = append(out, row) - } - } - return out, nil -} - -func appendBytes(dst, value []byte) []byte { - dst = binary.AppendUvarint(dst, uint64(len(value))) - return append(dst, value...) -} - -func requireFixed(src []byte, count, width int) error { - if len(src) != count*width { - return fmt.Errorf("fixed column size: got %d want %d", len(src), count*width) - } - return nil -} - -func int64At(src []byte, row int) int64 { return int64(binary.LittleEndian.Uint64(src[row*8:])) } -func float64At(src []byte, row int) float64 { - return math.Float64frombits(binary.LittleEndian.Uint64(src[row*8:])) -} -func uint32At(src []byte, row int) uint32 { return binary.LittleEndian.Uint32(src[row*4:]) } diff --git a/internal/telemetry/segment/span_store.go b/internal/telemetry/segment/span_store.go deleted file mode 100644 index f05131e8..00000000 --- a/internal/telemetry/segment/span_store.go +++ /dev/null @@ -1,1158 +0,0 @@ -// Package segment implements Fanout's append-optimized telemetry segments. -// Immutable, checksummed segment files are published through an atomically -// replaced manifest, so a process crash exposes either the old or new commit. -package segment - -import ( - "bufio" - "bytes" - "container/heap" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strconv" - "strings" - "sync" - - "github.com/klauspost/compress/zstd" - "github.com/labstack/fanout/internal/telemetry" - "github.com/zeebo/xxh3" -) - -// segmentDecoderMaxMemory bounds what one decompressed segment section may -// claim. A block holds at most rowsPerBlock rows and a section is decoded whole, -// so this is far above any sound file while refusing a corrupt or crafted frame -// long before zstd's 64 GiB default would. -const ( - segmentDecoderMaxMemory = 128 << 20 - segmentMaxCompressedBytes = segmentDecoderMaxMemory + (1 << 20) - segmentMaxBlocks = 1 << 20 -) - -var segmentIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) - -// ValidID reports whether id can safely name a durable segment and WAL file. -func ValidID(id string) bool { return segmentIDPattern.MatchString(id) } - -// newSegmentDecoder builds a decoder bounded to segmentDecoderMaxMemory. Every -// segment read goes through it, so no on-disk frame can size an allocation. -func newSegmentDecoder() (*zstd.Decoder, error) { - return newSegmentDecoderWithLimit(segmentDecoderMaxMemory) -} - -func newSegmentDecoderWithLimit(limit uint64) (*zstd.Decoder, error) { - return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(limit)) -} - -const ( - segmentMagic = "FANSEG04" - segmentVersion = uint32(4) - headerSize = 64 - blockDirSize = 32 - traceEntrySize = 12 - rowsPerBlock = 2048 -) - -type Span = telemetry.Span - -// ValidateSpanRows rejects rows whose columnar blocks could not be reopened -// within the production decoder budget. It must run before WAL staging so a -// deterministic format error never becomes a boot-blocking WAL. -func ValidateSpanRows(rows []Span) error { - return validateSpanRowsWithLimit(rows, segmentDecoderMaxMemory) -} - -func validateSpanRowsWithLimit(rows []Span, maxBlockBytes uint64) error { - if maxBlockBytes < columnarHeaderSize { - return fmt.Errorf("span block budget %d is smaller than its %d-byte header", maxBlockBytes, columnarHeaderSize) - } - for start := 0; start < len(rows); start += rowsPerBlock { - end := min(start+rowsPerBlock, len(rows)) - sizes := spanColumnPlainSizes(rows[start:end]) - total := uint64(columnarHeaderSize) - for id, size := range sizes { - if size > maxBlockBytes { - return fmt.Errorf("span block column %d uses %d bytes; maximum is %d", id, size, maxBlockBytes) - } - if size > maxBlockBytes-total { - return fmt.Errorf("span block uses more than %d bytes", maxBlockBytes) - } - total += size - } - } - return nil -} - -type Aggregate struct { - Calls uint64 - Errors uint64 - DurationMS float64 -} - -type blockDir struct { - offset uint64 - length uint32 - rows uint32 - min int64 - max int64 -} - -type traceEntry struct { - hash uint64 - block uint32 -} - -type indexCursor struct { - file *os.File - segment segment - blockBase uint32 - position uint32 - entry traceEntry -} - -type indexCursorHeap []*indexCursor - -func (h indexCursorHeap) Len() int { return len(h) } -func (h indexCursorHeap) Less(i, j int) bool { - if h[i].entry.hash != h[j].entry.hash { - return h[i].entry.hash < h[j].entry.hash - } - return h[i].entry.block < h[j].entry.block -} -func (h indexCursorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *indexCursorHeap) Push(value any) { *h = append(*h, value.(*indexCursor)) } -func (h *indexCursorHeap) Pop() any { - old := *h - last := old[len(old)-1] - *h = old[:len(old)-1] - return last -} - -type segment struct { - path string - rows uint32 - min int64 - max int64 - blocks []blockDir - indexOffset uint64 - indexCount uint32 -} - -type manifest struct { - NextID uint64 `json:"next_id"` - Files []string `json:"files"` -} - -// Store is a set of immutable segment files referenced by one atomically -// replaced manifest. Readers see either the old commit or the complete new one. -type Store struct { - dir string - writeMu sync.Mutex - mu sync.RWMutex - manifest manifest - segments []segment - encoder *zstd.Encoder - decoders sync.Pool -} - -func Open(dir string) (*Store, error) { - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("create segment directory: %w", err) - } - enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedFastest), zstd.WithEncoderConcurrency(1)) - if err != nil { - return nil, fmt.Errorf("create zstd encoder: %w", err) - } - s := &Store{dir: dir, encoder: enc} - s.decoders.New = func() any { - dec, decErr := newSegmentDecoder() - if decErr != nil { - panic(decErr) - } - return dec - } - if err := s.loadManifest(); err != nil { - enc.Close() - return nil, err - } - return s, nil -} - -func (s *Store) Close() error { - s.writeMu.Lock() - defer s.writeMu.Unlock() - return s.encoder.Close() -} - -func (s *Store) loadManifest() error { - path := filepath.Join(s.dir, "MANIFEST.json") - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - s.manifest = manifest{NextID: 1} - } else if err != nil { - return fmt.Errorf("read manifest: %w", err) - } else if err := json.Unmarshal(data, &s.manifest); err != nil { - return fmt.Errorf("decode manifest: %w", err) - } - // A crash can occur after a segment rename but before the manifest rename. - // Such an orphan is intentionally invisible, but its numeric name must still - // advance the allocator so the next append does not collide with it. - entries, err := os.ReadDir(s.dir) - if err != nil { - return fmt.Errorf("scan segment directory: %w", err) - } - for _, entry := range entries { - name := entry.Name() - if !strings.HasSuffix(name, ".fseg") { - continue - } - id, parseErr := strconv.ParseUint(strings.TrimSuffix(name, ".fseg"), 10, 64) - if parseErr == nil && id >= s.manifest.NextID { - s.manifest.NextID = id + 1 - } - } - for _, name := range s.manifest.Files { - seg, err := openSegment(filepath.Join(s.dir, name)) - if err != nil { - return fmt.Errorf("open committed segment %s: %w", name, err) - } - s.segments = append(s.segments, seg) - } - return nil -} - -// Append writes one crash-safe immutable segment and atomically publishes it. -// The on-disk trace index is built in the same pass as block encoding. -func (s *Store) Append(rows []Span) error { - if len(rows) == 0 { - return nil - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - - s.mu.RLock() - id := s.manifest.NextID - current := s.manifest - s.mu.RUnlock() - name := fmt.Sprintf("%020d.fseg", id) - tmp := filepath.Join(s.dir, name+".tmp") - final := filepath.Join(s.dir, name) - _ = os.Remove(tmp) - seg, err := s.writeSegment(tmp, rows) - if err != nil { - _ = os.Remove(tmp) - return err - } - if err := os.Rename(tmp, final); err != nil { - _ = os.Remove(tmp) - return fmt.Errorf("publish segment: %w", err) - } - if err := syncDir(s.dir); err != nil { - return err - } - - next := current - next.NextID = id + 1 - next.Files = append(append([]string(nil), current.Files...), name) - if err := writeManifest(s.dir, next); err != nil { - // The segment is an unreferenced orphan and therefore invisible after a - // restart. A later compactor can safely collect such files. - s.mu.Lock() - s.manifest.NextID = id + 1 - s.mu.Unlock() - return err - } - seg.path = final - s.mu.Lock() - s.manifest = next - s.segments = append(s.segments, seg) - s.mu.Unlock() - return nil -} - -// AppendID publishes one idempotent span segment for a durable ingest -// transaction. Replaying the same transaction after a crash is a no-op. -func (s *Store) AppendID(id string, rows []Span) error { - if len(rows) == 0 { - return nil - } - if !segmentIDPattern.MatchString(id) { - return fmt.Errorf("invalid segment id %q", id) - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - - name := id + ".fseg" - s.mu.RLock() - current := s.manifest - for _, existing := range current.Files { - if existing == name { - s.mu.RUnlock() - return nil - } - } - s.mu.RUnlock() - tmp := filepath.Join(s.dir, name+".tmp") - final := filepath.Join(s.dir, name) - _ = os.Remove(tmp) - seg, err := s.writeSegment(tmp, rows) - if err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - _ = os.Remove(tmp) - return fmt.Errorf("publish segment: %w", err) - } - if err := syncDir(s.dir); err != nil { - return err - } - next := current - next.Files = append(append([]string(nil), current.Files...), name) - if err := writeManifest(s.dir, next); err != nil { - return err - } - seg.path = final - s.mu.Lock() - s.manifest = next - s.segments = append(s.segments, seg) - s.mu.Unlock() - return nil -} - -// CompactOldest rewrites the oldest committed segments as one larger segment. -// The replacement is published with the same atomic-manifest protocol as an -// append; old files are removed only after in-flight readers release the -// snapshot they were using. -func (s *Store) CompactOldest(count int) error { - if count < 2 { - return nil - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.mu.RLock() - count = min(count, len(s.segments)) - if count < 2 { - s.mu.RUnlock() - return nil - } - old := append([]segment(nil), s.segments[:count]...) - rest := append([]segment(nil), s.segments[count:]...) - current := s.manifest - s.mu.RUnlock() - return s.compactSegments(old, rest, current) -} - -// CompactCommitted compacts only raw ingest segments whose batch IDs are in -// the authoritative repository manifest. A partially applied WAL is therefore -// never folded into a replacement that could defeat replay idempotence. -func (s *Store) CompactCommitted(committed map[string]struct{}, maxInputs int) (int, error) { - if maxInputs < 2 { - return 0, nil - } - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.mu.RLock() - current := s.manifest - selected := make([]segment, 0, maxInputs) - rest := make([]segment, 0, len(s.segments)) - for _, seg := range s.segments { - id := strings.TrimSuffix(filepath.Base(seg.path), ".fseg") - if len(selected) < maxInputs { - if _, ok := committed[id]; ok { - selected = append(selected, seg) - continue - } - } - rest = append(rest, seg) - } - s.mu.RUnlock() - if len(selected) < 2 { - return 0, nil - } - if err := s.compactSegments(selected, rest, current); err != nil { - return 0, err - } - return len(selected), nil -} - -// compactSegments publishes a replacement while writeMu is held. -func (s *Store) compactSegments(old, rest []segment, current manifest) error { - id := current.NextID - - name := fmt.Sprintf("%020d.fseg", id) - tmp, final := filepath.Join(s.dir, name+".tmp"), filepath.Join(s.dir, name) - replacement, err := s.writeCompactedSegment(tmp, old) - if err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - _ = os.Remove(tmp) - return fmt.Errorf("publish compacted segment: %w", err) - } - if err := syncDir(s.dir); err != nil { - return err - } - next := manifest{NextID: id + 1, Files: make([]string, 0, 1+len(rest))} - next.Files = append(next.Files, name) - for _, seg := range rest { - next.Files = append(next.Files, filepath.Base(seg.path)) - } - if err := writeManifest(s.dir, next); err != nil { - return err - } - replacement.path = final - - s.mu.Lock() - s.manifest = next - s.segments = append([]segment{replacement}, rest...) - var removeErr error - for _, seg := range old { - if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { - removeErr = errors.Join(removeErr, err) - } - } - s.mu.Unlock() - if removeErr != nil { - return fmt.Errorf("remove compacted segments: %w", removeErr) - } - return syncDir(s.dir) -} - -// PruneBefore removes segments whose newest event is older than cutoff. A -// boundary segment is retained intact, so retention never drops a newer row. -func (s *Store) PruneBefore(cutoff int64) (int, error) { - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.mu.RLock() - current := s.manifest - segments := append([]segment(nil), s.segments...) - s.mu.RUnlock() - kept := make([]segment, 0, len(segments)) - removed := make([]segment, 0) - for _, seg := range segments { - if seg.max < cutoff { - removed = append(removed, seg) - } else { - kept = append(kept, seg) - } - } - if len(removed) == 0 { - return 0, nil - } - next := current - next.Files = make([]string, 0, len(kept)) - for _, seg := range kept { - next.Files = append(next.Files, filepath.Base(seg.path)) - } - if err := writeManifest(s.dir, next); err != nil { - return 0, err - } - s.mu.Lock() - s.manifest = next - s.segments = kept - s.mu.Unlock() - var removeErr error - for _, seg := range removed { - if err := os.Remove(seg.path); err != nil && !errors.Is(err, os.ErrNotExist) { - removeErr = errors.Join(removeErr, err) - } - } - return len(removed), errors.Join(removeErr, syncDir(s.dir)) -} - -func (s *Store) writeSegment(path string, rows []Span) (segment, error) { - if err := ValidateSpanRows(rows); err != nil { - return segment{}, err - } - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) - if err != nil { - return segment{}, fmt.Errorf("create segment: %w", err) - } - ok := false - defer func() { - _ = f.Close() - if !ok { - _ = os.Remove(path) - } - }() - if _, err := f.Write(make([]byte, headerSize)); err != nil { - return segment{}, fmt.Errorf("reserve header: %w", err) - } - - seg := segment{rows: uint32(len(rows)), min: math.MaxInt64, max: math.MinInt64} - index := make([]traceEntry, 0, len(rows)) - var offset = uint64(headerSize) - for blockStart := 0; blockStart < len(rows); blockStart += rowsPerBlock { - blockEnd := min(blockStart+rowsPerBlock, len(rows)) - blockMin := int64(math.MaxInt64) - blockMax := int64(math.MinInt64) - blockTraces := make(map[uint64]struct{}, blockEnd-blockStart) - for _, row := range rows[blockStart:blockEnd] { - blockMin = min(blockMin, row.StartUnixNanos) - blockMax = max(blockMax, row.StartUnixNanos) - seg.min = min(seg.min, row.StartUnixNanos) - seg.max = max(seg.max, row.StartUnixNanos) - traceHash := xxh3.HashString(row.TraceID) - if _, exists := blockTraces[traceHash]; !exists { - index = append(index, traceEntry{hash: traceHash, block: uint32(len(seg.blocks))}) - blockTraces[traceHash] = struct{}{} - } - } - encoded, err := encodeColumnarBlock(s.encoder, rows[blockStart:blockEnd]) - if err != nil { - return segment{}, fmt.Errorf("encode block: %w", err) - } - if _, err := f.Write(encoded); err != nil { - return segment{}, fmt.Errorf("write block: %w", err) - } - seg.blocks = append(seg.blocks, blockDir{offset: offset, length: uint32(len(encoded)), rows: uint32(blockEnd - blockStart), min: blockMin, max: blockMax}) - offset += uint64(len(encoded)) - } - - sort.Slice(index, func(i, j int) bool { - if index[i].hash != index[j].hash { - return index[i].hash < index[j].hash - } - return index[i].block < index[j].block - }) - - dirOffset := offset - directory := make([]byte, len(seg.blocks)*blockDirSize) - for i, block := range seg.blocks { - buf := directory[i*blockDirSize:] - binary.LittleEndian.PutUint64(buf[0:8], block.offset) - binary.LittleEndian.PutUint32(buf[8:12], block.length) - binary.LittleEndian.PutUint32(buf[12:16], block.rows) - binary.LittleEndian.PutUint64(buf[16:24], uint64(block.min)) - binary.LittleEndian.PutUint64(buf[24:32], uint64(block.max)) - } - if _, err := f.Write(directory); err != nil { - return segment{}, fmt.Errorf("write block directory: %w", err) - } - indexOffset, _ := f.Seek(0, io.SeekCurrent) - indexPlain := make([]byte, len(index)*traceEntrySize) - for i, entry := range index { - buf := indexPlain[i*traceEntrySize:] - binary.LittleEndian.PutUint64(buf[0:8], entry.hash) - binary.LittleEndian.PutUint32(buf[8:12], entry.block) - } - if _, err := f.Write(indexPlain); err != nil { - return segment{}, fmt.Errorf("write trace index: %w", err) - } - indexEnd, _ := f.Seek(0, io.SeekCurrent) - seg.indexOffset = uint64(indexOffset) - seg.indexCount = uint32(len(index)) - - var header [headerSize]byte - copy(header[0:8], segmentMagic) - binary.LittleEndian.PutUint32(header[8:12], segmentVersion) - binary.LittleEndian.PutUint32(header[12:16], seg.rows) - binary.LittleEndian.PutUint32(header[16:20], uint32(len(seg.blocks))) - binary.LittleEndian.PutUint32(header[20:24], seg.indexCount) - binary.LittleEndian.PutUint64(header[24:32], uint64(seg.min)) - binary.LittleEndian.PutUint64(header[32:40], uint64(seg.max)) - binary.LittleEndian.PutUint64(header[40:48], dirOffset) - binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) - binary.LittleEndian.PutUint64(header[56:64], uint64(indexEnd)) - if _, err := f.WriteAt(header[:], 0); err != nil { - return segment{}, fmt.Errorf("write header: %w", err) - } - if err := f.Sync(); err != nil { - return segment{}, fmt.Errorf("sync segment: %w", err) - } - if err := f.Close(); err != nil { - return segment{}, fmt.Errorf("close segment: %w", err) - } - ok = true - return seg, nil -} - -func (s *Store) writeCompactedSegment(path string, inputs []segment) (segment, error) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) - if err != nil { - return segment{}, fmt.Errorf("create compacted segment: %w", err) - } - ok := false - defer func() { - _ = f.Close() - if !ok { - _ = os.Remove(path) - } - }() - if _, err := f.Write(make([]byte, headerSize)); err != nil { - return segment{}, err - } - replacement := segment{min: math.MaxInt64, max: math.MinInt64} - blockBases := make([]uint32, len(inputs)) - var offset = uint64(headerSize) - for i, input := range inputs { - source, err := os.Open(input.path) - if err != nil { - return segment{}, err - } - blockBase := uint32(len(replacement.blocks)) - blockBases[i] = blockBase - for _, block := range input.blocks { - if _, err := io.CopyN(f, io.NewSectionReader(source, int64(block.offset), int64(block.length)), int64(block.length)); err != nil { - _ = source.Close() - return segment{}, fmt.Errorf("copy compressed block: %w", err) - } - replacement.blocks = append(replacement.blocks, blockDir{offset: offset, length: block.length, rows: block.rows, min: block.min, max: block.max}) - offset += uint64(block.length) - } - if err := source.Close(); err != nil { - return segment{}, err - } - replacement.rows += input.rows - replacement.min = min(replacement.min, input.min) - replacement.max = max(replacement.max, input.max) - } - - dirOffset := offset - directory := make([]byte, len(replacement.blocks)*blockDirSize) - for i, block := range replacement.blocks { - buf := directory[i*blockDirSize:] - binary.LittleEndian.PutUint64(buf[0:8], block.offset) - binary.LittleEndian.PutUint32(buf[8:12], block.length) - binary.LittleEndian.PutUint32(buf[12:16], block.rows) - binary.LittleEndian.PutUint64(buf[16:24], uint64(block.min)) - binary.LittleEndian.PutUint64(buf[24:32], uint64(block.max)) - } - if _, err := f.Write(directory); err != nil { - return segment{}, err - } - indexOffset, _ := f.Seek(0, io.SeekCurrent) - indexCount, err := mergeTraceIndexes(f, inputs, blockBases) - if err != nil { - return segment{}, err - } - indexEnd, _ := f.Seek(0, io.SeekCurrent) - replacement.indexOffset = uint64(indexOffset) - replacement.indexCount = indexCount - var header [headerSize]byte - copy(header[0:8], segmentMagic) - binary.LittleEndian.PutUint32(header[8:12], segmentVersion) - binary.LittleEndian.PutUint32(header[12:16], replacement.rows) - binary.LittleEndian.PutUint32(header[16:20], uint32(len(replacement.blocks))) - binary.LittleEndian.PutUint32(header[20:24], replacement.indexCount) - binary.LittleEndian.PutUint64(header[24:32], uint64(replacement.min)) - binary.LittleEndian.PutUint64(header[32:40], uint64(replacement.max)) - binary.LittleEndian.PutUint64(header[40:48], dirOffset) - binary.LittleEndian.PutUint64(header[48:56], uint64(indexOffset)) - binary.LittleEndian.PutUint64(header[56:64], uint64(indexEnd)) - if _, err := f.WriteAt(header[:], 0); err != nil { - return segment{}, err - } - if err := f.Sync(); err != nil { - return segment{}, err - } - if err := f.Close(); err != nil { - return segment{}, err - } - ok = true - return replacement, nil -} - -// mergeTraceIndexes performs a bounded-memory k-way merge of fixed-width, -// sorted on-disk indexes. Compaction therefore scales with file count rather -// than retaining every trace entry from every generation in the Go heap. -func mergeTraceIndexes(dst io.Writer, inputs []segment, blockBases []uint32) (uint32, error) { - if len(inputs) != len(blockBases) { - return 0, errors.New("trace-index inputs and block bases disagree") - } - cursors := make(indexCursorHeap, 0, len(inputs)) - files := make([]*os.File, 0, len(inputs)) - defer func() { - for _, file := range files { - _ = file.Close() - } - }() - var total uint64 - for i, input := range inputs { - total += uint64(input.indexCount) - if total > math.MaxUint32 { - return 0, errors.New("compacted trace index exceeds entry limit") - } - if input.indexCount == 0 { - continue - } - f, err := os.Open(input.path) - if err != nil { - return 0, err - } - files = append(files, f) - cursor := &indexCursor{file: f, segment: input, blockBase: blockBases[i]} - entry, err := readTraceEntry(f, input, 0) - if err != nil { - _ = f.Close() - return 0, err - } - if entry.block > math.MaxUint32-cursor.blockBase { - _ = f.Close() - return 0, errors.New("compacted trace block id overflows") - } - entry.block += cursor.blockBase - cursor.entry = entry - cursors = append(cursors, cursor) - } - heap.Init(&cursors) - writer := bufio.NewWriterSize(dst, 64<<10) - var encoded [traceEntrySize]byte - for cursors.Len() > 0 { - cursor := heap.Pop(&cursors).(*indexCursor) - binary.LittleEndian.PutUint64(encoded[0:8], cursor.entry.hash) - binary.LittleEndian.PutUint32(encoded[8:12], cursor.entry.block) - if _, err := writer.Write(encoded[:]); err != nil { - return 0, err - } - cursor.position++ - if cursor.position < cursor.segment.indexCount { - entry, err := readTraceEntry(cursor.file, cursor.segment, cursor.position) - if err != nil { - return 0, err - } - if entry.block > math.MaxUint32-cursor.blockBase { - return 0, errors.New("compacted trace block id overflows") - } - entry.block += cursor.blockBase - cursor.entry = entry - heap.Push(&cursors, cursor) - } - } - if err := writer.Flush(); err != nil { - return 0, err - } - return uint32(total), nil -} - -func readTraceEntry(f *os.File, seg segment, position uint32) (traceEntry, error) { - if position >= seg.indexCount { - return traceEntry{}, io.EOF - } - var encoded [traceEntrySize]byte - offset := seg.indexOffset + uint64(position)*traceEntrySize - if _, err := f.ReadAt(encoded[:], int64(offset)); err != nil { - return traceEntry{}, err - } - entry := traceEntry{hash: binary.LittleEndian.Uint64(encoded[0:8]), block: binary.LittleEndian.Uint32(encoded[8:12])} - if int(entry.block) >= len(seg.blocks) { - return traceEntry{}, errors.New("trace index references a missing block") - } - return entry, nil -} - -func traceBlocks(seg segment, hash uint64) ([]uint32, error) { - f, err := os.Open(seg.path) - if err != nil { - return nil, err - } - defer f.Close() - low, high := uint32(0), seg.indexCount - for low < high { - middle := low + (high-low)/2 - entry, err := readTraceEntry(f, seg, middle) - if err != nil { - return nil, err - } - if entry.hash < hash { - low = middle + 1 - } else { - high = middle - } - } - blocks := make([]uint32, 0, 1) - for position := low; position < seg.indexCount; position++ { - entry, err := readTraceEntry(f, seg, position) - if err != nil { - return nil, err - } - if entry.hash != hash { - break - } - if len(blocks) == 0 || blocks[len(blocks)-1] != entry.block { - blocks = append(blocks, entry.block) - } - } - return blocks, nil -} - -// Trace performs a hash-index lookup and decompresses only the blocks that can -// contain the requested trace. The full trace ID is checked after hashing. -func (s *Store) Trace(traceID string) ([]Span, error) { - s.mu.RLock() - defer s.mu.RUnlock() - hash := xxh3.HashString(traceID) - var out []Span - for i := range s.segments { - seg := s.segments[i] - blocks, err := traceBlocks(seg, hash) - if err != nil { - return nil, err - } - for _, blockID := range blocks { - rows, err := s.readTraceBlock(seg, blockID, traceID) - if err != nil { - return nil, err - } - out = append(out, rows...) - } - } - sort.Slice(out, func(i, j int) bool { return out[i].StartUnixNanos < out[j].StartUnixNanos }) - return out, nil -} - -// ScanService is the deliberately expensive raw path. It demonstrates block -// time pruning and provides a fairer comparison with a general query engine. -func (s *Store) ScanService(namespace, service string, start, end int64) (Aggregate, error) { - s.mu.RLock() - defer s.mu.RUnlock() - var out Aggregate - wanted := []int{colNamespace, colServiceName, colStartUnixNanos, colDurationMS, colStatusCode} - namespaceNeedle, serviceNeedle := []byte(namespace), []byte(service) - for i := range s.segments { - seg := s.segments[i] - if seg.max < start || seg.min >= end { - continue - } - f, err := os.Open(seg.path) - if err != nil { - return Aggregate{}, err - } - for blockID := range seg.blocks { - block := seg.blocks[blockID] - if block.max < start || block.min >= end { - continue - } - columns, err := s.readColumns(f, block, wanted) - if err != nil { - _ = f.Close() - return Aggregate{}, err - } - if err := requireFixed(columns[colStartUnixNanos], int(block.rows), 8); err != nil { - _ = f.Close() - return Aggregate{}, err - } - if err := requireFixed(columns[colDurationMS], int(block.rows), 8); err != nil { - _ = f.Close() - return Aggregate{}, err - } - namespaceColumn := columns[colNamespace] - serviceColumn := columns[colServiceName] - statusColumn := columns[colStatusCode] - for row := range int(block.rows) { - namespaceValue, rest, err := consumeByteView(namespaceColumn) - if err != nil { - _ = f.Close() - return Aggregate{}, err - } - namespaceColumn = rest - serviceValue, rest, err := consumeByteView(serviceColumn) - if err != nil { - _ = f.Close() - return Aggregate{}, err - } - serviceColumn = rest - statusValue, rest, err := consumeByteView(statusColumn) - if err != nil { - _ = f.Close() - return Aggregate{}, err - } - statusColumn = rest - timestamp := int64At(columns[colStartUnixNanos], row) - if timestamp < start || timestamp >= end { - continue - } - if namespace != "" && !bytes.Equal(namespaceValue, namespaceNeedle) { - continue - } - if service != "" && !bytes.Equal(serviceValue, serviceNeedle) { - continue - } - out.Calls++ - out.DurationMS += float64At(columns[colDurationMS], row) - if isErrorStatus(string(statusValue)) { - out.Errors++ - } - } - } - if err := f.Close(); err != nil { - return Aggregate{}, err - } - } - return out, nil -} - -func (s *Store) readTraceBlock(seg segment, blockID uint32, traceID string) ([]Span, error) { - if int(blockID) >= len(seg.blocks) { - return nil, fmt.Errorf("block %d out of range", blockID) - } - block := seg.blocks[blockID] - f, err := os.Open(seg.path) - if err != nil { - return nil, err - } - defer f.Close() - columns, err := s.readColumns(f, block, allColumns) - if err != nil { - return nil, err - } - selected, err := matchingStringRows(columns[colTraceID], int(block.rows), []byte(traceID)) - if err != nil { - return nil, err - } - return decodeSelectedBlock(columns, int(block.rows), selected) -} - -func (s *Store) readColumns(f *os.File, block blockDir, wanted []int) (map[int][]byte, error) { - encoded := make([]byte, block.length) - if _, err := f.ReadAt(encoded, int64(block.offset)); err != nil { - return nil, err - } - dec := s.decoders.Get().(*zstd.Decoder) - columns, err := decodeColumns(dec, encoded, wanted) - s.decoders.Put(dec) - return columns, err -} - -// isErrorStatus matches both status spellings telemetry carries: OTLP ingest -// stores Status.Code.String() ("STATUS_CODE_ERROR"), while other producers and -// older rows use the bare code. The DuckDB rollups compare against the same -// pair. -func isErrorStatus(status string) bool { - return strings.EqualFold(status, "ERROR") || strings.EqualFold(status, "STATUS_CODE_ERROR") -} - -// validateSegmentSections bounds every header offset against the file before -// any of them is used to size an allocation. All comparisons are written as -// subtractions against size so a corrupt offset near the top of the address -// space cannot wrap past the check. -func validateSegmentSections(size, dirOffset, indexOffset, indexEnd uint64, blockCount, indexCount uint32) error { - if blockCount > segmentMaxBlocks { - return errors.New("segment block count is out of range") - } - if err := validateDirectory(size, dirOffset, blockCount, blockDirSize); err != nil { - return err - } - if indexOffset < dirOffset+uint64(blockCount)*blockDirSize || indexOffset > size { - return errors.New("corrupt trace index offset") - } - if indexEnd < indexOffset || indexEnd > size { - return errors.New("corrupt trace index end") - } - if uint64(indexCount) > (indexEnd-indexOffset)/traceEntrySize || indexEnd-indexOffset != uint64(indexCount)*traceEntrySize { - return errors.New("trace index size disagrees with the segment header") - } - if indexEnd != size { - return errors.New("segment has data past the trace index") - } - return nil -} - -// validateSegmentBlocks bounds every decoded block entry against the payload -// region, so a torn directory cannot drive a read or an allocation the file -// could never satisfy. -func validateSegmentBlocks(size, dirOffset uint64, blocks []blockDir, rows uint32) error { - var counted uint64 - for _, block := range blocks { - if block.offset < headerSize || block.offset > dirOffset { - return errors.New("block offset outside the segment payload") - } - if block.length == 0 || uint64(block.length) > dirOffset-block.offset || uint64(block.length) > segmentMaxCompressedBytes { - return errors.New("block extends past the block directory") - } - if block.rows == 0 || block.rows > rowsPerBlock { - return errors.New("block row count is out of range") - } - counted += uint64(block.rows) - } - if counted != uint64(rows) { - return errors.New("block row counts disagree with the segment header") - } - if dirOffset > size { - return errors.New("block directory starts past the end of the segment") - } - return nil -} - -// validateDirectory reports whether count fixed-size entries fit in the file -// when placed at offset. -func validateDirectory(size, offset uint64, count uint32, entrySize uint64) error { - if offset < headerSize || offset > size { - return errors.New("corrupt directory offset") - } - if uint64(count) > (size-offset)/entrySize { - return errors.New("directory does not fit in the segment") - } - return nil -} - -func openSegment(path string) (segment, error) { - f, err := os.Open(path) - if err != nil { - return segment{}, err - } - defer f.Close() - var header [headerSize]byte - if _, err := io.ReadFull(f, header[:]); err != nil { - return segment{}, err - } - if string(header[0:8]) != segmentMagic { - return segment{}, errors.New("invalid segment magic") - } - if binary.LittleEndian.Uint32(header[8:12]) != segmentVersion { - return segment{}, errors.New("unsupported segment version") - } - seg := segment{path: path, rows: binary.LittleEndian.Uint32(header[12:16]), min: int64(binary.LittleEndian.Uint64(header[24:32])), max: int64(binary.LittleEndian.Uint64(header[32:40]))} - blockCount := binary.LittleEndian.Uint32(header[16:20]) - indexCount := binary.LittleEndian.Uint32(header[20:24]) - dirOffset := binary.LittleEndian.Uint64(header[40:48]) - indexOffset := binary.LittleEndian.Uint64(header[48:56]) - indexEnd := binary.LittleEndian.Uint64(header[56:64]) - info, err := f.Stat() - if err != nil { - return segment{}, err - } - if err := validateSegmentSections(uint64(info.Size()), dirOffset, indexOffset, indexEnd, blockCount, indexCount); err != nil { - return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) - } - seg.blocks = make([]blockDir, blockCount) - buf := make([]byte, int(blockCount)*blockDirSize) - if _, err := f.ReadAt(buf, int64(dirOffset)); err != nil { - return segment{}, err - } - for i := range seg.blocks { - b := buf[i*blockDirSize:] - seg.blocks[i] = blockDir{offset: binary.LittleEndian.Uint64(b[0:8]), length: binary.LittleEndian.Uint32(b[8:12]), rows: binary.LittleEndian.Uint32(b[12:16]), min: int64(binary.LittleEndian.Uint64(b[16:24])), max: int64(binary.LittleEndian.Uint64(b[24:32]))} - } - if err := validateSegmentBlocks(uint64(info.Size()), dirOffset, seg.blocks, seg.rows); err != nil { - return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) - } - seg.indexOffset = indexOffset - seg.indexCount = indexCount - if err := validateTraceIndex(f, seg); err != nil { - return segment{}, fmt.Errorf("segment %s: %w", filepath.Base(path), err) - } - return seg, nil -} - -func validateTraceIndex(f *os.File, seg segment) error { - reader := bufio.NewReaderSize(io.NewSectionReader(f, int64(seg.indexOffset), int64(seg.indexCount)*traceEntrySize), 64<<10) - var encoded [traceEntrySize]byte - var previous traceEntry - for position := uint32(0); position < seg.indexCount; position++ { - if _, err := io.ReadFull(reader, encoded[:]); err != nil { - return err - } - entry := traceEntry{hash: binary.LittleEndian.Uint64(encoded[0:8]), block: binary.LittleEndian.Uint32(encoded[8:12])} - if int(entry.block) >= len(seg.blocks) { - return errors.New("trace index references a missing block") - } - if position > 0 && (entry.hash < previous.hash || entry.hash == previous.hash && entry.block < previous.block) { - return errors.New("trace index is not sorted") - } - previous = entry - } - return nil -} - -func appendString(dst []byte, value string) []byte { - dst = binary.AppendUvarint(dst, uint64(len(value))) - return append(dst, value...) -} - -func consumeByteView(src []byte) ([]byte, []byte, error) { - length, n := binary.Uvarint(src) - if n <= 0 { - return nil, nil, errors.New("invalid string length") - } - src = src[n:] - if length > uint64(len(src)) { - return nil, nil, io.ErrUnexpectedEOF - } - return src[:length], src[length:], nil -} - -func writeManifest(dir string, m manifest) error { - data, err := json.Marshal(m) - if err != nil { - return err - } - tmp := filepath.Join(dir, "MANIFEST.json.tmp") - f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - return err - } - if _, err := f.Write(data); err != nil { - _ = f.Close() - return err - } - if err := f.Sync(); err != nil { - _ = f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } - if err := os.Rename(tmp, filepath.Join(dir, "MANIFEST.json")); err != nil { - return err - } - return syncDir(dir) -} - -func syncDir(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - return f.Sync() -} - -// DiskBytes returns the committed segment and manifest size. -func (s *Store) DiskBytes() (int64, error) { - s.mu.RLock() - defer s.mu.RUnlock() - var total int64 - for _, name := range append(append([]string(nil), s.manifest.Files...), "MANIFEST.json") { - info, err := os.Stat(filepath.Join(s.dir, name)) - if errors.Is(err, os.ErrNotExist) && name == "MANIFEST.json" { - continue - } - if err != nil { - return 0, err - } - total += info.Size() - } - return total, nil -} - -// RowCount returns the number of committed rows. -func (s *Store) RowCount() uint64 { - s.mu.RLock() - defer s.mu.RUnlock() - var total uint64 - for i := range s.segments { - total += uint64(s.segments[i].rows) - } - return total -} - -func (s *Store) SegmentCount() int { - s.mu.RLock() - defer s.mu.RUnlock() - return len(s.segments) -} - -// EqualSpan compares every persisted field. It is intended for POC recovery -// and format-roundtrip validation, where nil and empty JSON are distinct. -func EqualSpan(a, b Span) bool { - return reflect.DeepEqual(a, b) -} diff --git a/internal/telemetry/segment/span_store_test.go b/internal/telemetry/segment/span_store_test.go deleted file mode 100644 index d062c443..00000000 --- a/internal/telemetry/segment/span_store_test.go +++ /dev/null @@ -1,428 +0,0 @@ -// Tests use the internal package to exercise crash boundaries and corruption. -package segment - -import ( - "encoding/binary" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/klauspost/compress/zstd" - "github.com/zeebo/xxh3" -) - -func TestStoreCommitReopenAndQueries(t *testing.T) { - dir := t.TempDir() - base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() - rows := []Span{ - {Namespace: "default", TraceID: "trace-a", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/users/:id", StartUnixNanos: base, EndUnixNanos: base + int64(10*time.Millisecond), DurationMS: 10, StatusCode: "OK", AttributesJSON: []byte(`{"tenant":"a"}`)}, - {Namespace: "default", TraceID: "trace-a", SpanID: "2", ParentSpanID: "1", ServiceName: "db", StartUnixNanos: base + int64(time.Millisecond), EndUnixNanos: base + int64(6*time.Millisecond), DurationMS: 5, StatusCode: "OK"}, - {Namespace: "default", TraceID: "trace-b", SpanID: "3", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/users/:id", StartUnixNanos: base + int64(time.Minute), EndUnixNanos: base + int64(time.Minute+50*time.Millisecond), DurationMS: 50, StatusCode: "ERROR"}, - } - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := store.Append(rows[:2]); err != nil { - t.Fatal(err) - } - if err := store.Append(rows[2:]); err != nil { - t.Fatal(err) - } - if got := store.RowCount(); got != 3 { - t.Fatalf("row count = %d", got) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - - // Files left by an interrupted append are not present in the committed - // manifest and must not become visible after recovery. - if err := os.WriteFile(filepath.Join(dir, "999.fseg.tmp"), []byte("partial"), 0o644); err != nil { - t.Fatal(err) - } - store, err = Open(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if got := store.RowCount(); got != 3 { - t.Fatalf("reopened row count = %d", got) - } - trace, err := store.Trace("trace-a") - if err != nil { - t.Fatal(err) - } - if len(trace) != 2 || !EqualSpan(trace[0], rows[0]) || !EqualSpan(trace[1], rows[1]) { - t.Fatalf("trace result = %#v", trace) - } - agg, err := store.ScanService("default", "api", base, base+int64(5*time.Minute)) - if err != nil { - t.Fatal(err) - } - if agg.Calls != 2 || agg.Errors != 1 || agg.DurationMS != 60 { - t.Fatalf("aggregate = %#v", agg) - } -} - -func TestStoreSkipsOrphanAndAdvancesSegmentID(t *testing.T) { - dir := t.TempDir() - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - base := time.Now().UnixNano() - row := Span{TraceID: "trace", SpanID: "span", StartUnixNanos: base} - if err := store.Append([]Span{row}); err != nil { - t.Fatal(err) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - // Simulate a crash after publishing segment 2 but before its manifest commit. - orphan := filepath.Join(dir, "00000000000000000002.fseg") - committed := filepath.Join(dir, "00000000000000000001.fseg") - data, err := os.ReadFile(committed) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(orphan, data, 0o644); err != nil { - t.Fatal(err) - } - store, err = Open(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if got := store.RowCount(); got != 1 { - t.Fatalf("orphan became visible: row count = %d", got) - } - if err := store.Append([]Span{row}); err != nil { - t.Fatalf("append after orphan: %v", err) - } - if got := store.RowCount(); got != 2 { - t.Fatalf("row count after append = %d", got) - } - if _, err := os.Stat(filepath.Join(dir, "00000000000000000003.fseg")); err != nil { - t.Fatalf("allocator did not advance past orphan: %v", err) - } -} - -func TestStoreCompactionPreservesRowsAndIndexes(t *testing.T) { - dir := t.TempDir() - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - base := time.Now().UnixNano() - for batch := range 3 { - rows := []Span{ - {TraceID: "shared", SpanID: string(rune('a' + batch)), ServiceName: "api", StartUnixNanos: base + int64(batch), StatusCode: "OK"}, - {TraceID: "other", SpanID: string(rune('x' + batch)), ServiceName: "worker", StartUnixNanos: base + int64(batch+10), StatusCode: "ERROR"}, - } - if err := store.Append(rows); err != nil { - t.Fatal(err) - } - } - if err := store.CompactOldest(2); err != nil { - t.Fatal(err) - } - if got := store.SegmentCount(); got != 2 { - t.Fatalf("segments after compaction = %d", got) - } - if got := store.RowCount(); got != 6 { - t.Fatalf("rows after compaction = %d", got) - } - trace, err := store.Trace("shared") - if err != nil { - t.Fatal(err) - } - if len(trace) != 3 { - t.Fatalf("trace rows after compaction = %d", len(trace)) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - store, err = Open(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if got := store.RowCount(); got != 6 { - t.Fatalf("reopened compacted rows = %d", got) - } -} - -func TestTraceIndexIsReadLazilyFromDisk(t *testing.T) { - dir := t.TempDir() - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if err := store.AppendID("lazy-index", []Span{{TraceID: "trace-a", SpanID: "span-a", StartUnixNanos: 1}}); err != nil { - t.Fatal(err) - } - store.mu.RLock() - seg := store.segments[0] - store.mu.RUnlock() - if seg.indexCount != 1 { - t.Fatalf("index count = %d, want 1", seg.indexCount) - } - f, err := os.OpenFile(seg.path, os.O_WRONLY, 0) - if err != nil { - t.Fatal(err) - } - var replacement [8]byte - binary.LittleEndian.PutUint64(replacement[:], xxh3.HashString("trace-b")) - if _, err := f.WriteAt(replacement[:], int64(seg.indexOffset)); err != nil { - _ = f.Close() - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - rows, err := store.Trace("trace-a") - if err != nil { - t.Fatal(err) - } - if len(rows) != 0 { - t.Fatalf("Trace returned %d rows from a stale resident index", len(rows)) - } -} - -func TestStoreRejectsCorruptCommittedSegment(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "broken.fseg"), []byte("bad"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "MANIFEST.json"), []byte(`{"next_id":2,"files":["broken.fseg"]}`), 0o644); err != nil { - t.Fatal(err) - } - if _, err := Open(dir); err == nil { - t.Fatal("Open succeeded with a corrupt committed segment") - } -} - -func TestOpenRejectsSegmentWithCorruptSectionOffsets(t *testing.T) { - base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() - rows := []Span{{Namespace: "default", TraceID: "trace-a", SpanID: "1", ServiceName: "api", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}} - corrupt := func(t *testing.T, mutate func(header []byte)) { - t.Helper() - dir := t.TempDir() - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := store.AppendID("seg-a", rows); err != nil { - t.Fatal(err) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - path := filepath.Join(dir, "seg-a.fseg") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - mutate(data[:headerSize]) - if err := os.WriteFile(path, data, 0o644); err != nil { - t.Fatal(err) - } - if _, err := Open(dir); err == nil { - t.Fatal("Open succeeded with corrupt section offsets") - } - } - t.Run("index end before index offset", func(t *testing.T) { - corrupt(t, func(header []byte) { - indexOffset := binary.LittleEndian.Uint64(header[48:56]) - binary.LittleEndian.PutUint64(header[56:64], indexOffset-1) - }) - }) - t.Run("index end beyond file size", func(t *testing.T) { - corrupt(t, func(header []byte) { - binary.LittleEndian.PutUint64(header[56:64], 1<<40) - }) - }) -} - -func TestScanServiceCountsCanonicalOTelErrorStatus(t *testing.T) { - dir := t.TempDir() - base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() - rows := []Span{ - {Namespace: "default", TraceID: "t1", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/x", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, - {Namespace: "default", TraceID: "t2", SpanID: "2", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: "/x", StartUnixNanos: base + 1, EndUnixNanos: base + 2, DurationMS: 1, StatusCode: "STATUS_CODE_OK"}, - } - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if err := store.AppendID("seg-status", rows); err != nil { - t.Fatal(err) - } - agg, err := store.ScanService("default", "api", base, base+int64(time.Minute)) - if err != nil { - t.Fatal(err) - } - if agg.Errors != 1 { - t.Fatalf("aggregate Errors = %d, want 1", agg.Errors) - } -} - -func TestValidateSegmentSectionsRejectsOutOfBoundsDirectory(t *testing.T) { - const size = 4096 - tests := []struct { - name string - dirOffset, indexOffset, indexEnd uint64 - blockCount, indexCount uint32 - }{ - {"wrapping directory end", ^uint64(0) - uint64(0xFFFFFFFF)*blockDirSize + 1, 512, 512, 0xFFFFFFFF, 0}, - {"block count past index", headerSize, 512, 512, 0xFFFFFFFF, 0}, - {"directory before header", 0, 512, 512, 1, 0}, - {"sections out of order", headerSize, 2048, 1024, 1, 0}, - {"index past end of file", headerSize, 512, size + 1, 1, 0}, - {"index count exceeds extent", headerSize, 512, 512, 1, 1}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := validateSegmentSections(size, test.dirOffset, test.indexOffset, test.indexEnd, test.blockCount, test.indexCount); err == nil { - t.Fatal("validateSegmentSections accepted a corrupt header") - } - }) - } - if err := validateSegmentSections(size, headerSize, size, size, 8, 0); err != nil { - t.Fatalf("validateSegmentSections rejected a sound header: %v", err) - } -} - -func TestValidateSegmentBlocksRejectsOutOfBoundsExtents(t *testing.T) { - const size, dirOffset = 4096, uint64(2048) - sound := []blockDir{ - {offset: headerSize, length: 512, rows: 10}, - {offset: headerSize + 512, length: 512, rows: 10}, - } - if err := validateSegmentBlocks(size, dirOffset, sound, 20); err != nil { - t.Fatalf("validateSegmentBlocks rejected sound blocks: %v", err) - } - tests := []struct { - name string - blocks []blockDir - rows uint32 - }{ - {"length past the directory", []blockDir{{offset: headerSize, length: 4096, rows: 10}}, 10}, - {"offset inside the header", []blockDir{{offset: 0, length: 16, rows: 10}}, 10}, - {"extent wraps", []blockDir{{offset: ^uint64(0) - 8, length: 64, rows: 10}}, 10}, - {"rows disagree with the header", []blockDir{{offset: headerSize, length: 16, rows: 10}}, 11}, - {"empty block", []blockDir{{offset: headerSize, length: 0, rows: 0}}, 0}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := validateSegmentBlocks(size, dirOffset, test.blocks, test.rows); err == nil { - t.Fatal("validateSegmentBlocks accepted a corrupt block directory") - } - }) - } -} - -func TestOpenRejectsSegmentWithCorruptBlockEntry(t *testing.T) { - dir := t.TempDir() - base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := store.AppendID("seg-block", []Span{{Namespace: "default", TraceID: "t", SpanID: "1", ServiceName: "api", StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}}); err != nil { - t.Fatal(err) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - path := filepath.Join(dir, "seg-block.fseg") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - dirOffset := binary.LittleEndian.Uint64(data[40:48]) - // Claim the first block runs far past the directory it precedes. - binary.LittleEndian.PutUint32(data[int(dirOffset)+8:], 1<<30) - if err := os.WriteFile(path, data, 0o644); err != nil { - t.Fatal(err) - } - _, err = Open(dir) - if err == nil { - t.Fatal("Open accepted a segment whose block extends past its directory") - } - if !strings.Contains(err.Error(), "block extends past the block directory") { - t.Fatalf("Open error = %v, want the block-extent guard to reject it", err) - } -} - -func TestStoreAcceptsSpanWithLargeRoute(t *testing.T) { - dir := t.TempDir() - base := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC).UnixNano() - // A pathological route must not be a deterministic publish failure: the WAL - // would then abort every restart with no way to make progress. - route := "/" + strings.Repeat("x", 70000) - rows := []Span{{Namespace: "default", TraceID: "t", SpanID: "1", ServiceName: "api", HTTPMethod: "GET", HTTPRoute: route, StartUnixNanos: base, EndUnixNanos: base + 1, DurationMS: 1, StatusCode: "OK"}} - store, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := store.AppendID("seg-big-key", rows); err != nil { - t.Fatalf("AppendID error = %v, want a large but bounded route to be publishable", err) - } - if err := store.Close(); err != nil { - t.Fatal(err) - } - reopened, err := Open(dir) - if err != nil { - t.Fatalf("Open error = %v, want the segment to round-trip", err) - } - defer reopened.Close() - trace, err := reopened.Trace("t") - if err != nil { - t.Fatal(err) - } - if len(trace) != 1 || trace[0].HTTPRoute != route { - t.Fatalf("trace = %#v, want the full route preserved", trace) - } -} - -func TestValidateSegmentBlocksRejectsRowsPastBlockCap(t *testing.T) { - blocks := []blockDir{{offset: headerSize, length: 16, rows: rowsPerBlock + 1}} - if err := validateSegmentBlocks(4096, 2048, blocks, rowsPerBlock+1); err == nil { - t.Fatal("validateSegmentBlocks accepted a block claiming more rows than a block can hold") - } -} - -func TestSegmentDecoderRejectsOversizedFrame(t *testing.T) { - const testLimit = 64 << 10 - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) - if err != nil { - t.Fatal(err) - } - defer encoder.Close() - frame := encoder.EncodeAll(make([]byte, testLimit+(1<<10)), nil) - decoder, err := newSegmentDecoderWithLimit(testLimit) - if err != nil { - t.Fatal(err) - } - defer decoder.Close() - if _, err := decoder.DecodeAll(frame, nil); err == nil { - t.Fatal("segment decoder accepted a frame declaring more memory than the product ever needs") - } -} - -func TestValidateSpanRowsRejectsUnreopenableBlock(t *testing.T) { - rows := []Span{{AttributesJSON: []byte(strings.Repeat("x", 1024))}} - if err := validateSpanRowsWithLimit(rows, 512); err == nil || !strings.Contains(err.Error(), "column") { - t.Fatal("validator accepted a column larger than the decoder budget") - } - rows = []Span{{TraceID: "a", SpanID: "b", HTTPRoute: "c"}} - if err := validateSpanRowsWithLimit(rows, uint64(columnarHeaderSize+3)); err == nil { - t.Fatal("validator accepted a block larger than the decoder budget") - } -} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 7e63f307..7bf18789 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -12,71 +12,79 @@ import ( "strings" "sync" "time" -) -type compactionMarker struct { - ID string `json:"id"` - Inputs []string `json:"inputs"` - Signals []string `json:"signals"` - MinNanos int64 `json:"min_nanos"` - MaxNanos int64 `json:"max_nanos"` - Generation uint32 `json:"generation"` - Sources []string `json:"sources"` -} + "github.com/labstack/fanout/internal/telemetry" +) const minCompactionInputs = 8 var parquetSignals = [...]string{"spans", "logs", "metrics"} -var renameCompactionFile = os.Rename +type compactionMarker struct { + Output telemetry.BatchMetadata `json:"output"` + Inputs []string `json:"inputs"` +} -// CompactParquet combines the oldest small atomic batches into larger files. -// A durable marker makes the multi-signal swap recoverable after a crash. +type compactionKey struct { + day int64 + generation uint32 +} + +// CompactParquet combines one same-day, same-generation group. The output is +// prepared outside the query gate and swapped as one batch directory. func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { - if db == nil || maxBatches < 2 { + if db == nil || maxBatches < minCompactionInputs { return 0, nil } r.compactionMu.Lock() defer r.compactionMu.Unlock() markerPath := filepath.Join(r.root, "COMPACTION.json") if exists, err := pathExists(markerPath); err != nil { - return 0, fmt.Errorf("inspect pending compaction: %w", err) + return 0, err } else if exists { return 0, errors.New("pending Parquet compaction must recover before another can start") } - r.mu.RLock() - selected := selectCompactionBatches(r.manifest.Batches, maxBatches) - r.mu.RUnlock() + selected := selectCompactionBatches(r.Parquet.BatchMetadata(), maxBatches) if len(selected) < minCompactionInputs { return 0, nil } - marker := compactionMarker{ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinNanos: math.MaxInt64, Generation: selected[0].Generation + 1, Sources: compactionSources(selected)} + output := telemetry.BatchMetadata{ + ID: fmt.Sprintf("compact-%d", time.Now().UnixNano()), MinIngestedNanos: math.MaxInt64, + Generation: selected[0].Generation + 1, + } + marker := compactionMarker{Output: output, Inputs: make([]string, 0, len(selected))} for _, batch := range selected { marker.Inputs = append(marker.Inputs, batch.ID) - if batch.MinNanos > 0 { - marker.MinNanos = min(marker.MinNanos, batch.MinNanos) + if batch.MinIngestedNanos > 0 { + marker.Output.MinIngestedNanos = min(marker.Output.MinIngestedNanos, batch.MinIngestedNanos) } - marker.MaxNanos = max(marker.MaxNanos, batch.MaxNanos) + marker.Output.MaxIngestedNanos = max(marker.Output.MaxIngestedNanos, batch.MaxIngestedNanos) + marker.Output.Spans += batch.Spans + marker.Output.Logs += batch.Logs + marker.Output.Metrics += batch.Metrics } - if marker.MinNanos == math.MaxInt64 { - marker.MinNanos = 0 + if marker.Output.MinIngestedNanos == math.MaxInt64 { + marker.Output.MinIngestedNanos = 0 } - stageDir := filepath.Join(r.root, marker.ID) - if err := os.Mkdir(stageDir, 0o755); err != nil { + stage := r.compactionStage(marker.Output.ID) + if err := os.RemoveAll(stage); err != nil { return 0, err } - recoverable := false + if err := os.MkdirAll(stage, 0o755); err != nil { + return 0, err + } + prepared := false defer func() { - if !recoverable { - _ = os.RemoveAll(stageDir) + if !prepared { + _ = os.RemoveAll(stage) } }() for _, signal := range parquetSignals { - var inputs []string - for _, id := range marker.Inputs { - path := filepath.Join(r.Parquet.Dir(), signal, id+".parquet") + inputs := make([]string, 0, len(selected)) + for _, batch := range selected { + path := filepath.Join(r.Parquet.BatchPath(batch.ID), signal+".parquet") if _, err := os.Stat(path); err == nil { - inputs = append(inputs, path) + inputs = append(inputs, sqlQuote(path)) } else if !errors.Is(err, os.ErrNotExist) { return 0, err } @@ -84,25 +92,21 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if len(inputs) == 0 { continue } - marker.Signals = append(marker.Signals, signal) - quoted := make([]string, len(inputs)) - for i, path := range inputs { - quoted[i] = sqlQuote(path) + outputPath := filepath.Join(stage, signal+".parquet") + query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(inputs, ",")) + if signal == "spans" { + query += " ORDER BY _trace_hash, start_unix_nano, span_id" } - output := filepath.Join(stageDir, signal+".parquet") - stmt := fmt.Sprintf("COPY (SELECT * FROM read_parquet([%s], union_by_name=true)) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, ROW_GROUP_SIZE 122880)", strings.Join(quoted, ","), sqlQuote(output)) + stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, sqlQuote(outputPath)) if _, err := db.ExecContext(ctx, stmt); err != nil { - return 0, fmt.Errorf("compact %s parquet: %w", signal, err) + return 0, fmt.Errorf("compact %s Parquet: %w", signal, err) } - if err := syncFile(output); err != nil { - return 0, fmt.Errorf("sync compacted %s parquet: %w", signal, err) + if err := syncFile(outputPath); err != nil { + return 0, err } } - if len(marker.Signals) == 0 { - return 0, errors.New("compaction selected batches without parquet inputs") - } - if err := syncDirectory(stageDir); err != nil { - return 0, fmt.Errorf("sync compaction staging directory: %w", err) + if err := r.Parquet.PrepareReplacement(stage, marker.Output); err != nil { + return 0, err } data, err := json.Marshal(marker) if err != nil { @@ -114,7 +118,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if err := syncDirectory(r.root); err != nil { return 0, err } - recoverable = true + prepared = true if publishLock != nil { publishLock.Lock() defer publishLock.Unlock() @@ -125,62 +129,29 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches return len(selected), nil } -// compactionSources builds the replay ledger for one compaction output. Only -// raw ingest batches folded in this pass need protection: their WAL files are -// removed durably when this compaction completes, and any writer still -// retrying one of them consults this ledger. Ledgers inherited from earlier -// outputs are dropped rather than folded forward — those WALs were already -// removed when their own compaction completed — which bounds the manifest to -// one generation of batch IDs instead of the whole retention window. -func compactionSources(selected []batchMetadata) []string { - sources := make([]string, 0, len(selected)) - for _, batch := range selected { - if len(batch.Sources) == 0 { - sources = append(sources, batch.ID) - } - } - return sources -} - -type compactionKey struct { - day int64 - generation uint32 -} - -// selectCompactionBatches implements a leveled, day-partitioned compaction -// policy. An output can only be merged with peers from the same day and -// generation, so maintenance never folds the complete retained corpus into -// one perpetually young file. At most minCompactionInputs-1 files remain at -// each level for a day. -func selectCompactionBatches(batches []batchMetadata, maxBatches int) []batchMetadata { +func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) []telemetry.BatchMetadata { if maxBatches < minCompactionInputs { return nil } counts := make(map[compactionKey]int) for _, batch := range batches { - if batch.MaxNanos <= 0 { - continue + if batch.MaxIngestedNanos > 0 { + counts[compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation}]++ } - key := compactionKey{day: batch.MaxNanos / int64(24*time.Hour), generation: batch.Generation} - counts[key]++ } var chosen compactionKey found := false for key, count := range counts { - if count < minCompactionInputs { - continue - } - if !found || key.day < chosen.day || (key.day == chosen.day && key.generation < chosen.generation) { + if count >= minCompactionInputs && (!found || key.day < chosen.day || key.day == chosen.day && key.generation < chosen.generation) { chosen, found = key, true } } if !found { return nil } - selected := make([]batchMetadata, 0, min(maxBatches, counts[chosen])) + selected := make([]telemetry.BatchMetadata, 0, min(maxBatches, counts[chosen])) for _, batch := range batches { - key := compactionKey{day: batch.MaxNanos / int64(24*time.Hour), generation: batch.Generation} - if batch.MaxNanos > 0 && key == chosen { + if (compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation}) == chosen { selected = append(selected, batch) if len(selected) == maxBatches { break @@ -190,31 +161,17 @@ func selectCompactionBatches(batches []batchMetadata, maxBatches int) []batchMet return selected } -// CompactParquetBacklog drains every currently eligible compaction group so a -// maintenance interval cannot create files faster than it retires them. func (r *Repository) CompactParquetBacklog(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { total := 0 for { - compacted, err := r.CompactParquet(ctx, db, maxBatches, publishLock) - total += compacted - if err != nil || compacted == 0 { + count, err := r.CompactParquet(ctx, db, maxBatches, publishLock) + total += count + if err != nil || count == 0 { return total, err } } } -func syncFile(path string) error { - f, err := os.OpenFile(path, os.O_RDWR, 0) - if err != nil { - return err - } - if err := f.Sync(); err != nil { - _ = f.Close() - return err - } - return f.Close() -} - func (r *Repository) recoverCompaction() error { data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) if errors.Is(err, os.ErrNotExist) { @@ -230,170 +187,40 @@ func (r *Repository) recoverCompaction() error { return r.completeCompaction(marker) } -func (r *Repository) completeCompaction(marker compactionMarker) (resultErr error) { - r.commitMu.Lock() - defer r.commitMu.Unlock() - r.mu.Lock() - defer r.mu.Unlock() - stageDir := filepath.Join(r.root, marker.ID) - committed := r.batchConsumedLocked(marker.ID) - defer func() { - if resultErr != nil && !committed { - resultErr = errors.Join(resultErr, r.rollbackCompactionSwap(marker, stageDir)) - } - }() - if err := r.validateCompactionOutputs(marker, stageDir); err != nil { +func (r *Repository) completeCompaction(marker compactionMarker) error { + if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs); err != nil { return err } - for _, signal := range marker.Signals { - dir := filepath.Join(r.Parquet.Dir(), signal) - for _, id := range marker.Inputs { - input := filepath.Join(dir, id+".parquet") - retired := input + ".retired-" + marker.ID - if _, err := os.Stat(input); err == nil { - if err := renameCompactionFile(input, retired); err != nil { - return err - } - } else if !errors.Is(err, os.ErrNotExist) { - return err - } - } - stage := filepath.Join(stageDir, signal+".parquet") - final := filepath.Join(dir, marker.ID+".parquet") - if _, err := os.Stat(stage); err == nil { - if err := renameCompactionFile(stage, final); err != nil { - return err - } - } - } - if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { - return err - } - for _, source := range marker.Sources { - if err := os.Remove(filepath.Join(r.walDir, source+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove compacted input WAL %s: %w", source, err) - } - } - if err := syncDirectory(r.walDir); err != nil { - return fmt.Errorf("sync compacted input WAL removals: %w", err) - } - inputSet := make(map[string]struct{}, len(marker.Inputs)) - for _, id := range marker.Inputs { - inputSet[id] = struct{}{} - } - kept := make([]batchMetadata, 0, len(r.manifest.Batches)-len(marker.Inputs)+1) - for _, batch := range r.manifest.Batches { - if _, compacted := inputSet[batch.ID]; !compacted && batch.ID != marker.ID { - kept = append(kept, batch) - } - } - kept = append(kept, batchMetadata{ID: marker.ID, MinNanos: marker.MinNanos, MaxNanos: marker.MaxNanos, Generation: marker.Generation, Sources: append([]string(nil), marker.Sources...)}) - next := repositoryManifest{Version: repositoryVersion, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} - if err := r.checkpointManifestLocked(next); err != nil { - // checkpointManifestLocked may fail while truncating the superseded journal - // after the new snapshot is already durable and installed in memory. In that - // case the compacted files are committed and must not be rolled back. - committed = r.batchConsumedLocked(marker.ID) - return err - } - committed = true - for _, signal := range marker.Signals { - for _, id := range marker.Inputs { - _ = os.Remove(filepath.Join(r.Parquet.Dir(), signal, id+".parquet.retired-"+marker.ID)) - } - } - _ = os.RemoveAll(stageDir) - if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { + markerPath := filepath.Join(r.root, "COMPACTION.json") + if err := os.Remove(markerPath); err != nil && !errors.Is(err, os.ErrNotExist) { return err } return syncDirectory(r.root) } -func (r *Repository) rollbackCompactionSwap(marker compactionMarker, stageDir string) error { - var rollbackErr error - for _, signal := range marker.Signals { - stage := filepath.Join(stageDir, signal+".parquet") - final := filepath.Join(r.Parquet.Dir(), signal, marker.ID+".parquet") - finalExists, err := pathExists(final) - if err != nil { - rollbackErr = errors.Join(rollbackErr, err) - continue - } - if !finalExists { - continue - } - stageExists, err := pathExists(stage) - if err != nil { - rollbackErr = errors.Join(rollbackErr, err) - continue - } - if stageExists { - if err := os.Remove(final); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } - } else if err := os.Rename(final, stage); err != nil { - rollbackErr = errors.Join(rollbackErr, err) - } - } - rollbackErr = errors.Join(rollbackErr, r.restoreCompactionInputs(marker)) - if err := syncDirectory(stageDir); err != nil && !errors.Is(err, os.ErrNotExist) { - rollbackErr = errors.Join(rollbackErr, err) - } - return rollbackErr +func (r *Repository) compactionStage(id string) string { + return filepath.Join(r.root, "compaction", id) } -func (r *Repository) validateCompactionOutputs(marker compactionMarker, stageDir string) error { - if len(marker.Signals) == 0 { - return errors.New("compaction marker has no required signals") - } - for _, signal := range marker.Signals { - stage := filepath.Join(stageDir, signal+".parquet") - final := filepath.Join(r.Parquet.Dir(), signal, marker.ID+".parquet") - stageExists, err := pathExists(stage) - if err != nil { - return fmt.Errorf("inspect staged %s output: %w", signal, err) - } - finalExists, err := pathExists(final) - if err != nil { - return fmt.Errorf("inspect final %s output: %w", signal, err) - } - if !stageExists && !finalExists { - return fmt.Errorf("compaction %s is missing required %s output", marker.ID, signal) - } +func syncFile(path string) error { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err } - return nil + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() } -func (r *Repository) restoreCompactionInputs(marker compactionMarker) error { - var restoreErr error - for _, signal := range marker.Signals { - dir := filepath.Join(r.Parquet.Dir(), signal) - for _, id := range marker.Inputs { - input := filepath.Join(dir, id+".parquet") - retired := input + ".retired-" + marker.ID - retiredExists, err := pathExists(retired) - if err != nil { - restoreErr = errors.Join(restoreErr, fmt.Errorf("inspect retired %s input %s: %w", signal, id, err)) - continue - } - if !retiredExists { - continue - } - inputExists, err := pathExists(input) - if err != nil { - restoreErr = errors.Join(restoreErr, fmt.Errorf("inspect active %s input %s: %w", signal, id, err)) - continue - } - if inputExists { - restoreErr = errors.Join(restoreErr, fmt.Errorf("restore retired %s input %s: active input already exists", signal, id)) - continue - } - if err := os.Rename(retired, input); err != nil { - restoreErr = errors.Join(restoreErr, fmt.Errorf("restore retired %s input %s: %w", signal, id, err)) - } - } +func syncDirectory(path string) error { + dir, err := os.Open(path) + if err != nil { + return err } - return errors.Join(restoreErr, syncParquetDirectories(r.Parquet.Dir())) + defer dir.Close() + return dir.Sync() } func pathExists(path string) (bool, error) { @@ -409,30 +236,22 @@ func pathExists(path string) (bool, error) { func writeDurableFile(path string, data []byte) error { tmp := path + ".tmp" - file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { return err } - if _, err := file.Write(data); err != nil { - _ = file.Close() + if _, err := f.Write(data); err != nil { + _ = f.Close() return err } - if err := file.Sync(); err != nil { - _ = file.Close() + if err := f.Sync(); err != nil { + _ = f.Close() return err } - if err := file.Close(); err != nil { + if err := f.Close(); err != nil { return err } return os.Rename(tmp, path) } func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } - -func syncParquetDirectories(root string) error { - var err error - for _, signal := range parquetSignals { - err = errors.Join(err, syncDirectory(filepath.Join(root, signal))) - } - return err -} diff --git a/internal/telemetry/store/publication_test.go b/internal/telemetry/store/publication_test.go deleted file mode 100644 index 557cef51..00000000 --- a/internal/telemetry/store/publication_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package store - -import ( - "os" - "path/filepath" - "testing" - - "github.com/labstack/fanout/internal/telemetry" -) - -type publicationInspectLock struct { - t *testing.T - repository *Repository - id string - locked bool - unlockedBeforeHot bool -} - -func (l *publicationInspectLock) Lock() { - l.locked = true - for _, signal := range []string{"spans", "logs", "metrics"} { - final := filepath.Join(l.repository.Parquet.Dir(), signal, l.id+".parquet") - if _, err := os.Stat(final); !os.IsNotExist(err) { - l.t.Fatalf("%s became visible before publication lock: %v", signal, err) - } - if _, err := os.Stat(final + ".pending"); err != nil { - l.t.Fatalf("%s was not durably staged before publication lock: %v", signal, err) - } - } -} - -func (l *publicationInspectLock) Unlock() { - if rows := l.repository.Spans.RowCount(); rows != 0 { - l.t.Fatalf("hot segment encoded while query publication gate was held: %d rows", rows) - } - l.unlockedBeforeHot = true - l.locked = false -} - -func TestCommitStagesOutsidePublicationLock(t *testing.T) { - repository, err := Open(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - const id = "atomic-batch" - lock := &publicationInspectLock{t: t, repository: repository, id: id} - repository.SetParquetPublishLock(lock) - batch := Batch{ - ID: id, - Spans: []telemetry.Span{{Namespace: "default", TraceID: "00000000000000000000000000000001", SpanID: "0000000000000001"}}, - Logs: []telemetry.Log{{Namespace: "default", Body: "body"}}, - Metrics: []telemetry.Metric{{Namespace: "default", Name: "metric"}}, - } - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - if lock.locked { - t.Fatal("publication lock remained held after commit") - } - if !lock.unlockedBeforeHot { - t.Fatal("publication lock was not released before hot-index encoding") - } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(repository.Parquet.Dir(), signal, id+".parquet")); err != nil { - t.Fatalf("%s final file: %v", signal, err) - } - } -} diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index c8825aef..3368b98a 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -1,26 +1,17 @@ -// Package store owns Fanout's authoritative telemetry commit path: a -// replayable ingest WAL, immutable hot segments, and open Parquet files. package store import ( - "bytes" - "encoding/gob" - "encoding/json" + "context" "errors" "fmt" - "io" - "log/slog" "math" "os" "path/filepath" - "sort" "strings" "sync" "time" - "github.com/klauspost/compress/zstd" "github.com/labstack/fanout/internal/telemetry" - "github.com/labstack/fanout/internal/telemetry/segment" ) type Batch struct { @@ -30,757 +21,150 @@ type Batch struct { Metrics []telemetry.Metric } -const ( - maxBatchRows = 50_000 - walDecoderMaxMemory = 128 << 20 - repositoryVersion = 2 - manifestCheckpointRecords = 4_096 -) - -type batchMetadata struct { - ID string `json:"id"` - MinNanos int64 `json:"min_nanos"` - MaxNanos int64 `json:"max_nanos"` - Generation uint32 `json:"generation"` - // Sources retains the raw ingest batch IDs folded into a compacted output - // by its own compaction pass. It is a one-generation replay ledger: a stale - // WAL can never resurrect rows already present in this output, and earlier - // generations need no entries because their WALs were removed durably when - // their own compaction completed. - Sources []string `json:"sources,omitempty"` -} - -type repositoryManifest struct { - Version uint32 `json:"version"` - Epoch uint64 `json:"epoch"` - HotCutoffNanos int64 `json:"hot_cutoff_nanos"` - Batches []batchMetadata `json:"batches"` -} - -type repositoryJournalRecord struct { - Epoch uint64 `json:"epoch"` - Batch batchMetadata `json:"batch"` -} +const maxBatchRows = 50_000 +// Repository publishes self-contained Parquet batch directories. The +// directory rename is the transaction and the filesystem is the catalog. type Repository struct { - mu sync.RWMutex - // hotMu makes the persisted prune watermark and the hot-segment snapshot one - // atomic read boundary. A query can never observe an old watermark after the - // corresponding segments have been retired. - hotMu sync.RWMutex - stageMu sync.Mutex - commitMu sync.Mutex - compactionMu sync.Mutex - parquetPublish sync.Locker - root string - walDir string - Spans *segment.Store - Parquet *telemetry.ParquetStore - manifest repositoryManifest - consumed map[string]struct{} - journal *os.File - journalRecords int -} - -// SetParquetPublishLock connects repository publication to the query engine's -// read gate. It must be called during startup, before the commit worker runs. -func (r *Repository) SetParquetPublishLock(lock sync.Locker) { - r.parquetPublish = lock + root string + Parquet *telemetry.ParquetStore + compactionMu sync.Mutex } func Open(root string) (*Repository, error) { - walDir := filepath.Join(root, "wal") - for _, dir := range []string{root, walDir, filepath.Join(root, "hot", "spans")} { - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - } - spans, err := openHotStore(root) - hotRebuilt := false - if err != nil { - quarantine := filepath.Join(root, fmt.Sprintf("hot.corrupt-%d", time.Now().UnixNano())) - if renameErr := os.Rename(filepath.Join(root, "hot"), quarantine); renameErr != nil { - return nil, errors.Join(fmt.Errorf("open hot telemetry tier: %w", err), fmt.Errorf("quarantine corrupt hot tier: %w", renameErr)) - } - if syncErr := syncDirectory(root); syncErr != nil { - return nil, fmt.Errorf("sync quarantined hot tier: %w", syncErr) - } - spans, err = openHotStore(root) - if err != nil { - return nil, fmt.Errorf("rebuild hot telemetry tier: %w", err) - } - hotRebuilt = true - slog.Warn("corrupt hot telemetry tier quarantined and reset; authoritative Parquet preserved", "path", quarantine) + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, err } - parquet, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) + parquetStore, err := telemetry.OpenParquetStore(filepath.Join(root, "parquet")) if err != nil { - _ = spans.Close() return nil, err } - r := &Repository{root: root, walDir: walDir, Spans: spans, Parquet: parquet} - if err := r.loadManifest(); err != nil { + r := &Repository{root: root, Parquet: parquetStore} + if err := r.recoverCompaction(); err != nil { _ = r.Close() - return nil, fmt.Errorf("load telemetry manifest: %w", err) - } - if hotRebuilt { - r.manifest.HotCutoffNanos = max(r.manifest.HotCutoffNanos, time.Now().UnixNano()) - if err := r.checkpointManifestLocked(r.manifest); err != nil { - _ = r.Close() - return nil, fmt.Errorf("publish rebuilt hot-tier cutoff: %w", err) - } + return nil, fmt.Errorf("recover Parquet compaction: %w", err) } - if err := r.recoverCompaction(); err != nil { + if err := r.cleanupCompactionArtifacts(); err != nil { _ = r.Close() - return nil, fmt.Errorf("recover parquet compaction: %w", err) + return nil, fmt.Errorf("clean Parquet compaction staging: %w", err) } - if err := r.recover(); err != nil { + if err := r.Parquet.CleanupRetired(); err != nil { _ = r.Close() - return nil, fmt.Errorf("recover telemetry WAL: %w", err) + return nil, fmt.Errorf("clean retired Parquet batches: %w", err) } return r, nil } -func openHotStore(root string) (*segment.Store, error) { - return segment.Open(filepath.Join(root, "hot", "spans")) -} - -func (r *Repository) Close() error { - var journalErr error - if r.journal != nil { - journalErr = r.journal.Close() - r.journal = nil - } - return errors.Join(journalErr, r.Spans.Close()) -} - -// PruneHot removes acceleration segments older than cutoff. Parquet remains -// authoritative for longer retention and SQL queries. -func (r *Repository) PruneHot(cutoff int64) (int, error) { - r.hotMu.Lock() - defer r.hotMu.Unlock() - - // Publish the boundary before retiring segments. A crash or partial prune can - // therefore create only harmless overlap (Parquet below the boundary and hot - // segments above it), never a hole after restart. - publishCutoff := func() error { - r.mu.Lock() - defer r.mu.Unlock() - if cutoff > r.manifest.HotCutoffNanos { - next := cloneRepositoryManifest(r.manifest) - next.HotCutoffNanos = cutoff - if err := r.checkpointManifestLocked(next); err != nil { - return err - } - } - return nil - } - if err := publishCutoff(); err != nil { - return 0, fmt.Errorf("publish hot prune cutoff: %w", err) - } - - return r.Spans.PruneBefore(cutoff) -} - -// CompactHot drains committed raw span-index segments into larger immutable -// files. It intentionally excludes any segment not present in the -// repository manifest, because that file may belong to a partially applied WAL -// transaction that still needs exact-ID replay. -func (r *Repository) CompactHot(maxInputs int) (int, error) { - if maxInputs < 2 { - return 0, nil - } - r.mu.RLock() - committed := make(map[string]struct{}, len(r.manifest.Batches)) - for _, batch := range r.manifest.Batches { - committed[batch.ID] = struct{}{} - for _, source := range batch.Sources { - committed[source] = struct{}{} - } +func (r *Repository) cleanupCompactionArtifacts() error { + if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { + return err } - r.mu.RUnlock() - total := 0 - var compactErr error - for { - n, err := r.Spans.CompactCommitted(committed, maxInputs) - total += n - compactErr = errors.Join(compactErr, err) - if err != nil || n < 2 { - break - } + if err := os.Remove(filepath.Join(r.root, "COMPACTION.json.tmp")); err != nil && !errors.Is(err, os.ErrNotExist) { + return err } - return total, compactErr -} - -// HotTrace returns the hot trace snapshot and the durable prune boundary that -// was in force for that snapshot. -func (r *Repository) HotTrace(traceID string) ([]telemetry.Span, int64, error) { - r.hotMu.RLock() - defer r.hotMu.RUnlock() - r.mu.RLock() - cutoff := r.manifest.HotCutoffNanos - r.mu.RUnlock() - spans, err := r.Spans.Trace(traceID) - return spans, cutoff, err + return syncDirectory(r.root) } -// PruneParquet removes complete ingest batches older than cutoff. A batch that -// straddles the boundary is retained intact, so retention never removes newer -// telemetry from another signal in the same atomic commit. -func (r *Repository) PruneParquet(cutoff int64) (int, error) { - r.commitMu.Lock() - defer r.commitMu.Unlock() - r.mu.Lock() - defer r.mu.Unlock() - kept := make([]batchMetadata, 0, len(r.manifest.Batches)) - removed := 0 - var removeErr error - for _, batch := range r.manifest.Batches { - if batch.MaxNanos <= 0 || batch.MaxNanos >= cutoff { - kept = append(kept, batch) - continue - } - batchOK := true - for _, signal := range []string{"spans", "logs", "metrics"} { - path := filepath.Join(r.Parquet.Dir(), signal, batch.ID+".parquet") - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - removeErr = errors.Join(removeErr, err) - batchOK = false - } - } - if batchOK { - removed++ - } else { - kept = append(kept, batch) - } - } - if removed == 0 { - return 0, removeErr - } - if err := syncParquetDirectories(r.Parquet.Dir()); err != nil { - return 0, errors.Join(removeErr, err) - } - next := repositoryManifest{Version: repositoryVersion, HotCutoffNanos: r.manifest.HotCutoffNanos, Batches: kept} - if err := r.checkpointManifestLocked(next); err != nil { - return 0, errors.Join(removeErr, err) - } - return removed, removeErr -} +func (r *Repository) Close() error { return r.Parquet.Close() } -// Commit durably records a batch and publishes its three signal projections -// exactly once. A crash at any point leaves the WAL for replay on next boot. func (r *Repository) Commit(batch Batch) error { normalizeBatch(&batch) if err := validateBatch(batch); err != nil { return err } - r.stageMu.Lock() - err := r.writeWAL(batch) - r.stageMu.Unlock() - if err != nil { - return err - } - // Parquet encoding and fsync happen before either the query publication gate - // or commit mutex is acquired. The query gate covers only the final Parquet - // renames; the hot index, journal, and WAL cleanup cannot block readers. - if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { - return err - } - publishLocked := false - unlockPublish := func() { - if publishLocked { - r.parquetPublish.Unlock() - publishLocked = false - } - } - if r.parquetPublish != nil { - r.parquetPublish.Lock() - publishLocked = true - defer unlockPublish() - } - r.commitMu.Lock() - defer r.commitMu.Unlock() - if r.batchConsumedLocked(batch.ID) { - unlockPublish() - return errors.Join(r.Parquet.DiscardBatch(batch.ID), r.removeWAL(batch.ID)) - } - _, err = r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0) - unlockPublish() - if err != nil { - return fmt.Errorf("publish parquet batch: %w", err) - } - err = r.publishHot(batch) - if err == nil { - r.mu.Lock() - err = r.recordBatch(batch) - r.mu.Unlock() - } - if err != nil { - return err + metadata := telemetry.BatchMetadata{ + ID: batch.ID, MinIngestedNanos: batchMinIngestedNanos(batch), MaxIngestedNanos: batchMaxIngestedNanos(batch), } - return r.removeWAL(batch.ID) + return r.Parquet.CommitBatch(metadata, batch.Spans, batch.Logs, batch.Metrics) } -// Stage durably records a batch in the WAL without publishing its projections. -// Writers call this before handing a batch to an asynchronous commit worker, so -// every queued or in-flight batch is replayable if shutdown interrupts retries. -func (r *Repository) Stage(batch Batch) error { - normalizeBatch(&batch) - if err := validateBatch(batch); err != nil { - return err - } - // WAL publication is independent from projection publication. Keeping this - // lock separate lets the next OTLP request become durable while the commit - // worker writes span indexes and Parquet for an earlier request. - r.stageMu.Lock() - defer r.stageMu.Unlock() - consumed := r.batchConsumed(batch.ID) - if consumed { - return r.removeWAL(batch.ID) - } - return r.writeWAL(batch) +func (r *Repository) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) { + return r.Parquet.Trace(ctx, query) +} + +func (r *Repository) RowCount() uint64 { return r.Parquet.RowCount() } + +func (r *Repository) PruneParquet(cutoff int64) (int, error) { + r.compactionMu.Lock() + defer r.compactionMu.Unlock() + return r.Parquet.PruneBefore(cutoff) } -// validateBatch rejects a batch no projection could ever publish. The segment -// stores name their files after the batch ID, so an ID they would refuse must -// be caught before the WAL promises to replay it forever. func validateBatch(batch Batch) error { if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { return errors.New("telemetry batch requires a safe ID") } - if !segment.ValidID(batch.ID) { - return fmt.Errorf("telemetry batch ID %q cannot name a segment", batch.ID) - } - if rows := len(batch.Spans) + len(batch.Logs) + len(batch.Metrics); rows > maxBatchRows { + if rows := batchRows(batch); rows == 0 || rows > maxBatchRows { return fmt.Errorf("telemetry batch has %d rows; maximum is %d", rows, maxBatchRows) } - if err := segment.ValidateSpanRows(batch.Spans); err != nil { - return fmt.Errorf("telemetry batch cannot be represented by the hot span tier: %w", err) - } - return nil -} - -func (r *Repository) publishHot(batch Batch) error { - // Parquet is authoritative and already queryable. If the disposable hot - // index fails, retain the WAL and retry/recover only that acceleration copy; - // a hot miss safely falls back to Parquet in the meantime. - r.hotMu.Lock() - defer r.hotMu.Unlock() - if err := r.Spans.AppendID(batch.ID, batch.Spans); err != nil { - return fmt.Errorf("commit span segment: %w", err) - } return nil } -func (r *Repository) writeWAL(batch Batch) error { - final := filepath.Join(r.walDir, batch.ID+".wal") - if _, err := os.Stat(final); err == nil { - return nil - } else if !errors.Is(err, os.ErrNotExist) { - return err - } - data, err := encodeWALBatch(batch, walDecoderMaxMemory) - if err != nil { - return err - } - tmp := final + ".tmp" - _ = os.Remove(tmp) - f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return err - } - if _, err := f.Write(data); err != nil { - _ = f.Close() - return err - } - if err := f.Sync(); err != nil { - _ = f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - return err - } - return syncDirectory(r.walDir) -} - -func encodeWALBatch(batch Batch, maxDecodedBytes int) ([]byte, error) { - var plain bytes.Buffer - if err := gob.NewEncoder(&plain).Encode(batch); err != nil { - return nil, err - } - if plain.Len() > maxDecodedBytes { - return nil, fmt.Errorf("telemetry batch encodes to %d bytes; maximum is %d", plain.Len(), maxDecodedBytes) - } - enc, err := zstd.NewWriter(nil, zstd.WithEncoderCRC(true), zstd.WithEncoderConcurrency(1)) - if err != nil { - return nil, err - } - data := enc.EncodeAll(plain.Bytes(), nil) - enc.Close() - return data, nil -} - -// newWALDecoder builds the bounded decoder every WAL read goes through. -func newWALDecoder() (*zstd.Decoder, error) { - return newWALDecoderWithLimit(walDecoderMaxMemory) -} - -func newWALDecoderWithLimit(limit uint64) (*zstd.Decoder, error) { - return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(limit)) -} - -func (r *Repository) recover() error { - entries, err := os.ReadDir(r.walDir) - if err != nil { - return err - } - var names []string - for _, entry := range entries { - if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".wal") { - names = append(names, entry.Name()) - } - } - sort.Strings(names) - dec, err := newWALDecoder() - if err != nil { - return err - } - defer dec.Close() - for _, name := range names { - data, err := os.ReadFile(filepath.Join(r.walDir, name)) - if err != nil { - return err - } - plain, err := dec.DecodeAll(data, nil) - if err != nil { - if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { - return quarantineErr - } - continue - } - var batch Batch - if err := gob.NewDecoder(bytes.NewReader(plain)).Decode(&batch); err != nil { - if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { - return quarantineErr - } - continue - } - if r.batchConsumed(batch.ID) { - if err := r.removeWAL(batch.ID); err != nil { - return fmt.Errorf("remove consumed replay %s: %w", name, err) - } - continue - } - // Poison is decided before anything is published: a payload the - // projections can never accept is moved aside so it cannot abort every - // boot. A publication or manifest failure after that point is environmental, - // so the WAL is retained and startup fails loudly — a later healthy boot - // must still be able to finish the batch, including one whose projection - // prefix this attempt already published. - normalizeBatch(&batch) - if err := validateBatch(batch); err != nil { - if quarantineErr := quarantineWAL(r.walDir, name, err); quarantineErr != nil { - return errors.Join(fmt.Errorf("replay %s: %w", name, err), quarantineErr) - } - continue - } - if err := r.Parquet.StageBatch(batch.ID, batch.Spans, batch.Logs, batch.Metrics); err != nil { - return fmt.Errorf("stage replay %s: %w", name, err) - } - if _, err := r.Parquet.PublishBatch(batch.ID, len(batch.Spans) > 0, len(batch.Logs) > 0, len(batch.Metrics) > 0); err != nil { - return fmt.Errorf("publish replayed Parquet %s: %w", name, err) - } - if err := r.publishHot(batch); err != nil { - return fmt.Errorf("publish replayed hot index %s: %w", name, err) - } - if err := r.recordBatch(batch); err != nil { - return fmt.Errorf("record replayed %s: %w", name, err) - } - if err := os.Remove(filepath.Join(r.walDir, name)); err != nil { - return err - } - } - return syncDirectory(r.walDir) -} - -func quarantineWAL(dir, name string, cause error) error { - source := filepath.Join(dir, name) - target := source + ".corrupt" - if _, err := os.Stat(target); err == nil { - target = fmt.Sprintf("%s.%d", target, time.Now().UnixNano()) - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("inspect WAL quarantine target: %w", err) - } - if err := os.Rename(source, target); err != nil { - return fmt.Errorf("quarantine corrupt WAL %s: %w", name, err) - } - if err := syncDirectory(dir); err != nil { - return fmt.Errorf("sync WAL quarantine: %w", err) - } - slog.Error("quarantined corrupt telemetry WAL", "file", filepath.Base(target), "error", cause) - return nil -} - -func (r *Repository) loadManifest() error { - path := filepath.Join(r.root, "MANIFEST.json") - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - r.manifest = repositoryManifest{Version: repositoryVersion, Epoch: 1} - if err := writeRepositoryManifest(r.root, r.manifest); err != nil { - return err - } - } else if err != nil { - return err - } else { - if err := json.Unmarshal(data, &r.manifest); err != nil { - return err +func normalizeBatch(batch *Batch) { + ingestedAt := time.Now().UnixNano() + for i := range batch.Spans { + batch.Spans[i].Namespace = telemetry.NormalizeNamespace(batch.Spans[i].Namespace) + if batch.Spans[i].IngestedAt == 0 { + batch.Spans[i].IngestedAt = ingestedAt } - if r.manifest.Version != repositoryVersion || r.manifest.Epoch == 0 { - return fmt.Errorf("unsupported telemetry manifest version %d epoch %d", r.manifest.Version, r.manifest.Epoch) + if batch.Spans[i].StartUnixNanos == 0 { + batch.Spans[i].StartUnixNanos = batch.Spans[i].IngestedAt } } - r.rebuildConsumedLocked() - - journalPath := filepath.Join(r.root, "MANIFEST.log") - journalData, err := os.ReadFile(journalPath) - journalNew := errors.Is(err, os.ErrNotExist) - if err != nil && !journalNew { - return err - } - validBytes := 0 - for validBytes < len(journalData) { - relativeEnd := bytes.IndexByte(journalData[validBytes:], '\n') - if relativeEnd < 0 { - break - } - lineStart := validBytes - lineEnd := lineStart + relativeEnd - line := journalData[lineStart:lineEnd] - validBytes = lineEnd + 1 - if len(bytes.TrimSpace(line)) == 0 { - continue - } - var record repositoryJournalRecord - if err := json.Unmarshal(line, &record); err != nil { - return fmt.Errorf("decode telemetry manifest journal at byte %d: %w", lineStart, err) - } - if record.Epoch != r.manifest.Epoch || r.batchConsumedLocked(record.Batch.ID) { - continue + for i := range batch.Logs { + batch.Logs[i].Namespace = telemetry.NormalizeNamespace(batch.Logs[i].Namespace) + if batch.Logs[i].IngestedAt == 0 { + batch.Logs[i].IngestedAt = ingestedAt } - if record.Batch.ID == "" || !segment.ValidID(record.Batch.ID) { - return fmt.Errorf("telemetry manifest journal contains invalid batch ID %q", record.Batch.ID) + if batch.Logs[i].EventUnixNanos == 0 { + batch.Logs[i].EventUnixNanos = firstNonzero(batch.Logs[i].TimeUnixNanos, batch.Logs[i].ObservedTimeNanos, batch.Logs[i].IngestedAt) } - r.manifest.Batches = append(r.manifest.Batches, record.Batch) - r.addConsumedLocked(record.Batch) - r.journalRecords++ } - r.journal, err = os.OpenFile(journalPath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644) - if err != nil { - return err - } - if validBytes != len(journalData) { - if err := r.journal.Truncate(int64(validBytes)); err != nil { - return fmt.Errorf("truncate partial telemetry manifest journal: %w", err) - } - if err := r.journal.Sync(); err != nil { - return fmt.Errorf("sync repaired telemetry manifest journal: %w", err) + for i := range batch.Metrics { + batch.Metrics[i].Namespace = telemetry.NormalizeNamespace(batch.Metrics[i].Namespace) + if batch.Metrics[i].IngestedAt == 0 { + batch.Metrics[i].IngestedAt = ingestedAt } - } - if journalNew { - return syncDirectory(r.root) - } - return nil -} - -func (r *Repository) recordBatch(batch Batch) error { - if r.batchConsumedLocked(batch.ID) { - return nil - } - metadata := batchMetadata{ID: batch.ID, MinNanos: batchMinNanos(batch), MaxNanos: batchMaxNanos(batch)} - line, err := json.Marshal(repositoryJournalRecord{Epoch: r.manifest.Epoch, Batch: metadata}) - if err != nil { - return err - } - line = append(line, '\n') - if r.journal == nil { - return errors.New("telemetry manifest journal is closed") - } - if _, err := r.journal.Write(line); err != nil { - return fmt.Errorf("append telemetry manifest journal: %w", err) - } - if err := r.journal.Sync(); err != nil { - return fmt.Errorf("sync telemetry manifest journal: %w", err) - } - r.manifest.Batches = append(r.manifest.Batches, metadata) - r.addConsumedLocked(metadata) - r.journalRecords++ - if r.journalRecords >= manifestCheckpointRecords { - if err := r.checkpointManifestLocked(r.manifest); err != nil { - return fmt.Errorf("checkpoint telemetry manifest journal: %w", err) + if batch.Metrics[i].EventUnixNanos == 0 { + batch.Metrics[i].EventUnixNanos = firstNonzero(batch.Metrics[i].TimeUnixNanos, batch.Metrics[i].IngestedAt) } } - return nil } -func (r *Repository) checkpointManifestLocked(next repositoryManifest) error { - next = cloneRepositoryManifest(next) - next.Version = repositoryVersion - next.Epoch = max(next.Epoch, r.manifest.Epoch+1) - if err := writeRepositoryManifest(r.root, next); err != nil { - return err - } - r.manifest = next - r.rebuildConsumedLocked() - r.journalRecords = 0 - if r.journal == nil { - return nil - } - if err := r.journal.Truncate(0); err != nil { - return fmt.Errorf("truncate telemetry manifest journal: %w", err) - } - if _, err := r.journal.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("rewind telemetry manifest journal: %w", err) - } - if err := r.journal.Sync(); err != nil { - return fmt.Errorf("sync telemetry manifest journal checkpoint: %w", err) - } - return nil -} - -func cloneRepositoryManifest(manifest repositoryManifest) repositoryManifest { - clone := manifest - clone.Batches = append([]batchMetadata(nil), manifest.Batches...) - for i := range clone.Batches { - clone.Batches[i].Sources = append([]string(nil), manifest.Batches[i].Sources...) - } - return clone -} - -func (r *Repository) rebuildConsumedLocked() { - r.consumed = make(map[string]struct{}, len(r.manifest.Batches)) - for _, batch := range r.manifest.Batches { - r.addConsumedLocked(batch) - } -} - -func (r *Repository) addConsumedLocked(batch batchMetadata) { - if r.consumed == nil { - r.consumed = make(map[string]struct{}) - } - r.consumed[batch.ID] = struct{}{} - for _, source := range batch.Sources { - r.consumed[source] = struct{}{} - } -} - -func (r *Repository) batchConsumed(id string) bool { - r.mu.RLock() - defer r.mu.RUnlock() - return r.batchConsumedLocked(id) -} - -func (r *Repository) batchConsumedLocked(id string) bool { - _, exists := r.consumed[id] - return exists -} - -func (r *Repository) removeWAL(id string) error { - if err := os.Remove(filepath.Join(r.walDir, id+".wal")); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove committed telemetry WAL: %w", err) - } - return syncDirectory(r.walDir) -} - -func batchMaxNanos(batch Batch) int64 { - var maxNanos int64 +func batchMaxIngestedNanos(batch Batch) int64 { + var value int64 for _, row := range batch.Spans { - maxNanos = max(maxNanos, max(row.StartUnixNanos, row.IngestedAt)) + value = max(value, row.IngestedAt) } for _, row := range batch.Logs { - maxNanos = max(maxNanos, max(row.EventUnixNanos, row.IngestedAt)) + value = max(value, row.IngestedAt) } for _, row := range batch.Metrics { - maxNanos = max(maxNanos, max(row.EventUnixNanos, row.IngestedAt)) + value = max(value, row.IngestedAt) } - return maxNanos + return value } -func batchMinNanos(batch Batch) int64 { - minNanos := int64(math.MaxInt64) - include := func(value int64) { - if value > 0 { - minNanos = min(minNanos, value) +func batchMinIngestedNanos(batch Batch) int64 { + value := int64(math.MaxInt64) + include := func(candidate int64) { + if candidate > 0 { + value = min(value, candidate) } } for _, row := range batch.Spans { - include(row.StartUnixNanos) include(row.IngestedAt) } for _, row := range batch.Logs { - include(row.EventUnixNanos) include(row.IngestedAt) } for _, row := range batch.Metrics { - include(row.EventUnixNanos) include(row.IngestedAt) } - if minNanos == math.MaxInt64 { + if value == math.MaxInt64 { return 0 } - return minNanos -} - -func writeRepositoryManifest(root string, manifest repositoryManifest) error { - data, err := json.Marshal(manifest) - if err != nil { - return err - } - tmp := filepath.Join(root, "MANIFEST.json.tmp") - final := filepath.Join(root, "MANIFEST.json") - file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - if err != nil { - return err - } - if _, err := file.Write(data); err != nil { - _ = file.Close() - return err - } - if err := file.Sync(); err != nil { - _ = file.Close() - return err - } - if err := file.Close(); err != nil { - return err - } - if err := os.Rename(tmp, final); err != nil { - return err - } - return syncDirectory(root) -} - -func normalizeBatch(batch *Batch) { - for i := range batch.Spans { - batch.Spans[i].Namespace = telemetry.NormalizeNamespace(batch.Spans[i].Namespace) - if batch.Spans[i].StartUnixNanos == 0 { - // Mirror the Parquet start_time coalesce so hot segments and SQL scans - // key a zero-start span on the same instant. - batch.Spans[i].StartUnixNanos = batch.Spans[i].IngestedAt - } - } - for i := range batch.Logs { - batch.Logs[i].Namespace = telemetry.NormalizeNamespace(batch.Logs[i].Namespace) - if batch.Logs[i].EventUnixNanos == 0 { - batch.Logs[i].EventUnixNanos = firstNonzero(batch.Logs[i].TimeUnixNanos, batch.Logs[i].ObservedTimeNanos, batch.Logs[i].IngestedAt) - } - } - for i := range batch.Metrics { - batch.Metrics[i].Namespace = telemetry.NormalizeNamespace(batch.Metrics[i].Namespace) - if batch.Metrics[i].EventUnixNanos == 0 { - batch.Metrics[i].EventUnixNanos = firstNonzero(batch.Metrics[i].TimeUnixNanos, batch.Metrics[i].IngestedAt) - } - } + return value } func firstNonzero(values ...int64) int64 { @@ -791,15 +175,3 @@ func firstNonzero(values ...int64) int64 { } return 0 } - -func syncDirectory(dir string) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - if err := f.Sync(); err != nil && !errors.Is(err, io.ErrClosedPipe) { - return err - } - return nil -} diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index a67b81b7..4124cbad 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -4,14 +4,10 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" - "github.com/klauspost/compress/zstd" "os" "path/filepath" - "strings" "testing" - "time" _ "github.com/duckdb/duckdb-go/v2" "github.com/labstack/fanout/internal/telemetry" @@ -19,14 +15,17 @@ import ( func testBatch() Batch { return Batch{ - ID: "0198f4a0-test-batch", - Spans: []telemetry.Span{{Namespace: "", TraceID: "trace-1", SpanID: "span-1", ServiceName: "api", Name: "GET /", StartUnixNanos: 100, EndUnixNanos: 200, DurationMS: .0001, StatusCode: "OK", IngestedAt: 300}}, + ID: "batch-test", + Spans: []telemetry.Span{{ + TraceID: "trace-1", SpanID: "span-1", ServiceName: "api", Name: "GET /", + StartUnixNanos: 100, EndUnixNanos: 200, DurationMS: 0.0001, StatusCode: "OK", IngestedAt: 300, + }}, Logs: []telemetry.Log{{TimeUnixNanos: 110, Severity: "INFO", Body: "ready", ServiceName: "api", TraceID: "trace-1", IngestedAt: 300}}, Metrics: []telemetry.Metric{{TimeUnixNanos: 120, Name: "requests", Type: "sum", ServiceName: "api", Value: 1, IngestedAt: 300}}, } } -func TestRepositoryCommitIsIdempotentAndQueryable(t *testing.T) { +func TestRepositoryCommitIsIdempotentDurableAndQueryable(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { @@ -37,250 +36,103 @@ func TestRepositoryCommitIsIdempotentAndQueryable(t *testing.T) { t.Fatal(err) } if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - if got := repository.Spans.RowCount(); got != 1 { - t.Fatalf("span rows = %d", got) + t.Fatalf("idempotent commit: %v", err) } - for _, signal := range []string{"logs", "metrics"} { - if _, err := os.Stat(filepath.Join(dir, "hot", signal)); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("unused hot %s copy exists: %v", signal, err) - } + if got := repository.RowCount(); got != 3 { + t.Fatalf("rows = %d, want 3", got) } if err := repository.Close(); err != nil { t.Fatal(err) } - db, err := sql.Open("duckdb", "") + reopened, err := Open(dir) if err != nil { t.Fatal(err) } + defer reopened.Close() + if got := reopened.RowCount(); got != 3 { + t.Fatalf("reopened rows = %d, want 3", got) + } + spans, err := traceAll(reopened, "trace-1") + if err != nil || len(spans) != 1 || spans[0].Namespace != "default" { + t.Fatalf("reopened trace = %#v, %v", spans, err) + } + + db := openTestDuckDB(t) defer db.Close() for _, signal := range []string{"spans", "logs", "metrics"} { var count int - pattern := filepath.ToSlash(filepath.Join(dir, "parquet", signal, "*.parquet")) - if err := db.QueryRowContext(context.Background(), "SELECT count(*) FROM read_parquet(?)", pattern).Scan(&count); err != nil { - t.Fatal(err) + if err := db.QueryRowContext(context.Background(), "SELECT count(*) FROM read_parquet(?)", reopened.Parquet.Pattern(signal)).Scan(&count); err != nil { + t.Fatalf("query %s: %v", signal, err) } if count != 1 { - t.Fatalf("%s parquet rows = %d", signal, count) + t.Fatalf("%s rows = %d, want 1", signal, count) } } } -func TestRepositoryCommitIODoesNotHoldMetadataLock(t *testing.T) { +func TestRepositoryHasOneAuthoritativeStorageLayout(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } defer repository.Close() - batch := testBatch() - batch.ID = "lock-scope-batch" - repository.mu.Lock() - committed := make(chan error, 1) - go func() { committed <- repository.Commit(batch) }() - parquet := filepath.Join(dir, "parquet", "spans", batch.ID+".parquet") - deadline := time.Now().Add(2 * time.Second) - for { - if _, err := os.Stat(parquet); err == nil { - break - } - if time.Now().After(deadline) { - repository.mu.Unlock() - t.Fatal("commit projection I/O remained blocked by repository metadata lock") - } - time.Sleep(time.Millisecond) - } - select { - case err := <-committed: - repository.mu.Unlock() - t.Fatalf("Commit returned before metadata publication lock was released: %v", err) - default: - } - repository.mu.Unlock() - if err := <-committed; err != nil { - t.Fatal(err) - } -} - -func TestRepositoryStageDoesNotWaitForProjectionCommitLock(t *testing.T) { - repository, err := Open(t.TempDir()) - if err != nil { + if err := repository.Commit(testBatch()); err != nil { t.Fatal(err) } - defer repository.Close() - repository.commitMu.Lock() - staged := make(chan error, 1) - go func() { - staged <- repository.Stage(Batch{ID: "independent-stage", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}) - }() - select { - case err := <-staged: - if err != nil { - t.Fatal(err) + for _, removed := range []string{"wal", "hot", "MANIFEST.json", "MANIFEST.log", "ducklake.sqlite"} { + if _, err := os.Stat(filepath.Join(dir, removed)); !os.IsNotExist(err) { + t.Fatalf("removed storage artifact %s exists: %v", removed, err) } - case <-time.After(2 * time.Second): - repository.commitMu.Unlock() - t.Fatal("WAL staging waited for projection commit I/O") } - repository.commitMu.Unlock() -} - -func TestCompactHotDoesNotTakeRepositoryIngestOrReadLocks(t *testing.T) { - repository, err := Open(t.TempDir()) + entries, err := os.ReadDir(repository.Parquet.BatchPath("batch-test")) if err != nil { t.Fatal(err) } - defer repository.Close() - repository.hotMu.Lock() - repository.commitMu.Lock() - done := make(chan error, 1) - go func() { - _, err := repository.CompactHot(2) - done <- err - }() - select { - case err := <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(2 * time.Second): - repository.commitMu.Unlock() - repository.hotMu.Unlock() - t.Fatal("hot compaction acquired a repository-wide ingest or read lock") + if len(entries) != 5 { + t.Fatalf("batch contains %d files, want Parquet signals, trace index, and metadata", len(entries)) } - repository.commitMu.Unlock() - repository.hotMu.Unlock() } -func TestRepositoryManifestJournalReplaysAndRepairsPartialTail(t *testing.T) { +func TestRepositoryCleansUnpublishedCompactionArtifacts(t *testing.T) { dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - for i := range 3 { - batch := testBatch() - batch.ID = fmt.Sprintf("journal-%d", i) - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - } - var snapshot repositoryManifest - data, err := os.ReadFile(filepath.Join(dir, "MANIFEST.json")) - if err != nil { - t.Fatal(err) - } - if err := json.Unmarshal(data, &snapshot); err != nil { - t.Fatal(err) - } - if len(snapshot.Batches) != 0 { - t.Fatalf("per-commit path rewrote manifest snapshot with %d batches", len(snapshot.Batches)) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - journalPath := filepath.Join(dir, "MANIFEST.log") - journal, err := os.OpenFile(journalPath, os.O_APPEND|os.O_WRONLY, 0o644) - if err != nil { + staging := filepath.Join(dir, "compaction", "orphan") + if err := os.MkdirAll(staging, 0o755); err != nil { t.Fatal(err) } - if _, err := journal.WriteString(`{"epoch":1,"batch":`); err != nil { + temporaryMarker := filepath.Join(dir, "COMPACTION.json.tmp") + if err := os.WriteFile(temporaryMarker, []byte("partial"), 0o600); err != nil { t.Fatal(err) } - if err := journal.Close(); err != nil { - t.Fatal(err) - } - - reopened, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer reopened.Close() - if len(reopened.manifest.Batches) != 3 || !reopened.batchConsumed("journal-2") { - t.Fatalf("journal replay batches = %#v", reopened.manifest.Batches) - } - repaired, err := os.ReadFile(journalPath) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(repaired), `"batch":`) && !strings.HasSuffix(string(repaired), "}\n") { - t.Fatalf("partial journal tail was not truncated: %q", repaired) - } -} - -func TestRepositoryReplaysDurableWAL(t *testing.T) { - dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } - batch := testBatch() - if err := repository.writeWAL(batch); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - - recovered, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer recovered.Close() - if recovered.Spans.RowCount() != 1 { - t.Fatal("WAL recovery did not restore the span index") - } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, batch.ID+".parquet")); err != nil { - t.Fatalf("WAL recovery did not restore %s parquet: %v", signal, err) + defer repository.Close() + for _, path := range []string{filepath.Join(dir, "compaction"), temporaryMarker} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("unpublished compaction artifact remains at %s: %v", path, err) } } - entries, err := filepath.Glob(filepath.Join(dir, "wal", "*.wal")) - if err != nil { - t.Fatal(err) - } - if len(entries) != 0 { - t.Fatalf("committed WAL files remain: %v", entries) - } } -func TestRepositoryQuarantinesCorruptWAL(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "wal", "poison.wal"), []byte("not-zstd"), 0o600); err != nil { - t.Fatal(err) - } - recovered, err := Open(dir) - if err != nil { - t.Fatalf("Open with poison WAL: %v", err) - } - defer recovered.Close() - if _, err := os.Stat(filepath.Join(dir, "wal", "poison.wal.corrupt")); err != nil { - t.Fatalf("quarantined WAL missing: %v", err) - } -} - -func TestRepositoryPrunesOnlyCompleteExpiredParquetBatches(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) +func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { + repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) } defer repository.Close() old := testBatch() - old.ID = "old-batch" - old.Spans[0].IngestedAt, old.Logs[0].IngestedAt, old.Metrics[0].IngestedAt = 100, 100, 100 + old.ID = "old" + old.Spans[0].StartUnixNanos, old.Spans[0].IngestedAt = 100, 100 + old.Logs[0].TimeUnixNanos, old.Logs[0].IngestedAt = 100, 100 + old.Metrics[0].TimeUnixNanos, old.Metrics[0].IngestedAt = 100, 100 newer := testBatch() - newer.ID = "new-batch" - newer.Spans[0].IngestedAt, newer.Logs[0].IngestedAt, newer.Metrics[0].IngestedAt = 1000, 1000, 1000 + newer.ID = "new" + newer.Spans[0].StartUnixNanos, newer.Spans[0].IngestedAt = 1_000, 1_000 + newer.Logs[0].TimeUnixNanos, newer.Logs[0].IngestedAt = 1_000, 1_000 + newer.Metrics[0].TimeUnixNanos, newer.Metrics[0].IngestedAt = 1_000, 1_000 if err := repository.Commit(old); err != nil { t.Fatal(err) } @@ -288,303 +140,135 @@ func TestRepositoryPrunesOnlyCompleteExpiredParquetBatches(t *testing.T) { t.Fatal(err) } removed, err := repository.PruneParquet(500) - if err != nil { - t.Fatal(err) + if err != nil || removed != 1 { + t.Fatalf("prune = %d, %v", removed, err) } - if removed != 1 { - t.Fatalf("removed batches = %d, want 1", removed) + if _, err := os.Stat(repository.Parquet.BatchPath("old")); !os.IsNotExist(err) { + t.Fatalf("expired batch remains: %v", err) } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, "old-batch.parquet")); !os.IsNotExist(err) { - t.Fatalf("expired %s file remains: %v", signal, err) - } - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, "new-batch.parquet")); err != nil { - t.Fatalf("new %s file missing: %v", signal, err) - } + if _, err := os.Stat(repository.Parquet.BatchPath("new")); err != nil { + t.Fatalf("current batch missing: %v", err) + } + if got := repository.RowCount(); got != 3 { + t.Fatalf("retained rows = %d, want 3", got) } } -func TestRepositoryPersistsHotPruneBoundary(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) +func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { + repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) } + defer repository.Close() batch := testBatch() - batch.ID = "hot-boundary" - batch.Spans = []telemetry.Span{{TraceID: "trace-boundary", SpanID: "span", StartUnixNanos: 300}} + batch.ID = "future-events" + batch.Spans[0].StartUnixNanos = 1 << 62 + batch.Logs[0].TimeUnixNanos = 1 << 62 + batch.Metrics[0].TimeUnixNanos = 1 << 62 + batch.Spans[0].IngestedAt = 100 + batch.Logs[0].IngestedAt = 100 + batch.Metrics[0].IngestedAt = 100 if err := repository.Commit(batch); err != nil { t.Fatal(err) } - if _, err := repository.PruneHot(250); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - reopened, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer reopened.Close() - spans, cutoff, err := reopened.HotTrace("trace-boundary") - if err != nil { - t.Fatal(err) - } - if cutoff != 250 || len(spans) != 1 { - t.Fatalf("cutoff=%d spans=%d, want cutoff 250 and one retained boundary span", cutoff, len(spans)) + removed, err := repository.PruneParquet(500) + if err != nil || removed != 1 { + t.Fatalf("prune future-dated events = %d, %v; want one batch expired by ingest time", removed, err) } } -func TestRepositoryCompactsParquetBatchesWithoutChangingRows(t *testing.T) { +func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } defer repository.Close() - for i := range 8 { + for i := range minCompactionInputs { batch := testBatch() batch.ID = fmt.Sprintf("batch-%d", i) batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + batch.Spans[0].StartUnixNanos = int64(100 + i) if err := repository.Commit(batch); err != nil { t.Fatal(err) } } - if _, err := repository.PruneHot(50); err != nil { - t.Fatal(err) - } - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatal(err) - } + db := openTestDuckDB(t) defer db.Close() compacted, err := repository.CompactParquet(context.Background(), db, 64, nil) if err != nil { t.Fatal(err) } - if compacted != 8 { - t.Fatalf("compacted batches = %d, want 8", compacted) - } - stats, err := repository.Parquet.Stats() - if err != nil { - t.Fatal(err) + if compacted != minCompactionInputs { + t.Fatalf("compacted inputs = %d", compacted) } - if stats["spans"].Files != 1 { - t.Fatalf("span files = %d, want 1", stats["spans"].Files) + metadata := repository.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].Generation != 1 || metadata[0].Spans != minCompactionInputs || metadata[0].Logs != minCompactionInputs || metadata[0].Metrics != minCompactionInputs { + t.Fatalf("compacted metadata = %#v", metadata) } - if repository.manifest.HotCutoffNanos != 50 { - t.Fatalf("hot cutoff after compaction = %d, want 50", repository.manifest.HotCutoffNanos) + if got := repository.RowCount(); got != 3*minCompactionInputs { + t.Fatalf("compacted rows = %d", got) } - pattern := filepath.ToSlash(filepath.Join(dir, "parquet", "spans", "*.parquet")) - var rows int - if err := db.QueryRow("SELECT count(*) FROM read_parquet(?)", pattern).Scan(&rows); err != nil { - t.Fatal(err) - } - if rows != 8 { - t.Fatalf("compacted rows = %d, want 8", rows) - } -} - -func TestRepositorySkipsStaleWALForCompactedBatch(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) + spans, err := traceAll(repository, "trace-1") + if err != nil || len(spans) != minCompactionInputs { + t.Fatalf("compacted trace spans = %d, %v", len(spans), err) } - var stale Batch - for i := range 8 { - batch := testBatch() - batch.ID = fmt.Sprintf("replay-source-%d", i) - batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if i == 0 { - stale = batch - } - if err := repository.Commit(batch); err != nil { + for _, signal := range []string{"spans", "logs", "metrics"} { + var count int + if err := db.QueryRowContext(context.Background(), "SELECT count(*) FROM read_parquet(?)", repository.Parquet.Pattern(signal)).Scan(&count); err != nil { t.Fatal(err) } + if count != minCompactionInputs { + t.Fatalf("compacted %s rows = %d", signal, count) + } } - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatal(err) - } - if _, err := repository.CompactParquet(context.Background(), db, 64, nil); err != nil { - db.Close() - t.Fatal(err) - } - if err := repository.writeWAL(stale); err != nil { - db.Close() - t.Fatal(err) - } - if err := repository.Close(); err != nil { - db.Close() - t.Fatal(err) - } - recovered, err := Open(dir) - if err != nil { - db.Close() - t.Fatal(err) - } - defer recovered.Close() - defer db.Close() - if _, err := os.Stat(filepath.Join(dir, "wal", stale.ID+".wal")); !os.IsNotExist(err) { - t.Fatalf("consumed WAL remains after recovery: %v", err) - } - if _, err := os.Stat(filepath.Join(dir, "parquet", "spans", stale.ID+".parquet")); !os.IsNotExist(err) { - t.Fatalf("consumed source parquet was resurrected: %v", err) - } - pattern := filepath.ToSlash(filepath.Join(dir, "parquet", "spans", "*.parquet")) - var rows int - if err := db.QueryRow("SELECT count(*) FROM read_parquet(?)", pattern).Scan(&rows); err != nil { - t.Fatal(err) - } - if rows != 8 { - t.Fatalf("rows after stale WAL recovery = %d, want 8", rows) + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { + t.Fatalf("completed compaction marker remains: %v", err) } } -func TestRepositoryCompactionDrainsBacklog(t *testing.T) { +func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } - defer repository.Close() - for i := range 72 { + for i := range minCompactionInputs { batch := testBatch() - batch.ID = fmt.Sprintf("backlog-%d", i) + batch.ID = fmt.Sprintf("recover-%d", i) batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) if err := repository.Commit(batch); err != nil { t.Fatal(err) } } - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatal(err) - } - defer db.Close() - compacted, err := repository.CompactParquetBacklog(context.Background(), db, 64, nil) - if err != nil { - t.Fatal(err) - } - if compacted <= 64 { - t.Fatalf("compacted batches = %d, want multiple groups", compacted) - } - if len(repository.manifest.Batches) != 2 { - t.Fatalf("manifest batches = %d, want 2 bounded level-1 outputs", len(repository.manifest.Batches)) - } - for _, batch := range repository.manifest.Batches { - if batch.Generation != 1 { - t.Fatalf("batch %s generation = %d, want 1", batch.ID, batch.Generation) - } - } - again, err := repository.CompactParquet(context.Background(), db, 64, nil) - if err != nil { - t.Fatal(err) - } - if again != 0 { - t.Fatalf("second compaction rewrote %d already-compacted inputs, want 0", again) - } -} - -func TestRepositoryCompactionPreservesRetentionPartitions(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - oldTime := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC).UnixNano() - newTime := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC).UnixNano() - for day, timestamp := range []int64{oldTime, newTime} { - for i := range 8 { - batch := testBatchAt(timestamp) - batch.ID = fmt.Sprintf("day-%d-batch-%d", day, i) - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - } - } - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatal(err) - } + db := openTestDuckDB(t) defer db.Close() - if _, err := repository.CompactParquetBacklog(context.Background(), db, 64, nil); err != nil { - t.Fatal(err) - } - if len(repository.manifest.Batches) != 2 { - t.Fatalf("manifest batches = %d, want one output per day", len(repository.manifest.Batches)) - } - var oldID, newID string - for _, batch := range repository.manifest.Batches { - switch batch.MaxNanos { - case oldTime: - oldID = batch.ID - case newTime: - newID = batch.ID - } + output := telemetry.BatchMetadata{ + ID: "compact-recovery", MinIngestedNanos: 100, MaxIngestedNanos: 300, Generation: 1, + Spans: minCompactionInputs, Logs: minCompactionInputs, Metrics: minCompactionInputs, } - removed, err := repository.PruneParquet(time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC).UnixNano()) - if err != nil { + stage := repository.compactionStage(output.ID) + if err := os.MkdirAll(stage, 0o755); err != nil { t.Fatal(err) } - if removed != 1 || oldID == "" || newID == "" { - t.Fatalf("removed=%d oldID=%q newID=%q, want exactly the old partition", removed, oldID, newID) - } - for _, signal := range []string{"spans", "logs", "metrics"} { - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, oldID+".parquet")); !os.IsNotExist(err) { - t.Fatalf("expired compacted %s file remains: %v", signal, err) - } - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, newID+".parquet")); err != nil { - t.Fatalf("current compacted %s file missing: %v", signal, err) + for _, signal := range parquetSignals { + query := fmt.Sprintf("SELECT * FROM read_parquet(%s)", sqlQuote(repository.Parquet.Pattern(signal))) + if signal == "spans" { + query += " ORDER BY _trace_hash, start_unix_nano, span_id" } - } -} - -func TestRepositoryCompactionRecoveryRestoresRetiredInputsWhenStageMissing(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - marker := compactionMarker{ - ID: "compact-recovery", - Inputs: []string{"recovery-a", "recovery-b"}, - Signals: parquetSignals[:], - MinNanos: 100, - MaxNanos: 120, - Generation: 1, - } - for _, id := range marker.Inputs { - batch := testBatch() - batch.ID = id - if err := repository.Commit(batch); err != nil { + stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD)", query, sqlQuote(filepath.Join(stage, signal+".parquet"))) + if _, err := db.Exec(stmt); err != nil { t.Fatal(err) } } - stageDir := filepath.Join(dir, marker.ID) - if err := os.Mkdir(stageDir, 0o755); err != nil { + if err := repository.Parquet.PrepareReplacement(stage, output); err != nil { t.Fatal(err) } - for _, signal := range marker.Signals { - if signal != "spans" { - data, err := os.ReadFile(filepath.Join(dir, "parquet", signal, marker.Inputs[0]+".parquet")) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(stageDir, signal+".parquet"), data, 0o644); err != nil { - t.Fatal(err) - } - } - for _, id := range marker.Inputs { - input := filepath.Join(dir, "parquet", signal, id+".parquet") - if err := os.Rename(input, input+".retired-"+marker.ID); err != nil { - t.Fatal(err) - } - } + inputs := make([]string, minCompactionInputs) + for i := range inputs { + inputs[i] = fmt.Sprintf("recover-%d", i) } + marker := compactionMarker{Output: output, Inputs: inputs} data, err := json.Marshal(marker) if err != nil { t.Fatal(err) @@ -592,342 +276,46 @@ func TestRepositoryCompactionRecoveryRestoresRetiredInputsWhenStageMissing(t *te if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { t.Fatal(err) } - if err := repository.recoverCompaction(); err == nil || !strings.Contains(err.Error(), "missing required spans output") { - t.Fatalf("recover compaction error = %v, want missing spans output", err) - } - for _, signal := range marker.Signals { - for _, id := range marker.Inputs { - input := filepath.Join(dir, "parquet", signal, id+".parquet") - if _, err := os.Stat(input); err != nil { - t.Fatalf("restored %s input %s: %v", signal, id, err) - } - if _, err := os.Stat(input + ".retired-" + marker.ID); !os.IsNotExist(err) { - t.Fatalf("retired %s input %s remains: %v", signal, id, err) - } - } - if _, err := os.Stat(filepath.Join(dir, "parquet", signal, marker.ID+".parquet")); !os.IsNotExist(err) { - t.Fatalf("unexpected compacted %s output: %v", signal, err) - } - } - if len(repository.manifest.Batches) != len(marker.Inputs) { - t.Fatalf("manifest batches = %d, want original %d", len(repository.manifest.Batches), len(marker.Inputs)) - } -} - -func TestRepositoryCompactionRollsBackMidSwapFailure(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - marker := compactionMarker{ID: "compact-rollback", Inputs: []string{"rollback-a", "rollback-b"}, Signals: []string{"spans", "logs"}, MinNanos: 100, MaxNanos: 120, Generation: 1} - for _, id := range marker.Inputs { - batch := testBatch() - batch.ID = id - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - } - stageDir := filepath.Join(dir, marker.ID) - if err := os.Mkdir(stageDir, 0o755); err != nil { - t.Fatal(err) - } - for _, signal := range marker.Signals { - data, err := os.ReadFile(filepath.Join(repository.Parquet.Dir(), signal, marker.Inputs[0]+".parquet")) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(stageDir, signal+".parquet"), data, 0o644); err != nil { + // Reproduce a crash halfway through retiring inputs. Startup must finish + // the intended publication, not expose a half-old/half-new file set. + for _, id := range inputs[:4] { + if err := os.Rename(repository.Parquet.BatchPath(id), filepath.Join(repository.Parquet.BatchesDir(), id+".retired-"+output.ID)); err != nil { t.Fatal(err) } } - originalRename := renameCompactionFile - renameCompactionFile = func(oldPath, newPath string) error { - if oldPath == filepath.Join(stageDir, "logs.parquet") { - return errors.New("injected log publish failure") - } - return os.Rename(oldPath, newPath) - } - defer func() { renameCompactionFile = originalRename }() - if err := repository.completeCompaction(marker); err == nil || !strings.Contains(err.Error(), "injected log publish failure") { - t.Fatalf("complete compaction error = %v", err) - } - for _, signal := range marker.Signals { - for _, id := range marker.Inputs { - input := filepath.Join(repository.Parquet.Dir(), signal, id+".parquet") - if _, err := os.Stat(input); err != nil { - t.Fatalf("restored %s input %s: %v", signal, id, err) - } - if _, err := os.Stat(input + ".retired-" + marker.ID); !os.IsNotExist(err) { - t.Fatalf("retired %s input %s remains: %v", signal, id, err) - } - } - if _, err := os.Stat(filepath.Join(repository.Parquet.Dir(), signal, marker.ID+".parquet")); !os.IsNotExist(err) { - t.Fatalf("partial compacted %s output remains: %v", signal, err) - } - if _, err := os.Stat(filepath.Join(stageDir, signal+".parquet")); err != nil { - t.Fatalf("restaged %s output: %v", signal, err) - } - } -} -func TestRepositoryRefusesToOverwritePendingCompactionMarker(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - markerPath := filepath.Join(dir, "COMPACTION.json") - original := []byte(`{"id":"compact-pending"}`) - if err := writeDurableFile(markerPath, original); err != nil { - t.Fatal(err) - } - db, err := sql.Open("duckdb", "") - if err != nil { - t.Fatal(err) - } - defer db.Close() - if _, err := repository.CompactParquet(context.Background(), db, 64, nil); err == nil || !strings.Contains(err.Error(), "must recover") { - t.Fatalf("CompactParquet error = %v, want pending-marker refusal", err) - } - got, err := os.ReadFile(markerPath) - if err != nil { - t.Fatal(err) - } - if string(got) != string(original) { - t.Fatalf("pending marker was overwritten: %q", got) - } -} - -func testBatchAt(timestamp int64) Batch { - batch := testBatch() - batch.Spans[0].StartUnixNanos = timestamp - batch.Spans[0].EndUnixNanos = timestamp + 1 - batch.Spans[0].IngestedAt = timestamp - batch.Logs[0].TimeUnixNanos = timestamp - batch.Logs[0].EventUnixNanos = timestamp - batch.Logs[0].IngestedAt = timestamp - batch.Metrics[0].TimeUnixNanos = timestamp - batch.Metrics[0].EventUnixNanos = timestamp - batch.Metrics[0].IngestedAt = timestamp - return batch -} - -func TestNormalizeBatchBackfillsSpanStartFromIngestedAt(t *testing.T) { - batch := Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", IngestedAt: 12345}}} - normalizeBatch(&batch) - if got := batch.Spans[0].StartUnixNanos; got != 12345 { - t.Fatalf("StartUnixNanos = %d, want ingested-at fallback 12345", got) - } -} - -func TestCompactionSourcesDropInheritedLedger(t *testing.T) { - selected := []batchMetadata{ - {ID: "raw-1"}, - {ID: "out-1", Sources: []string{"old-a", "old-b"}}, - {ID: "raw-2"}, - } - got := compactionSources(selected) - want := []string{"raw-1", "raw-2"} - if len(got) != len(want) { - t.Fatalf("compactionSources = %v, want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("compactionSources = %v, want %v", got, want) - } - } -} - -func TestRecoverQuarantinesBatchThatCanNeverApply(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - // A batch ID the segment stores can never accept: it decodes cleanly, so - // only an apply attempt can reject it, and it will do so on every boot. - poison := testBatch() - poison.ID = "poison batch" - if err := repository.writeWAL(poison); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - reopened, err := Open(dir) - if err != nil { - t.Fatalf("Open error = %v, want the unappliable batch quarantined instead of a boot loop", err) - } - defer reopened.Close() - entries, err := os.ReadDir(filepath.Join(dir, "wal")) - if err != nil { - t.Fatal(err) - } - corrupt, live := 0, 0 - for _, entry := range entries { - switch { - case strings.HasSuffix(entry.Name(), ".corrupt"): - corrupt++ - case strings.HasSuffix(entry.Name(), ".wal"): - live++ - } - } - if corrupt != 1 || live != 0 { - t.Fatalf("quarantined = %d, live = %d, want the poison WAL renamed aside", corrupt, live) - } -} - -func TestRecoverRetainsWALWhenApplyFailsFromEnvironment(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("running as root bypasses the directory permissions this test relies on") - } - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - batch := testBatch() - if err := repository.Stage(batch); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - // Hot segments still accept the batch, so replay publishes a projection - // prefix and then fails on Parquet: an environmental failure that a later - // healthy boot must be able to finish. - parquetSpans := filepath.Join(dir, "parquet", "spans") - if err := os.Chmod(parquetSpans, 0o555); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(parquetSpans, 0o755) }) - if _, err := Open(dir); err == nil { - t.Fatal("Open succeeded although replay could not publish the batch") - } - entries, err := os.ReadDir(filepath.Join(dir, "wal")) - if err != nil { - t.Fatal(err) - } - for _, entry := range entries { - if strings.HasSuffix(entry.Name(), ".corrupt") { - t.Fatalf("environmental failure quarantined %s; a healthy restart can no longer finish the batch", entry.Name()) - } - } - if err := os.Chmod(parquetSpans, 0o755); err != nil { - t.Fatal(err) - } - healthy, err := Open(dir) - if err != nil { - t.Fatalf("Open error = %v, want the retained WAL to replay once the environment recovered", err) - } - defer healthy.Close() - if got := healthy.Spans.RowCount(); got != uint64(len(batch.Spans)) { - t.Fatalf("replayed spans = %d, want %d", got, len(batch.Spans)) - } -} - -func TestStageRejectsBatchTheSegmentStoresCannotAccept(t *testing.T) { - repository, err := Open(t.TempDir()) + recovered, err := Open(dir) if err != nil { t.Fatal(err) } - defer repository.Close() - batch := testBatch() - batch.ID = "poison batch" - if err := repository.Stage(batch); err == nil { - t.Fatal("Stage accepted a batch ID no projection can ever publish") - } -} - -func TestWALDecoderRejectsOversizedFrame(t *testing.T) { - const testLimit = 64 << 10 - encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) - if err != nil { - t.Fatal(err) + defer recovered.Close() + metadata := recovered.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].ID != output.ID || recovered.RowCount() != 3*minCompactionInputs { + t.Fatalf("recovered metadata = %#v rows=%d", metadata, recovered.RowCount()) } - defer encoder.Close() - frame := encoder.EncodeAll(make([]byte, testLimit+(1<<10)), nil) - decoder, err := newWALDecoderWithLimit(testLimit) - if err != nil { - t.Fatal(err) + spans, err := traceAll(recovered, "trace-1") + if err != nil || len(spans) != minCompactionInputs { + t.Fatalf("recovered trace spans = %d, %v", len(spans), err) } - defer decoder.Close() - if _, err := decoder.DecodeAll(frame, nil); err == nil { - t.Fatal("WAL decoder accepted a frame declaring more memory than any batch needs") + if retired, err := filepath.Glob(filepath.Join(recovered.Parquet.BatchesDir(), "*.retired*")); err != nil || len(retired) != 0 { + t.Fatalf("retired inputs after recovery = %v, %v", retired, err) } -} - -func TestWALWriterRejectsBatchLargerThanDecoderBudget(t *testing.T) { - batch := Batch{ID: "large", Logs: []telemetry.Log{{Body: strings.Repeat("x", 2048)}}} - if _, err := encodeWALBatch(batch, 1024); err == nil { - t.Fatal("WAL writer produced a frame its paired decoder budget could not reopen") + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { + t.Fatalf("compaction marker remains after recovery: %v", err) } } -func TestCompactHotCompactsSpanIndexAndPreservesRows(t *testing.T) { - repository, err := Open(t.TempDir()) +func openTestDuckDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("duckdb", "") if err != nil { t.Fatal(err) } - defer repository.Close() - for i := range 6 { - now := int64(100 + i) - batch := Batch{ - ID: fmt.Sprintf("batch-%d", i), - Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: now, IngestedAt: now}}, - Logs: []telemetry.Log{{Body: fmt.Sprintf("log-%d", i), EventUnixNanos: now, IngestedAt: now}}, - Metrics: []telemetry.Metric{{Name: "requests", EventUnixNanos: now, IngestedAt: now}}, - } - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - } - if _, err := repository.CompactHot(3); err != nil { - t.Fatal(err) - } - if got := repository.Spans.SegmentCount(); got != 2 { - t.Fatalf("span segments = %d, want 2 compacted files", got) - } - if repository.Spans.RowCount() != 6 { - t.Fatal("hot span compaction changed row counts") - } + return db } -func TestOpenQuarantinesCorruptDisposableHotTier(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - batch := Batch{ID: "committed", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}} - if err := repository.Commit(batch); err != nil { - t.Fatal(err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "hot", "spans", "committed.fseg"), []byte("corrupt"), 0o644); err != nil { - t.Fatal(err) - } - reopened, err := Open(dir) - if err != nil { - t.Fatalf("Open failed on disposable hot corruption: %v", err) - } - defer reopened.Close() - if got := reopened.Spans.RowCount(); got != 0 { - t.Fatalf("rebuilt hot rows = %d, want empty acceleration tier", got) - } - if reopened.manifest.HotCutoffNanos == 0 { - t.Fatal("rebuilt hot tier did not move the authoritative boundary to Parquet") - } - matches, err := filepath.Glob(filepath.Join(dir, "hot.corrupt-*")) - if err != nil || len(matches) != 1 { - t.Fatalf("hot quarantine paths = %v, err = %v", matches, err) - } - if _, err := os.Stat(filepath.Join(dir, "parquet", "spans", "committed.parquet")); err != nil { - t.Fatalf("authoritative Parquet was not preserved: %v", err) - } +func traceAll(repository *Repository, traceID string) ([]telemetry.IndexedSpan, error) { + return repository.Trace(context.Background(), telemetry.TraceQuery{ + TraceID: traceID, StartNanos: -1 << 63, EndNanos: 1<<63 - 1, Limit: 500, + }) } diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 38579794..3dd71166 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "log/slog" + "runtime" + "sync" "time" "github.com/google/uuid" @@ -12,14 +14,14 @@ import ( ) const ( - commitQueueDepth = 4 - commitRetryLimit = 8 - writerShutdownGrace = 5 * time.Second + commitQueueDepth = 256 + commitRetryLimit = 3 + maxCommitWorkers = 4 submissionQueueDepth = 256 + writerShutdownGrace = 30 * time.Second ) type batchCommitter interface { - Stage(Batch) error Commit(Batch) error } @@ -37,17 +39,21 @@ type submission struct { ack chan error } +type commitJob struct { + batches []Batch + acks []chan error +} + func NewWriter(repository *Repository, batchSize int) *Writer { return &Writer{repository: repository, batchSize: batchSize, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth)} } func (w *Writer) Wait() { <-w.done } -// Submit accepts one decoded OTLP request. It returns only after the complete -// request is fsynced to the WAL, so a successful OTLP response is durable even -// though publication to the query projections continues asynchronously. +// Submit returns after every row in the request belongs to a durably published +// atomic Parquet batch directory. func (w *Writer) Submit(ctx context.Context, batch Batch) error { - if len(batch.Spans)+len(batch.Logs)+len(batch.Metrics) == 0 { + if batchRows(batch) == 0 { return nil } request := submission{batch: batch, ack: make(chan error, 1)} @@ -70,88 +76,100 @@ func (w *Writer) Submit(ctx context.Context, batch Batch) error { func (w *Writer) Run(ctx context.Context) error { defer close(w.done) - commits := make(chan Batch, commitQueueDepth) - workerDone := make(chan error, 1) - workerCtx, cancelWorker := context.WithCancel(context.Background()) - defer cancelWorker() - go w.commitWorker(workerCtx, commits, workerDone) - finish := func() error { - close(commits) - return <-workerDone + jobs := make(chan commitJob, commitQueueDepth) + workerCtx, cancelWorkers := context.WithCancel(context.Background()) + defer cancelWorkers() + fatal := make(chan error, 1) + var workers sync.WaitGroup + for range min(maxCommitWorkers, max(1, runtime.GOMAXPROCS(0))) { + workers.Add(1) + go func() { + defer workers.Done() + w.commitWorker(workerCtx, jobs, fatal) + }() } - finishBounded := func() error { + + finish := func(graceful bool) error { + close(jobs) + if !graceful { + cancelWorkers() + } + finished := make(chan struct{}) + go func() { workers.Wait(); close(finished) }() grace := w.shutdownGrace if grace <= 0 { grace = writerShutdownGrace } - timer := time.AfterFunc(grace, cancelWorker) + timer := time.NewTimer(grace) defer timer.Stop() - return finish() + select { + case <-finished: + case <-timer.C: + cancelWorkers() + <-finished + } + select { + case err := <-fatal: + return err + default: + return nil + } } + for { select { case request := <-w.submissions: - if err := w.stageSubmission(request, commits, workerDone); err != nil { - return err + if err := w.enqueueSubmissions(request, jobs, fatal); err != nil { + return errors.Join(err, finish(false)) } case <-ctx.Done(): - return finishBounded() - case err := <-workerDone: - return err + return finish(true) + case err := <-fatal: + cancelWorkers() + return errors.Join(err, finish(false)) } } } -func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDone <-chan error) error { +func (w *Writer) enqueueSubmissions(request submission, out chan<- commitJob, fatal <-chan error) error { requests := []submission{request} - draining := true - for draining && len(requests) < submissionQueueDepth { + for len(requests) < submissionQueueDepth { select { case next := <-w.submissions: requests = append(requests, next) default: - draining = false + goto drained } } + +drained: limit := w.batchLimit() for len(requests) > 0 { - if batchRows(requests[0].batch) > limit { + firstRows := batchRows(requests[0].batch) + if firstRows > limit { oversized := requests[0] requests = requests[1:] chunks := splitBatch(oversized.batch, limit) - staged := make([]Batch, 0, len(chunks)) - var stageErr error - for _, chunk := range chunks { - chunk.ID = uuid.NewString() - if stageErr = w.repository.Stage(chunk); stageErr != nil { - metrics.FlushErrors.WithLabelValues("stage").Inc() - break - } - staged = append(staged, chunk) + for i := range chunks { + chunks[i].ID = uuid.NewString() } - if stageErr != nil { - // Already-staged chunks remain replayable. Do not enqueue only a - // prefix for live publication; recovery will publish that durable - // prefix after the storage fault is repaired. - oversized.ack <- stageErr - if len(staged) > 0 { - return fmt.Errorf("stage oversized telemetry request after %d durable chunks: %w", len(staged), stageErr) - } - continue + if err := enqueueJob(out, fatal, commitJob{batches: chunks, acks: []chan error{oversized.ack}}); err != nil { + return err } - metrics.RecordIngest("spans", len(oversized.batch.Spans)) - metrics.RecordIngest("logs", len(oversized.batch.Logs)) - metrics.RecordIngest("metrics", len(oversized.batch.Metrics)) - oversized.ack <- nil - for _, chunk := range staged { - select { - case out <- chunk: - case err := <-workerDone: - return err - } + continue + } + // A full batch, a lone request, or a request that cannot share the + // next batch needs no row copy through the group-commit buffer. + if firstRows == limit || len(requests) == 1 || firstRows+batchRows(requests[1].batch) > limit { + direct := requests[0] + requests = requests[1:] + direct.batch.ID = uuid.NewString() + if err := enqueueJob(out, fatal, commitJob{batches: []Batch{direct.batch}, acks: []chan error{direct.ack}}); err != nil { + return err } continue } + batch := Batch{ID: uuid.NewString()} group := make([]submission, 0, len(requests)) rows := 0 @@ -171,23 +189,89 @@ func (w *Writer) stageSubmission(request submission, out chan<- Batch, workerDon break } } - if err := w.repository.Stage(batch); err != nil { - metrics.FlushErrors.WithLabelValues("stage").Inc() - for _, item := range group { - item.ack <- err - } - continue + acks := make([]chan error, len(group)) + for i := range group { + acks[i] = group[i].ack } - metrics.RecordIngest("spans", len(batch.Spans)) - metrics.RecordIngest("logs", len(batch.Logs)) - metrics.RecordIngest("metrics", len(batch.Metrics)) - for _, item := range group { - item.ack <- nil + if err := enqueueJob(out, fatal, commitJob{batches: []Batch{batch}, acks: acks}); err != nil { + return err } + } + return nil +} + +func enqueueJob(out chan<- commitJob, fatal <-chan error, job commitJob) error { + select { + case out <- job: + return nil + case err := <-fatal: + return err + } +} + +func (w *Writer) commitWorker(ctx context.Context, jobs <-chan commitJob, fatal chan<- error) { + for { select { - case out <- batch: - case err := <-workerDone: - return err + case <-ctx.Done(): + return + case job, ok := <-jobs: + if !ok { + return + } + if err := w.commitJob(ctx, job); err != nil { + for _, ack := range job.acks { + ack <- err + } + select { + case fatal <- err: + default: + } + return + } + for _, batch := range job.batches { + metrics.RecordIngest("spans", len(batch.Spans)) + metrics.RecordIngest("logs", len(batch.Logs)) + metrics.RecordIngest("metrics", len(batch.Metrics)) + } + for _, ack := range job.acks { + ack <- nil + } + } + } +} + +func (w *Writer) commitJob(ctx context.Context, job commitJob) error { + for _, batch := range job.batches { + var lastErr error + for attempt := 0; attempt < commitRetryLimit; attempt++ { + if err := w.repository.Commit(batch); err == nil { + lastErr = nil + break + } else { + lastErr = err + } + metrics.FlushErrors.WithLabelValues("batch").Inc() + slog.Warn("telemetry batch commit failed; retrying", "batch_id", batch.ID, "attempt", attempt+1, "error", lastErr) + if attempt+1 == commitRetryLimit { + break + } + delay := defaultCommitRetryDelay(attempt) + if w.retryDelay != nil { + delay = w.retryDelay(attempt) + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return ctx.Err() + case <-timer.C: + } + } + if lastErr != nil { + metrics.FlushErrors.WithLabelValues("failed").Inc() + return fmt.Errorf("commit telemetry batch %s after %d attempts: %w", batch.ID, commitRetryLimit, lastErr) } } return nil @@ -205,8 +289,6 @@ func (w *Writer) batchLimit() int { return limit } -// splitBatch partitions one request without copying telemetry payloads. Each -// chunk is independently WAL-safe and no chunk exceeds the projection limit. func splitBatch(batch Batch, limit int) []Batch { chunks := make([]Batch, 0, (batchRows(batch)+limit-1)/limit) for batchRows(batch) > 0 { @@ -228,71 +310,6 @@ func splitBatch(batch Batch, limit int) []Batch { return chunks } -func (w *Writer) commitWorker(ctx context.Context, in <-chan Batch, done chan<- error) { - for { - var batch Batch - select { - case <-ctx.Done(): - done <- ctx.Err() - return - case next, ok := <-in: - if !ok { - done <- nil - return - } - batch = next - } - committed := false - var lastErr error - for attempt := 0; attempt < commitRetryLimit; attempt++ { - err := w.repository.Commit(batch) - if err == nil { - committed = true - break - } - lastErr = err - metrics.FlushErrors.WithLabelValues("batch").Inc() - // Log immediately and then at powers of two so a persistent storage - // outage stays visible without producing an unbounded log storm. - if attempt == 0 || attempt&(attempt-1) == 0 { - slog.Warn("telemetry batch commit failed; retrying", "batch_id", batch.ID, "attempt", attempt+1, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", err) - } - delay := defaultCommitRetryDelay(attempt) - if w.retryDelay != nil { - delay = w.retryDelay(attempt) - } - if attempt+1 == commitRetryLimit { - break - } - if delay > 0 { - timer := time.NewTimer(delay) - select { - case <-ctx.Done(): - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } - done <- ctx.Err() - return - case <-timer.C: - } - } - } - if !committed { - // The batch stays in the WAL. Its invisible Parquet staging files may - // already be durable, and replay can finish publication and register the - // batch without re-encoding them. - metrics.FlushErrors.WithLabelValues("deferred").Inc() - slog.Error("telemetry batch commit failed after bounded retries; stopping ingest with WAL retained for replay", "batch_id", batch.ID, "attempts", commitRetryLimit, "spans", len(batch.Spans), "logs", len(batch.Logs), "metrics", len(batch.Metrics), "error", lastErr) - done <- fmt.Errorf("commit telemetry batch %s after %d attempts: %w", batch.ID, commitRetryLimit, lastErr) - return - } - } -} - func defaultCommitRetryDelay(attempt int) time.Duration { - shift := min(attempt, 6) - return min(100*time.Millisecond*time.Duration(1< maxBatchRows { - t.Fatalf("staged batch rows = %d", rows) - } - total += batchRows(batch) + if len(committer.batches) != 2 { + t.Fatalf("committed batches = %d, want 2", len(committer.batches)) } - if total != maxBatchRows+1 { - t.Fatalf("staged rows = %d, want %d", total, maxBatchRows+1) + if batchRows(committer.batches[0])+batchRows(committer.batches[1]) != maxBatchRows+1 { + t.Fatal("split lost rows") } -} - -type secondStageFailCommitter struct { - mu sync.Mutex - stages []Batch -} - -func (c *secondStageFailCommitter) Stage(batch Batch) error { - c.mu.Lock() - defer c.mu.Unlock() - if len(c.stages) == 1 { - return errors.New("WAL device failed mid-request") - } - c.stages = append(c.stages, batch) - return nil -} -func (*secondStageFailCommitter) Commit(Batch) error { return nil } - -func TestWriterStopsAfterPartialOversizedSubmissionStage(t *testing.T) { - committer := &secondStageFailCommitter{} - w := testWriter(committer, maxBatchRows) - runDone := make(chan error, 1) - go func() { runDone <- w.Run(context.Background()) }() - if err := w.Submit(context.Background(), Batch{Spans: make([]telemetry.Span, maxBatchRows+1)}); err == nil { - t.Fatal("Submit returned nil after only a prefix became durable") - } - if err := <-runDone; err == nil || !strings.Contains(err.Error(), "durable chunks") { - t.Fatalf("Run error = %v, want fatal partial-stage error", err) + for _, batch := range committer.batches { + if batch.ID == "" || batchRows(batch) > maxBatchRows { + t.Fatalf("invalid chunk: id=%q rows=%d", batch.ID, batchRows(batch)) + } } } -func TestWriterRetainsBatchAcrossCommitFailures(t *testing.T) { - committer := &recoveringCommitter{failures: 6} +func TestWriterRetriesTransientCommitFailure(t *testing.T) { + committer := &recordingCommitter{failures: 2} w := testWriter(committer, 1) w.retryDelay = func(int) time.Duration { return 0 } ctx, cancel := context.WithCancel(context.Background()) runDone := make(chan error, 1) go func() { runDone <- w.Run(ctx) }() - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace"}}}); err != nil { t.Fatal(err) } cancel() if err := <-runDone; err != nil { t.Fatal(err) } - if committer.calls != 7 || len(committer.staged) != 1 || len(committer.batches) != 1 { - t.Fatalf("calls=%d staged=%d committed=%d", committer.calls, len(committer.staged), len(committer.batches)) - } -} - -type durableFailCommitter struct { - repository *Repository - attempted chan struct{} - once sync.Once -} - -func (c *durableFailCommitter) Stage(batch Batch) error { return c.repository.Stage(batch) } -func (c *durableFailCommitter) Commit(Batch) error { - c.once.Do(func() { close(c.attempted) }) - return errors.New("storage stalled") -} - -func TestWriterShutdownReplaysEveryAcknowledgedBatch(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - committer := &durableFailCommitter{repository: repository, attempted: make(chan struct{})} - w := testWriter(committer, 1) - w.retryDelay = func(int) time.Duration { return time.Hour } - w.shutdownGrace = 25 * time.Millisecond - ctx, cancel := context.WithCancel(context.Background()) - finished := make(chan error, 1) - go func() { finished <- w.Run(ctx) }() - for i := range 5 { - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: int64(100 + i)}}}); err != nil { - t.Fatal(err) - } - } - <-committer.attempted - cancel() - if err := <-finished; !errors.Is(err, context.Canceled) { - t.Fatalf("Run error = %v, want context canceled", err) - } - if err := repository.Close(); err != nil { - t.Fatal(err) - } - recovered, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer recovered.Close() - if got := recovered.Spans.RowCount(); got != 5 { - t.Fatalf("replayed spans = %d, want 5", got) + committer.mu.Lock() + defer committer.mu.Unlock() + if committer.calls != 3 || len(committer.batches) != 1 { + t.Fatalf("calls=%d committed=%d", committer.calls, len(committer.batches)) } } -func TestWriterSurfacesPermanentCommitFailureAfterBoundedRetries(t *testing.T) { - committer := &recoveringCommitter{failures: commitRetryLimit + 1} +func TestWriterSurfacesPermanentFailureToSubmitAndRun(t *testing.T) { + committer := &recordingCommitter{failures: commitRetryLimit + 1} w := testWriter(committer, 1) w.retryDelay = func(int) time.Duration { return 0 } runDone := make(chan error, 1) go func() { runDone <- w.Run(context.Background()) }() - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { - t.Fatal(err) + if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace"}}}); err == nil { + t.Fatal("Submit succeeded after permanent storage failure") } if err := <-runDone; err == nil { - t.Fatal("Run returned nil after permanent commit failure") + t.Fatal("Run succeeded after permanent storage failure") } + committer.mu.Lock() + defer committer.mu.Unlock() if committer.calls != commitRetryLimit || len(committer.batches) != 0 { t.Fatalf("calls=%d committed=%d", committer.calls, len(committer.batches)) } } -func TestWriterCancellationInterruptsCommitBackoff(t *testing.T) { - committer := &recoveringCommitter{failures: commitRetryLimit + 1} - w := testWriter(committer, 1) - w.retryDelay = func(int) time.Duration { return time.Hour } - w.shutdownGrace = 25 * time.Millisecond - ctx, cancel := context.WithCancel(context.Background()) - finished := make(chan error, 1) - go func() { finished <- w.Run(ctx) }() - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err != nil { - t.Fatal(err) - } - deadline := time.Now().Add(time.Second) - for { - committer.mu.Lock() - calls := committer.calls - committer.mu.Unlock() - if calls > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("commit attempt did not start") - } - time.Sleep(time.Millisecond) - } - cancel() - select { - case err := <-finished: - if !errors.Is(err, context.Canceled) { - t.Fatalf("Run error = %v, want context canceled", err) - } - case <-time.After(time.Second): - t.Fatal("writer shutdown remained blocked in retry backoff") - } -} - -type stageFailCommitter struct{ stages, commits int } - -func (c *stageFailCommitter) Stage(Batch) error { - c.stages++ - return errors.New("wal device unavailable") +type parallelCommitter struct { + mu sync.Mutex + active int + maxActive int + started chan struct{} + release chan struct{} + target int + once sync.Once } -func (c *stageFailCommitter) Commit(Batch) error { c.commits++; return nil } -func TestWriterSurvivesRequestStageFailure(t *testing.T) { - committer := &stageFailCommitter{} - w := testWriter(committer, 1) - ctx, cancel := context.WithCancel(context.Background()) - runDone := make(chan error, 1) - go func() { runDone <- w.Run(ctx) }() - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}}); err == nil { - t.Fatal("Submit returned nil after Stage failed") - } - cancel() - if err := <-runDone; err != nil { - t.Fatal(err) - } - if committer.stages != 1 || committer.commits != 0 { - t.Fatalf("stages=%d commits=%d", committer.stages, committer.commits) - } -} - -type ioFailCommitter struct{ repository *Repository } - -func (c *ioFailCommitter) Stage(batch Batch) error { return c.repository.Stage(batch) } -func (*ioFailCommitter) Commit(Batch) error { return errors.New("storage unavailable") } - -func TestWriterKeepsWALWhenCommitRetriesAreExhausted(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - w := testWriter(&ioFailCommitter{repository: repository}, 1) - w.retryDelay = func(int) time.Duration { return 0 } - runDone := make(chan error, 1) - go func() { runDone <- w.Run(context.Background()) }() - if err := w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}}); err != nil { - t.Fatal(err) - } - if err := <-runDone; err == nil { - t.Fatal("Run returned nil after permanent commit failure") - } - entries, err := os.ReadDir(filepath.Join(dir, "wal")) - if err != nil { - t.Fatal(err) - } - kept := 0 - for _, entry := range entries { - if strings.HasSuffix(entry.Name(), ".wal") { - kept++ - } - } - if kept != 1 { - t.Fatalf("retained WAL files = %d, want 1", kept) - } -} - -type transientStageCommitter struct { - repository *Repository - mu sync.Mutex - stages int - committed []Batch -} - -func (c *transientStageCommitter) Stage(batch Batch) error { +func (c *parallelCommitter) Commit(Batch) error { c.mu.Lock() - c.stages++ - first := c.stages == 1 - c.mu.Unlock() - if first { - return errors.New("wal device busy") - } - return c.repository.Stage(batch) -} -func (c *transientStageCommitter) Commit(batch Batch) error { - if err := c.repository.Commit(batch); err != nil { - return err + c.active++ + c.maxActive = max(c.maxActive, c.active) + if c.active == c.target { + c.once.Do(func() { close(c.started) }) } + c.mu.Unlock() + <-c.release c.mu.Lock() - c.committed = append(c.committed, batch) + c.active-- c.mu.Unlock() return nil } -func TestWriterAcceptsCallerRetryAfterTransientStageFailure(t *testing.T) { - repository, err := Open(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - committer := &transientStageCommitter{repository: repository} +func TestWriterCommitsIndependentBatchesInParallel(t *testing.T) { + jobs := min(maxCommitWorkers, max(1, runtime.GOMAXPROCS(0))) + committer := ¶llelCommitter{started: make(chan struct{}), release: make(chan struct{}), target: jobs} w := testWriter(committer, 1) + results := make(chan error, jobs) + for i := range jobs { + go func() { + results <- w.Submit(context.Background(), Batch{Spans: []telemetry.Span{{SpanID: fmt.Sprint(i)}}}) + }() + } + waitForQueue(t, w, jobs) ctx, cancel := context.WithCancel(context.Background()) runDone := make(chan error, 1) go func() { runDone <- w.Run(ctx) }() - batch := Batch{Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 100, IngestedAt: 100}}} - if err := w.Submit(context.Background(), batch); err == nil { - t.Fatal("first Submit returned nil") + select { + case <-committer.started: + case <-time.After(2 * time.Second): + close(committer.release) + t.Fatal("commit workers did not run four independent batches concurrently") } - if err := w.Submit(context.Background(), batch); err != nil { - t.Fatal(err) + close(committer.release) + for range jobs { + if err := <-results; err != nil { + t.Fatal(err) + } } cancel() if err := <-runDone; err != nil { t.Fatal(err) } - committer.mu.Lock() - defer committer.mu.Unlock() - if len(committer.committed) != 1 { - t.Fatalf("committed batches = %d, want 1", len(committer.committed)) + if committer.maxActive != jobs { + t.Fatalf("maximum concurrent commits = %d, want %d", committer.maxActive, jobs) + } +} + +func waitForQueue(t *testing.T, w *Writer, count int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for len(w.submissions) != count { + if time.Now().After(deadline) { + t.Fatalf("queued submissions = %d, want %d", len(w.submissions), count) + } + time.Sleep(time.Millisecond) } } diff --git a/internal/telemetry/trace_index.go b/internal/telemetry/trace_index.go new file mode 100644 index 00000000..5fb8fddb --- /dev/null +++ b/internal/telemetry/trace_index.go @@ -0,0 +1,250 @@ +package telemetry + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "os" +) + +const ( + traceIndexMagic = "FANIDX02" + traceIndexHeaderSize = 16 + traceIndexEntrySize = 24 +) + +type traceRange struct { + hash uint64 + row uint64 + count uint64 +} + +// traceIndex keeps only fixed-size index metadata in memory. Entries stay in +// the sidecar and are located with binary search, so retained trace cardinality +// does not become retained Go heap. +type traceIndex struct { + path string + entries uint64 +} + +type traceIndexWriter struct { + file *os.File + path string + last traceRange + rows uint64 + entries uint64 + have bool + closed bool +} + +func newTraceIndexWriter(path string) (*traceIndexWriter, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + if _, err := file.Write(make([]byte, traceIndexHeaderSize)); err != nil { + _ = file.Close() + _ = os.Remove(path) + return nil, err + } + return &traceIndexWriter{file: file, path: path}, nil +} + +// Append adds one Parquet row hash. Input must be ordered by hash, which lets +// even very large compactions build the fixed-width index in constant memory. +func (w *traceIndexWriter) Append(hash uint64) error { + if w.closed { + return errors.New("append to closed trace index") + } + row := w.rows + w.rows++ + if !w.have { + w.last = traceRange{hash: hash, row: row, count: 1} + w.have = true + return nil + } + if hash < w.last.hash { + return errors.New("trace index input is not sorted by hash") + } + if hash == w.last.hash { + w.last.count++ + return nil + } + if err := w.flush(); err != nil { + return err + } + w.last = traceRange{hash: hash, row: row, count: 1} + w.have = true + return nil +} + +func (w *traceIndexWriter) Close() error { + if w.closed { + return nil + } + w.closed = true + if w.have { + if err := w.flush(); err != nil { + return w.fail(err) + } + } + var header [traceIndexHeaderSize]byte + copy(header[:], traceIndexMagic) + binary.LittleEndian.PutUint64(header[8:], w.entries) + if _, err := w.file.WriteAt(header[:], 0); err != nil { + return w.fail(err) + } + if err := w.file.Sync(); err != nil { + return w.fail(err) + } + if err := w.file.Close(); err != nil { + _ = os.Remove(w.path) + return err + } + return nil +} + +func (w *traceIndexWriter) Abort() { + if w.closed { + return + } + w.closed = true + _ = w.file.Close() + _ = os.Remove(w.path) +} + +func (w *traceIndexWriter) flush() error { + var encoded [traceIndexEntrySize]byte + binary.LittleEndian.PutUint64(encoded[0:8], w.last.hash) + binary.LittleEndian.PutUint64(encoded[8:16], w.last.row) + binary.LittleEndian.PutUint64(encoded[16:24], w.last.count) + if _, err := w.file.Write(encoded[:]); err != nil { + return err + } + w.entries++ + w.have = false + return nil +} + +func (w *traceIndexWriter) fail(err error) error { + _ = w.file.Close() + _ = os.Remove(w.path) + return err +} + +func writeTraceIndex(path string, rows []spanParquetRow) error { + writer, err := newTraceIndexWriter(path) + if err != nil { + return err + } + for i := range rows { + if err := writer.Append(rows[i].TraceHash); err != nil { + writer.Abort() + return err + } + } + return writer.Close() +} + +func loadTraceIndex(path string, spanCount int) (index traceIndex, err error) { + file, err := os.Open(path) + if err != nil { + return traceIndex{}, err + } + defer func() { err = errors.Join(err, file.Close()) }() + info, err := file.Stat() + if err != nil { + return traceIndex{}, err + } + if !info.Mode().IsRegular() { + return traceIndex{}, errors.New("trace index is not a regular file") + } + var header [traceIndexHeaderSize]byte + if _, err := io.ReadFull(file, header[:]); err != nil || string(header[:8]) != traceIndexMagic { + return traceIndex{}, errors.New("invalid trace index header") + } + count := binary.LittleEndian.Uint64(header[8:]) + if count > uint64((math.MaxInt64-traceIndexHeaderSize)/traceIndexEntrySize) || + info.Size() != traceIndexHeaderSize+int64(count)*traceIndexEntrySize { + return traceIndex{}, errors.New("invalid trace index length") + } + reader := bufio.NewReaderSize(file, 64<<10) + var encoded [traceIndexEntrySize]byte + var previous traceRange + for i := uint64(0); i < count; i++ { + if _, err := io.ReadFull(reader, encoded[:]); err != nil { + return traceIndex{}, err + } + entry := decodeTraceRange(encoded[:]) + if entry.count == 0 || entry.row > uint64(spanCount) || entry.count > uint64(spanCount)-entry.row { + return traceIndex{}, errors.New("trace index entry exceeds span file") + } + if i == 0 && entry.row != 0 { + return traceIndex{}, errors.New("trace index does not start at row zero") + } + if i > 0 && (previous.hash >= entry.hash || previous.row+previous.count != entry.row) { + return traceIndex{}, errors.New("trace index is not strictly sorted and contiguous") + } + previous = entry + } + if count > 0 { + if previous.row+previous.count != uint64(spanCount) { + return traceIndex{}, fmt.Errorf("trace index covers %d of %d span rows", previous.row+previous.count, spanCount) + } + } else if spanCount != 0 { + return traceIndex{}, errors.New("trace index is empty for a non-empty span file") + } + return traceIndex{path: path, entries: count}, nil +} + +func (index traceIndex) Lookup(hash uint64) (match traceRange, found bool, err error) { + if index.entries == 0 { + return traceRange{}, false, nil + } + file, err := os.Open(index.path) + if err != nil { + return traceRange{}, false, err + } + defer func() { err = errors.Join(err, file.Close()) }() + low, high := uint64(0), index.entries + for low < high { + middle := low + (high-low)/2 + entry, readErr := readTraceRangeAt(file, middle) + if readErr != nil { + return traceRange{}, false, readErr + } + if entry.hash < hash { + low = middle + 1 + } else { + high = middle + } + } + if low == index.entries { + return traceRange{}, false, nil + } + entry, err := readTraceRangeAt(file, low) + if err != nil { + return traceRange{}, false, err + } + return entry, entry.hash == hash, nil +} + +func readTraceRangeAt(file *os.File, position uint64) (traceRange, error) { + var encoded [traceIndexEntrySize]byte + offset := int64(traceIndexHeaderSize) + int64(position)*traceIndexEntrySize + if _, err := file.ReadAt(encoded[:], offset); err != nil { + return traceRange{}, err + } + return decodeTraceRange(encoded[:]), nil +} + +func decodeTraceRange(encoded []byte) traceRange { + return traceRange{ + hash: binary.LittleEndian.Uint64(encoded[0:8]), + row: binary.LittleEndian.Uint64(encoded[8:16]), + count: binary.LittleEndian.Uint64(encoded[16:24]), + } +} diff --git a/internal/telemetry/trace_index_test.go b/internal/telemetry/trace_index_test.go new file mode 100644 index 00000000..002691b8 --- /dev/null +++ b/internal/telemetry/trace_index_test.go @@ -0,0 +1,57 @@ +package telemetry + +import ( + "os" + "path/filepath" + "testing" +) + +func TestTraceIndexLookupKeepsEntriesOnDisk(t *testing.T) { + path := filepath.Join(t.TempDir(), "trace.fidx") + rows := make([]spanParquetRow, 10_000) + for i := range rows { + rows[i].TraceHash = uint64(i * 2) + } + if err := writeTraceIndex(path, rows); err != nil { + t.Fatal(err) + } + index, err := loadTraceIndex(path, len(rows)) + if err != nil { + t.Fatal(err) + } + if index.entries != uint64(len(rows)) || index.path != path { + t.Fatalf("index metadata = %#v", index) + } + for _, hash := range []uint64{0, 9_998, 19_998} { + match, found, err := index.Lookup(hash) + if err != nil || !found || match.hash != hash || match.row != hash/2 || match.count != 1 { + t.Fatalf("lookup(%d) = %#v, %v, %v", hash, match, found, err) + } + } + if _, found, err := index.Lookup(9_999); err != nil || found { + t.Fatalf("missing lookup = %v, %v", found, err) + } +} + +func TestTraceIndexRejectsNoncontiguousEntry(t *testing.T) { + path := filepath.Join(t.TempDir(), "trace.fidx") + rows := []spanParquetRow{{TraceHash: 1}, {TraceHash: 2}} + if err := writeTraceIndex(path, rows); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + // The second entry must begin at row one; force it to row two. + if _, err := file.WriteAt([]byte{2}, traceIndexHeaderSize+traceIndexEntrySize+8); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := loadTraceIndex(path, len(rows)); err == nil { + t.Fatal("loadTraceIndex accepted a noncontiguous range") + } +} diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index 8aa2b252..c59989bd 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -1,7 +1,7 @@ --- title: How storage works description: What Fanout writes to disk, how it is queried, and why each component exists. -summary: Durable WAL ingestion, authoritative Parquet, DuckDB queries, and a focused hot span index. +summary: Atomic Parquet ingestion, indexed trace reads, DuckDB analytics, and SQLite control state. read_when: - You are sizing disk, or deciding what to back up. - You want to understand ingest durability or query performance. @@ -14,11 +14,9 @@ state—users, sessions, dashboards, alert rules, and agent history—lives in S | Component | Purpose | |---|---| -| WAL | Acknowledges an OTLP request only after its complete bounded batch is durable | -| Parquet | Authoritative retained spans, logs, and metrics | +| Atomic Parquet batches | Authoritative retained spans, logs, and metrics | +| Trace sidecar | Persistent row-range index for targeted trace reads from Parquet | | DuckDB | SQL, log filtering, aggregation, and broad analytical reads | -| Hot span index | Fast trace lookup for recent spans | -| Manifest journal | Constant-time commit ledger and crash-safe Parquet file lifecycle | | SQLite | Transactional application and identity state | There is no Iceberg, DuckLake, external catalog, or server database in the @@ -26,34 +24,39 @@ telemetry path. ## Writes are durable immediately -Each decoded OTLP request goes directly to the WAL. Concurrent small requests -may share one group commit; a request larger than `FANOUT_INGEST_BATCH_SIZE` is -split into independently recoverable chunks. Fanout returns success only after -every chunk is fsynced. +Concurrent small OTLP requests may share one bounded Parquet batch. Up to four +commit workers encode independent batches in parallel. Each worker writes all +present signals, metadata, and the trace index into a staging directory, fsyncs +them, and publishes the complete directory with one atomic rename. Fanout +returns success only after every batch belonging to the request is published. -An asynchronous worker then writes Parquet and updates the hot span index. A -crash or permanent I/O failure leaves the WAL in place, and startup replays it -idempotently. There is no timer-based flush window and no acknowledged telemetry -that exists only in memory. +A crash before the rename leaves only an unacknowledged staging directory, +which startup removes. A crash after the rename leaves a complete batch that +startup discovers directly from the filesystem. There is no separate WAL, +manifest, catalog, timer-based flush window, or acknowledged memory-only state. ## One authoritative copy for logs and metrics -Logs and metrics are written once, to Parquet. DuckDB applies filters, ordering, -`LIMIT`, and aggregation inside its vectorized scan. Keeping separate custom hot -copies would add write latency and compaction work without serving a production -query. +Every signal is written once, to typed Zstandard-compressed Parquet. DuckDB +applies filters, ordering, `LIMIT`, and aggregation inside its vectorized scan. +Trace lookup binary-searches a fixed-width sidecar on disk, seeks directly to the +matching Parquet row ranges, and verifies the complete trace ID after hashing. +The sidecar is an index, not another telemetry representation. -Spans additionally use a compact purpose-built on-disk index because trace -lookup benefits from it. Parquet remains authoritative when the recent span -index is pruned or rebuilt. Dashboard rollups are rebuildable DuckDB caches, -not a second resident copy in the hot index. +Dashboard rollups are rebuildable DuckDB caches. SQLite remains independent and +stores only transactional product state. ## Small files are bounded by compaction -Request-level durability can create many Parquet files. Maintenance drains every +Atomic ingestion creates immutable batch directories. Maintenance drains every eligible compaction group, combines files within bounded day/generation levels, -and applies retention. Active DuckDB readers pin their immutable files while a -publish is in progress, so reads see a consistent set. +builds a replacement trace index, and atomically swaps the replacement for its +inputs. A durable compaction marker makes an interrupted swap resumable. Active +DuckDB readers pin immutable files while retention or compaction removes them, +so reads never lose an open input file. + +Retention is based on ingestion time rather than event time. A service with a +bad clock therefore cannot pin a batch on disk by emitting a far-future event. ## Rollups lag deliberately diff --git a/site/src/content/docs/guides/back-up-and-restore.mdx b/site/src/content/docs/guides/back-up-and-restore.mdx index 5c6c466e..d8cae5bd 100644 --- a/site/src/content/docs/guides/back-up-and-restore.mdx +++ b/site/src/content/docs/guides/back-up-and-restore.mdx @@ -11,16 +11,16 @@ status: shipped Fanout is one process and one persistent data directory. Back up the directory as a unit. -The directory holds the telemetry WAL, commit manifest, Parquet files, -rebuildable query state, and the control SQLite database. Copying one -subdirectory produces something that looks like a backup and does not restore. +The directory holds atomic telemetry Parquet batches, rebuildable query state, +and the control SQLite database. Copying one subdirectory produces something +that looks like a backup and does not restore. ## Take a cold backup 1. Record the running version and configuration, keeping secrets out of ordinary logs and tickets. 2. Stop Fanout cleanly and wait for the process to exit. Shutdown closes both - OTLP listeners before draining the telemetry commit worker, so a clean exit + OTLP listeners before draining the telemetry commit workers, so a clean exit leaves no accepted request waiting for publication. 3. Copy or snapshot the complete data directory, preserving ownership and permissions. diff --git a/site/src/content/docs/guides/tune-retention.mdx b/site/src/content/docs/guides/tune-retention.mdx index 1c2bae68..722cbb0e 100644 --- a/site/src/content/docs/guides/tune-retention.mdx +++ b/site/src/content/docs/guides/tune-retention.mdx @@ -1,14 +1,14 @@ --- title: Tune retention description: Control how long telemetry is kept and how aggressively Fanout compacts what it has written. -summary: Retention, the hourly maintenance cycle, and the frequent merge pass that keeps query latency bounded. +summary: Retention, atomic batch sizing, and the maintenance pass that keeps query latency bounded. read_when: - Disk use is growing faster than expected. - Query or rollup latency is climbing on an instance that was fine. status: preview --- -Three settings govern what is on disk and how it is shaped. They do different +These settings govern what is on disk and how it is shaped. They do different jobs and are worth separating before changing any of them. ## How long data is kept @@ -30,42 +30,26 @@ FANOUT_MAINTENANCE_INTERVAL=1h This is the expensive pass: retention deletes plus full compaction. Lower it to reclaim space sooner, at the cost of running the heavy work more often. -## The merge pass - -```sh -FANOUT_MERGE_INTERVAL=1m -``` - -This is the cheap one, and it is the setting that most often matters for -latency. It consolidates the newest small Parquet files and deletes nothing. -Running it often keeps the queryable file count continuously low, which is what -bounds rollup and query scan time — without the churn, the deletion race or the -catalog cost of the full maintenance pass. `0s` disables it. - -If query latency is climbing on an instance whose data volume has not changed, -this is the first thing to look at: a high file count from many small writes -costs more per scan than the same bytes in fewer files. - ## Publication batching -The maximum number of telemetry rows in one WAL and Parquet batch: +The maximum number of telemetry rows in one atomic Parquet batch: ```sh FANOUT_INGEST_BATCH_SIZE=50000 ``` A larger batch can improve sustained write throughput and create fewer files -under concurrent ingest. Every request is still acknowledged only after its WAL -record is durable; this setting does not create a timer window or put accepted -telemetry at risk. Requests larger than the limit are split into recoverable -chunks. +under concurrent ingest. Every request is still acknowledged only after all of +its Parquet batches are durably published; this setting does not create a timer +window or put accepted telemetry at risk. Requests larger than the limit are +split into bounded atomic batches. ## What the knobs interact with -Maintenance briefly gates Parquet publication while it swaps immutable files; -WAL acknowledgement and Parquet encoding continue independently. Running -maintenance much more often still creates extra disk and CPU work, so change -one setting at a time and watch `/-/metrics`. +Maintenance briefly gates readers while it swaps or removes immutable files. +New batches continue encoding independently and need only the short publication +lock. Running maintenance much more often still creates extra disk and CPU +work, so change one setting at a time and watch `/-/metrics`. The full list, with defaults and types, is in the [storage settings](/reference/settings/storage) reference. diff --git a/site/src/content/docs/reference/data-layout.mdx b/site/src/content/docs/reference/data-layout.mdx index 406212e3..6f738ebd 100644 --- a/site/src/content/docs/reference/data-layout.mdx +++ b/site/src/content/docs/reference/data-layout.mdx @@ -13,20 +13,21 @@ Everything Fanout persists lives under `FANOUT_DATA_DIR` (`./data` by default, | Path | Holds | |---|---| -| `telemetry/wal/` | Durable requests waiting for publication or cleanup | -| `telemetry/parquet/{spans,logs,metrics}/` | Authoritative retained telemetry | -| `telemetry/hot/spans/` | Rebuildable recent-span index | -| `telemetry/MANIFEST.json` | Checkpoint of published WAL batches | -| `telemetry/MANIFEST.log` | Append-only commit journal since the checkpoint | +| `telemetry/parquet/batches/*.batch/` | Atomic batches containing Parquet signals, metadata, and a trace index | +| `telemetry/parquet/batches/_schema.batch/` | Empty schema anchors that keep every DuckDB view queryable | +| `telemetry/parquet/staging/` | Unacknowledged writes being prepared for atomic publication | +| `telemetry/compaction/` | Prepared replacement batches during compaction | +| `telemetry/COMPACTION.json` | Transient durable marker for an interrupted compaction swap | | `query/catalog.duckdb` | Rebuildable DuckDB rollups and query state | | `query/tmp/` | Spill space for queries that exceed the memory cap | | `control/fanout.sqlite` | Users, sessions, dashboards, alert rules, agent history | ## Why it is one unit -The WAL, Parquet files, and manifest form one crash-recovery unit. The hot span -index and DuckDB rollups can be rebuilt, but keeping the complete directory is -the supported and fastest restore path. +Each published Parquet batch directory is a self-contained crash-recovery unit. +Its trace sidecar is an index over the batch's span file, while DuckDB rollups +can be rebuilt. Keeping the complete data directory is the supported and fastest +restore path. `control/fanout.sqlite` is independent of both in format but not in meaning: it holds the dashboards and alert rules that refer to the telemetry, and the @@ -54,7 +55,7 @@ disk fails the query rather than degrading it. Retention is the main lever — `FANOUT_RETENTION_DAYS`, with expiry applied on the maintenance cycle rather than the moment you change it. File count matters -as much as byte count for query latency, which is what the merge pass exists to +as much as byte count for query latency, which is what the compaction pass exists to bound. [Tune retention](/guides/tune-retention) covers both. :::caution[Do not edit these files] diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index efd002ec..87809fdf 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -22,7 +22,6 @@ as a refusal to start rather than as a default nobody chose. | `storage.duckdb.max_connections` | `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | | `storage.duckdb.memory` | `FANOUT_DUCKDB_MEMORY` | string | — | | `storage.duckdb.threads` | `FANOUT_DUCKDB_THREADS` | integer | — | -| `storage.hot_retention` | `FANOUT_HOT_RETENTION` | duration | `24h` | | `storage.maintenance_interval` | `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | | `storage.retention_days` | `FANOUT_RETENTION_DAYS` | integer | `30` | | `storage.rollup_interval` | `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | @@ -42,13 +41,9 @@ Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected mem Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. -### `storage.hot_retention` - -Controls how long the custom indexed segments are retained. Older telemetry remains queryable in Parquet through DuckDB. - ### `storage.maintenance_interval` -Controls hot-segment pruning, Parquet retention and compaction, and query-cache checkpointing. +Controls Parquet retention and compaction, and query-cache checkpointing. ### `storage.rollup_skip_to_latest` diff --git a/site/src/content/docs/start/first-boot.mdx b/site/src/content/docs/start/first-boot.mdx index 4d177d59..00d959a2 100644 --- a/site/src/content/docs/start/first-boot.mdx +++ b/site/src/content/docs/start/first-boot.mdx @@ -20,7 +20,6 @@ group rather than individually: - Addresses and the data directory must not be empty. - `FANOUT_ROLLUP_INTERVAL` must be at least `1s`; - `FANOUT_MERGE_INTERVAL` must be `0s` or at least `1s`. - `FANOUT_AUTH_MODE` must be `local` or `oidc`. - `FANOUT_SESSION_IDLE_TTL` must be at least `5m`, and the absolute TTL must be positive and no shorter than the idle one. From d87e8a33a79391da8a51f26b1d9e74a80f0cd4b1 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 10:11:21 -0700 Subject: [PATCH 15/31] fix(storage): close Parquet concurrency gaps Serialize DuckDB maintenance with rollups, keep file publication brief, and make indexed reads use the same snapshot gate. Recover compaction in-process and publish oversized requests as one atomic batch. --- cmd/fanout/main.go | 5 +- internal/api/health.go | 7 +- internal/metrics/metrics.go | 10 +-- internal/metrics/metrics_test.go | 11 +-- internal/observability/service.go | 13 ++- internal/observability/service_test.go | 26 +++--- internal/query/duck.go | 86 ++++++++++++-------- internal/query/duck_test.go | 34 +++++++- internal/query/parquet_gate.go | 37 +++++++-- internal/telemetry/parquet.go | 18 +++-- internal/telemetry/store/compaction.go | 39 ++++----- internal/telemetry/store/repository.go | 6 +- internal/telemetry/store/repository_test.go | 89 ++++++++++++++++++++- internal/telemetry/store/writer.go | 63 +++++++-------- internal/telemetry/store/writer_test.go | 60 +++++++++++--- 15 files changed, 342 insertions(+), 162 deletions(-) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 4d8ecf19..9f0fcab8 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -111,8 +111,7 @@ func main() { } defer repository.Close() - // DuckDB is the SQL engine over open Parquet; indexed trace reads use the same - // repository directly through the typed observability kernel. + // DuckDB is the query facade over open Parquet and its indexed trace sidecars. q, err := query.NewDuck(ctx, cfg, repository) if err != nil { slog.Error("duckdb init failed", "err", err) @@ -292,7 +291,7 @@ func main() { // typed query kernel through deterministic HTTP or standard MCP tools. // Route both HTTP and MCP reads through Duck's retrying adapter. Passing the // raw *sql.DB here bypassed the Telemetry maintenance-race protection. - queries := observability.New(q, repository) + queries := observability.New(q, q) api.NewObservabilityHandler(queries).Register(e.Group("/api/observability", api.RequireCapability(api.ReadTelemetry))) api.RegisterIntelligenceRoutes(e, detector) dashboards := dashboard.New(sqlite.DB) diff --git a/internal/api/health.go b/internal/api/health.go index 5cf7c52c..fb98a719 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -190,10 +190,9 @@ func (h *HealthHandler) checkTelemetry() CheckResult { defer cancel() var one int - // Readiness must not queue behind routine Parquet publication. Maintenance - // health is reported separately; this probe only verifies that DuckDB and the - // telemetry view can plan and execute. - err := h.duck.DB.QueryRowContext(ctx, "SELECT 1 FROM telemetry.spans LIMIT 1").Scan(&one) + // Use the same snapshot gate as public reads so a short compaction/retention + // publication cannot race the Parquet scan and report a false outage. + err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM telemetry.spans LIMIT 1") if err != nil && err != sql.ErrNoRows { return CheckResult{ Status: "unhealthy", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 25821f00..1bdb1c4b 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -50,7 +50,7 @@ var ( IngestQueueDepth = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "fanout_ingest_queue_depth", - Help: "Current queue depth per signal", + Help: "Current ingest submission queue depth", }, []string{"signal"}) FlushTotal = promauto.NewCounterVec(prometheus.CounterOpts{ @@ -58,11 +58,6 @@ var ( Help: "Total flush operations", }, []string{"signal"}) - FlushBytes = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "fanout_flush_bytes_total", - Help: "Total bytes flushed", - }, []string{"signal"}) - FlushDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "fanout_flush_duration_seconds", Help: "Flush duration in seconds", @@ -223,9 +218,8 @@ func RecordIngest(signal string, count int) { } // RecordFlush records a flush event -func RecordFlush(signal string, bytes int64, durationSec float64) { +func RecordFlush(signal string, durationSec float64) { FlushTotal.WithLabelValues(signal).Inc() - FlushBytes.WithLabelValues(signal).Add(float64(bytes)) FlushDuration.WithLabelValues(signal).Observe(durationSec) } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 2ea313d2..c6e71e74 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -31,22 +31,15 @@ func TestRecordIngest(t *testing.T) { func TestRecordFlush(t *testing.T) { // Reset metrics FlushTotal.Reset() - FlushBytes.Reset() - RecordFlush("spans", 1024, 0.5) - RecordFlush("spans", 2048, 0.3) + RecordFlush("spans", 0.5) + RecordFlush("spans", 0.3) // Check counter incremented flushCount := testutil.ToFloat64(FlushTotal.WithLabelValues("spans")) if flushCount != 2 { t.Errorf("FlushTotal[spans] = %f, want 2", flushCount) } - - // Check bytes accumulated - bytesCount := testutil.ToFloat64(FlushBytes.WithLabelValues("spans")) - if bytesCount != 3072 { - t.Errorf("FlushBytes[spans] = %f, want 3072", bytesCount) - } } func TestRecordQuery(t *testing.T) { diff --git a/internal/observability/service.go b/internal/observability/service.go index 6c7ee585..efd1aa15 100644 --- a/internal/observability/service.go +++ b/internal/observability/service.go @@ -1,6 +1,7 @@ package observability import ( + "context" "errors" "fmt" "strings" @@ -9,7 +10,7 @@ import ( appid "github.com/labstack/fanout/internal/id" "github.com/labstack/fanout/internal/queryrows" - telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/labstack/fanout/internal/telemetry" ) const ( @@ -26,16 +27,20 @@ var ( type DB = queryrows.Queryer +type traceReader interface { + Trace(context.Context, telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) +} + type Service struct { db DB - repository *telemetrystore.Repository + repository traceReader now func() time.Time endpointMature atomic.Bool } -func New(db DB, repository *telemetrystore.Repository) *Service { +func New(db DB, repository traceReader) *Service { if db == nil || repository == nil { - panic("observability requires query engine and telemetry repository") + panic("observability requires query engine and indexed trace reader") } return &Service{db: db, repository: repository, now: time.Now} } diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 5dc8f04e..63666573 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -217,7 +217,7 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta(recentTraceQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout"). WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "ERROR", StatusMsg: "declined"}, {Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"}, }, Logs: []telemetry.Log{ @@ -254,7 +254,7 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "logs-fixture", Logs: []telemetry.Log{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "logs-fixture", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "payment declined", TraceID: "trace-1", SpanID: "root"}, {Namespace: "prod", TimeUnixNanos: start.Add(time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "card declined: token=abc123", TraceID: "trace-2", SpanID: "root2"}, {Namespace: "prod", TimeUnixNanos: start.Add(2 * time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: `auth declined: {"password":"hunter2"}`, TraceID: "trace-3", SpanID: "root3"}, @@ -298,7 +298,7 @@ func TestLogsRetainsOnlyNewestLimit(t *testing.T) { for i := range logs { logs[i] = telemetry.Log{Namespace: "prod", TimeUnixNanos: start.Add(time.Duration(i) * time.Millisecond).UnixNano(), Severity: "INFO", Body: "entry"} } - if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { t.Fatal(err) } entryRows := sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}) @@ -324,7 +324,7 @@ func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "parquet-logs", Logs: []telemetry.Log{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "parquet-logs", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "token=secret", TraceID: "trace-parquet", }}}); err != nil { @@ -354,13 +354,13 @@ func TestLogsQueryParquetAcrossBatches(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Second) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.Add(150 * time.Millisecond).UnixNano(), Body: "newer-old", Severity: "INFO"}, {Namespace: "prod", TimeUnixNanos: start.Add(300 * time.Millisecond).UnixNano(), Body: "newer-batch", Severity: "INFO"}, }}); err != nil { t.Fatal(err) } - if err := svc.repository.Commit(telemetrystore.Batch{ID: "overlap-late", Logs: []telemetry.Log{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "overlap-late", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.Add(100 * time.Millisecond).UnixNano(), Body: "late-old", Severity: "INFO"}, {Namespace: "prod", TimeUnixNanos: start.Add(200 * time.Millisecond).UnixNano(), Body: "late-boundary", Severity: "INFO"}, }}); err != nil { @@ -402,7 +402,7 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { {ID: "trace-outside-span", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(2 * time.Minute).UnixNano(), Body: "unrelated later event"}}}, } for _, batch := range batches { - if err := svc.repository.Commit(batch); err != nil { + if err := svc.repository.(*telemetrystore.Repository).Commit(batch); err != nil { t.Fatal(err) } } @@ -426,7 +426,7 @@ func TestTraceUsesIndexedParquet(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "indexed-trace", Spans: []telemetry.Span{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "indexed-trace", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "parquet-trace", SpanID: "root", ServiceName: "checkout", Name: "pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 25, StatusCode: "ERROR", StatusMsg: "declined", }}}); err != nil { @@ -455,13 +455,13 @@ func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "root", ServiceName: "frontend", StartUnixNanos: start.Add(10 * time.Minute).UnixNano(), DurationMS: 100, StatusCode: "ERROR", }}}); err != nil { t.Fatal(err) } - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-newer-child", Spans: []telemetry.Span{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-newer-child", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "child", ParentSpanID: "root", ServiceName: "backend", StartUnixNanos: start.Add(50 * time.Minute).UnixNano(), DurationMS: 25, StatusCode: "OK", }}}); err != nil { @@ -487,7 +487,7 @@ func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-after-rebuild", Spans: []telemetry.Span{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-after-rebuild", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "new-trace", SpanID: "root", ServiceName: "frontend", StartUnixNanos: cutoff.Add(time.Minute).UnixNano(), DurationMS: 10, StatusCode: "OK", }}}); err != nil { @@ -513,7 +513,7 @@ func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "trace-cross-namespace", Spans: []telemetry.Span{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-cross-namespace", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "shared-trace", SpanID: "child", ParentSpanID: "old-root", StartUnixNanos: cutoff.Add(time.Minute).UnixNano()}, {Namespace: "staging", TraceID: "shared-trace", SpanID: "root", StartUnixNanos: cutoff.Add(2 * time.Minute).UnixNano()}, }}); err != nil { @@ -540,7 +540,7 @@ func TestLogsBoundsParquetQueryWithLimitAndAggregatedBuckets(t *testing.T) { svc, mock := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.Commit(telemetrystore.Batch{ID: "bounded-parquet", Logs: []telemetry.Log{{ + if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "bounded-parquet", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "hello", TraceID: "trace-parquet", }}}); err != nil { diff --git a/internal/query/duck.go b/internal/query/duck.go index 3630fc05..f9f178e5 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -19,6 +19,7 @@ import ( "github.com/labstack/fanout/internal/metrics" "github.com/labstack/fanout/internal/query/writegate" "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -445,16 +446,18 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() + unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) var pruneErr error if d.repository != nil { var parquetErr error if d.cfg.RetentionDays > 0 { - d.parquetMu.Lock() - _, parquetErr = d.repository.PruneParquet(time.Now().Add(-time.Duration(d.cfg.RetentionDays) * 24 * time.Hour).UnixNano()) - d.parquetMu.Unlock() + parquetErr = d.PublishParquet(func() error { + _, err := d.repository.PruneParquet(time.Now().Add(-time.Duration(d.cfg.RetentionDays) * 24 * time.Hour).UnixNano()) + return err + }) } compactStart := time.Now() - compacted, compactErr := d.repository.CompactParquetBacklog(ctx, d.DB, 64, &d.parquetMu) + compacted, compactErr := d.repository.CompactParquetBacklog(ctx, d, 64) compactResult := metrics.TelemetryNoop if compactErr != nil { compactResult = metrics.TelemetryError @@ -465,7 +468,6 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { pruneErr = errors.Join(pruneErr, parquetErr, compactErr) } var cacheErr error - unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) if d.cfg.RetentionDays > 0 { for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { @@ -510,16 +512,15 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupService, true, watermark, sourceMax) } }() - if err := d.lockParquetRead(ctx); err != nil { - return 0, err - } - defer d.parquetMu.RUnlock() - - // Serialize against other writers (edge rollup, maintenance, ingest flushes). + // Serialize against other DuckDB writers (edge rollup and maintenance). // The write gate is always acquired before a connection to keep lock ordering // consistent and deadlock-free. unlock := d.writeGate.Lock(writegate.WriteRollupService) defer unlock() + if err := d.lockParquetRead(ctx); err != nil { + return 0, err + } + defer d.parquetMu.RUnlock() tx, err := d.DB.BeginTx(ctx, nil) if err != nil { @@ -635,14 +636,13 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupEndpoint, true, watermark, sourceMax) } }() + unlock := d.writeGate.Lock(writegate.WriteRollupEndpoint) + defer unlock() if err := d.lockParquetRead(ctx); err != nil { return 0, err } defer d.parquetMu.RUnlock() - unlock := d.writeGate.Lock(writegate.WriteRollupEndpoint) - defer unlock() - tx, err := d.DB.BeginTx(ctx, nil) if err != nil { return 0, err @@ -759,14 +759,13 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { updateRollupProgress(metrics.RollupEdge, true, watermark, sourceMax) } }() + unlock := d.writeGate.Lock(writegate.WriteRollupEdge) + defer unlock() if err := d.lockParquetRead(ctx); err != nil { return 0, err } defer d.parquetMu.RUnlock() - unlock := d.writeGate.Lock(writegate.WriteRollupEdge) - defer unlock() - tx, err := d.DB.BeginTx(ctx, nil) if err != nil { return 0, err @@ -1416,24 +1415,43 @@ func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args return d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) } -func (d *Duck) lockParquetRead(ctx context.Context) error { - for { - if d.parquetMu.TryRLock() { - return nil - } - timer := time.NewTimer(time.Millisecond) - select { - case <-ctx.Done(): - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } - return ctx.Err() - case <-timer.C: - } +// Trace pins the immutable Parquet snapshot for the full indexed-file read. +func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) { + if err := d.lockParquetRead(ctx); err != nil { + return nil, err } + defer d.parquetMu.RUnlock() + return d.repository.Trace(ctx, query) +} + +// MergeParquet executes the query-engine-specific half of compaction. The +// maintenance caller already holds writeGate, so this cannot contend with a +// rollup transaction while it uses the shared DuckDB pool. +func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { + quoted := make([]string, len(inputs)) + for i, input := range inputs { + quoted[i] = quoteDuckString(input) + } + query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(quoted, ",")) + if signal == "spans" { + query += " ORDER BY _trace_hash, start_unix_nano, span_id" + } + stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, quoteDuckString(output)) + _, err := d.DB.ExecContext(ctx, stmt) + return err +} + +// PublishParquet limits reader exclusion to the atomic directory swap. +func (d *Duck) PublishParquet(publish func() error) error { + d.parquetMu.Lock() + defer d.parquetMu.Unlock() + return publish() +} + +func quoteDuckString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + +func (d *Duck) lockParquetRead(ctx context.Context) error { + return d.parquetMu.RLockContext(ctx) } // ---- Queries for API ---- diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 98839af3..22da7a7c 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -343,14 +343,40 @@ func TestRollupReadLockHonorsContext(t *testing.T) { } } +func TestIndexedTraceReadHonorsParquetGateContext(t *testing.T) { + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d := &Duck{repository: repository} + d.parquetMu.Lock() + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + _, err = d.Trace(ctx, telemetry.TraceQuery{TraceID: "trace", StartNanos: 1, EndNanos: 2, Limit: 1}) + d.parquetMu.Unlock() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Trace error = %v, want deadline exceeded", err) + } +} + func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatal(err) } defer db.Close() + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + mock.ExpectExec("DELETE FROM service_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DELETE FROM endpoint_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DELETE FROM edge_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) - d := &Duck{DB: db, cfg: config.Config{MaintenanceInterval: time.Nanosecond}} + d := &Duck{DB: db, repository: repository, cfg: config.Config{MaintenanceInterval: time.Nanosecond, RetentionDays: 1}} + d.parquetMu.RLock() release := d.writeGate.Lock(writegate.WriteRollupService) done := make(chan error, 1) go func() { done <- d.runRepositoryMaintenance(context.Background()) }() @@ -360,6 +386,12 @@ func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { t.Fatalf("maintenance bypassed write gate: %v", err) case <-time.After(25 * time.Millisecond): } + if waiting := d.parquetMu.WaitingWriters(); waiting != 0 { + d.parquetMu.RUnlock() + release() + t.Fatalf("maintenance queued Parquet publication before owning DuckDB write gate: %d", waiting) + } + d.parquetMu.RUnlock() release() if err := <-done; err != nil { t.Fatal(err) diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index c3093ceb..6ae834d9 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -1,6 +1,7 @@ package query import ( + "context" "sync" "time" ) @@ -19,7 +20,7 @@ const defaultWriterGrace = 30 * time.Second type parquetReadGate struct { once sync.Once mu sync.Mutex - changed *sync.Cond + changed chan struct{} readers int writer bool waiting []parquetWaiter @@ -34,7 +35,12 @@ type parquetWaiter struct { } func (g *parquetReadGate) init() { - g.once.Do(func() { g.changed = sync.NewCond(&g.mu) }) + g.once.Do(func() { g.changed = make(chan struct{}) }) +} + +func (g *parquetReadGate) notifyLocked() { + close(g.changed) + g.changed = make(chan struct{}) } func (g *parquetReadGate) clock() time.Time { @@ -76,13 +82,27 @@ func (g *parquetReadGate) TryRLock() bool { } func (g *parquetReadGate) RLock() { + if err := g.RLockContext(context.Background()); err != nil { + panic(err) + } +} + +func (g *parquetReadGate) RLockContext(ctx context.Context) error { g.init() g.mu.Lock() for !g.admitsReaderLocked() { - g.changed.Wait() + changed := g.changed + g.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-changed: + } + g.mu.Lock() } g.readers++ g.mu.Unlock() + return nil } func (g *parquetReadGate) RUnlock() { @@ -94,7 +114,7 @@ func (g *parquetReadGate) RUnlock() { panic("query: parquetReadGate RUnlock without RLock") } if g.readers == 0 { - g.changed.Broadcast() + g.notifyLocked() } g.mu.Unlock() } @@ -107,9 +127,12 @@ func (g *parquetReadGate) Lock() { g.waiting = append(g.waiting, waiter) // Wake any readers parked on an earlier publisher so they re-evaluate this // publisher's grace, and so the grace clock starts for readers immediately. - g.changed.Broadcast() + g.notifyLocked() for g.writer || g.readers > 0 { - g.changed.Wait() + changed := g.changed + g.mu.Unlock() + <-changed + g.mu.Lock() } for i, queued := range g.waiting { if queued.id == waiter.id { @@ -129,7 +152,7 @@ func (g *parquetReadGate) Unlock() { panic("query: parquetReadGate Unlock without Lock") } g.writer = false - g.changed.Broadcast() + g.notifyLocked() g.mu.Unlock() } diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 64903001..0343fec5 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -291,9 +291,13 @@ func (p *ParquetStore) Trace(ctx context.Context, query TraceQuery) ([]IndexedSp } hash := xxh3.HashString(query.TraceID) p.mu.RLock() - defer p.mu.RUnlock() - selected := make(indexedSpanHeap, 0, query.Limit) + batches := make([]*storedBatch, 0, len(p.batches)) for _, batch := range p.batches { + batches = append(batches, batch) + } + p.mu.RUnlock() + selected := make(indexedSpanHeap, 0, query.Limit) + for _, batch := range batches { if err := ctx.Err(); err != nil { return nil, err } @@ -431,11 +435,15 @@ func (p *ParquetStore) PruneBefore(cutoff int64) (int, error) { func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { p.mu.RLock() - defer p.mu.RUnlock() - stats := map[string]ParquetStats{"spans": {}, "logs": {}, "metrics": {}} + dirs := make([]string, 0, len(p.batches)) for _, batch := range p.batches { + dirs = append(dirs, batch.dir) + } + p.mu.RUnlock() + stats := map[string]ParquetStats{"spans": {}, "logs": {}, "metrics": {}} + for _, dir := range dirs { for signal := range stats { - info, err := os.Stat(filepath.Join(batch.dir, signal+".parquet")) + info, err := os.Stat(filepath.Join(dir, signal+".parquet")) if errors.Is(err, os.ErrNotExist) { continue } diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 7bf18789..a9f43917 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -2,15 +2,12 @@ package store import ( "context" - "database/sql" "encoding/json" "errors" "fmt" "math" "os" "path/filepath" - "strings" - "sync" "time" "github.com/labstack/fanout/internal/telemetry" @@ -30,10 +27,17 @@ type compactionKey struct { generation uint32 } +// ParquetCompactor keeps DuckDB execution and publication locking in the query +// layer while storage owns batch selection and crash-safe replacement state. +type ParquetCompactor interface { + MergeParquet(context.Context, string, []string, string) error + PublishParquet(func() error) error +} + // CompactParquet combines one same-day, same-generation group. The output is // prepared outside the query gate and swapped as one batch directory. -func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { - if db == nil || maxBatches < minCompactionInputs { +func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompactor, maxBatches int) (int, error) { + if compactor == nil || maxBatches < minCompactionInputs { return 0, nil } r.compactionMu.Lock() @@ -42,7 +46,9 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches if exists, err := pathExists(markerPath); err != nil { return 0, err } else if exists { - return 0, errors.New("pending Parquet compaction must recover before another can start") + if err := compactor.PublishParquet(r.recoverCompaction); err != nil { + return 0, fmt.Errorf("recover pending Parquet compaction: %w", err) + } } selected := selectCompactionBatches(r.Parquet.BatchMetadata(), maxBatches) if len(selected) < minCompactionInputs { @@ -84,7 +90,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches for _, batch := range selected { path := filepath.Join(r.Parquet.BatchPath(batch.ID), signal+".parquet") if _, err := os.Stat(path); err == nil { - inputs = append(inputs, sqlQuote(path)) + inputs = append(inputs, path) } else if !errors.Is(err, os.ErrNotExist) { return 0, err } @@ -93,12 +99,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches continue } outputPath := filepath.Join(stage, signal+".parquet") - query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(inputs, ",")) - if signal == "spans" { - query += " ORDER BY _trace_hash, start_unix_nano, span_id" - } - stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, sqlQuote(outputPath)) - if _, err := db.ExecContext(ctx, stmt); err != nil { + if err := compactor.MergeParquet(ctx, signal, inputs, outputPath); err != nil { return 0, fmt.Errorf("compact %s Parquet: %w", signal, err) } if err := syncFile(outputPath); err != nil { @@ -119,11 +120,7 @@ func (r *Repository) CompactParquet(ctx context.Context, db *sql.DB, maxBatches return 0, err } prepared = true - if publishLock != nil { - publishLock.Lock() - defer publishLock.Unlock() - } - if err := r.completeCompaction(marker); err != nil { + if err := compactor.PublishParquet(func() error { return r.completeCompaction(marker) }); err != nil { return 0, err } return len(selected), nil @@ -161,10 +158,10 @@ func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) return selected } -func (r *Repository) CompactParquetBacklog(ctx context.Context, db *sql.DB, maxBatches int, publishLock sync.Locker) (int, error) { +func (r *Repository) CompactParquetBacklog(ctx context.Context, compactor ParquetCompactor, maxBatches int) (int, error) { total := 0 for { - count, err := r.CompactParquet(ctx, db, maxBatches, publishLock) + count, err := r.CompactParquet(ctx, compactor, maxBatches) total += count if err != nil || count == 0 { return total, err @@ -253,5 +250,3 @@ func writeDurableFile(path string, data []byte) error { } return os.Rename(tmp, path) } - -func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 3368b98a..d154c301 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -21,8 +21,6 @@ type Batch struct { Metrics []telemetry.Metric } -const maxBatchRows = 50_000 - // Repository publishes self-contained Parquet batch directories. The // directory rename is the transaction and the filesystem is the catalog. type Repository struct { @@ -94,8 +92,8 @@ func validateBatch(batch Batch) error { if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { return errors.New("telemetry batch requires a safe ID") } - if rows := batchRows(batch); rows == 0 || rows > maxBatchRows { - return fmt.Errorf("telemetry batch has %d rows; maximum is %d", rows, maxBatchRows) + if rows := batchRows(batch); rows == 0 { + return errors.New("telemetry batch is empty") } return nil } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 4124cbad..7bec33e2 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -4,15 +4,44 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "os" "path/filepath" + "strings" "testing" _ "github.com/duckdb/duckdb-go/v2" "github.com/labstack/fanout/internal/telemetry" ) +type testParquetCompactor struct { + db *sql.DB + publishErr error +} + +func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { + quoted := make([]string, len(inputs)) + for i, input := range inputs { + quoted[i] = sqlQuote(input) + } + query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(quoted, ",")) + if signal == "spans" { + query += " ORDER BY _trace_hash, start_unix_nano, span_id" + } + _, err := c.db.ExecContext(ctx, fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD)", query, sqlQuote(output))) + return err +} + +func (c *testParquetCompactor) PublishParquet(publish func() error) error { + if c.publishErr != nil { + return c.publishErr + } + return publish() +} + +func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + func testBatch() Batch { return Batch{ ID: "batch-test", @@ -71,6 +100,28 @@ func TestRepositoryCommitIsIdempotentDurableAndQueryable(t *testing.T) { } } +func TestRepositoryCommitsOversizedRequestAsOneBatch(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + spans := make([]telemetry.Span, maxGroupBatchRows+1) + for i := range spans { + spans[i] = telemetry.Span{ + TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), + StartUnixNanos: int64(i + 1), EndUnixNanos: int64(i + 2), IngestedAt: 1, + } + } + if err := repository.Commit(Batch{ID: "oversized", Spans: spans}); err != nil { + t.Fatal(err) + } + metadata := repository.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].Spans != len(spans) { + t.Fatalf("oversized batch metadata = %#v", metadata) + } +} + func TestRepositoryHasOneAuthoritativeStorageLayout(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -195,7 +246,7 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } db := openTestDuckDB(t) defer db.Close() - compacted, err := repository.CompactParquet(context.Background(), db, 64, nil) + compacted, err := repository.CompactParquet(context.Background(), &testParquetCompactor{db: db}, 64) if err != nil { t.Fatal(err) } @@ -227,6 +278,42 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } } +func TestRepositoryRecoversPendingCompactionWithoutRestart(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range minCompactionInputs { + batch := testBatch() + batch.ID = fmt.Sprintf("pending-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + db := openTestDuckDB(t) + defer db.Close() + compactor := &testParquetCompactor{db: db, publishErr: errors.New("publication unavailable")} + if _, err := repository.CompactParquet(context.Background(), compactor, 64); err == nil { + t.Fatal("compaction succeeded despite publication failure") + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("pending marker: %v", err) + } + compactor.publishErr = nil + if compacted, err := repository.CompactParquet(context.Background(), compactor, 64); err != nil || compacted != 0 { + t.Fatalf("resume compaction = %d, %v", compacted, err) + } + if got := repository.Parquet.BatchMetadata(); len(got) != 1 || got[0].Generation != 1 { + t.Fatalf("recovered metadata = %#v", got) + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { + t.Fatalf("recovered compaction marker remains: %v", err) + } +} + func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 3dd71166..8369dd33 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -16,6 +16,7 @@ import ( const ( commitQueueDepth = 256 commitRetryLimit = 3 + maxGroupBatchRows = 50_000 maxCommitWorkers = 4 submissionQueueDepth = 256 writerShutdownGrace = 30 * time.Second @@ -59,6 +60,7 @@ func (w *Writer) Submit(ctx context.Context, batch Batch) error { request := submission{batch: batch, ack: make(chan error, 1)} select { case w.submissions <- request: + metrics.UpdateQueueDepth("batch", len(w.submissions)) case <-w.done: return errors.New("telemetry writer is stopped") case <-ctx.Done(): @@ -76,6 +78,7 @@ func (w *Writer) Submit(ctx context.Context, batch Batch) error { func (w *Writer) Run(ctx context.Context) error { defer close(w.done) + defer metrics.UpdateQueueDepth("batch", 0) jobs := make(chan commitJob, commitQueueDepth) workerCtx, cancelWorkers := context.WithCancel(context.Background()) defer cancelWorkers() @@ -119,6 +122,7 @@ func (w *Writer) Run(ctx context.Context) error { for { select { case request := <-w.submissions: + metrics.UpdateQueueDepth("batch", len(w.submissions)) if err := w.enqueueSubmissions(request, jobs, fatal); err != nil { return errors.Join(err, finish(false)) } @@ -143,24 +147,15 @@ func (w *Writer) enqueueSubmissions(request submission, out chan<- commitJob, fa } drained: + metrics.UpdateQueueDepth("batch", len(w.submissions)) limit := w.batchLimit() for len(requests) > 0 { firstRows := batchRows(requests[0].batch) - if firstRows > limit { - oversized := requests[0] - requests = requests[1:] - chunks := splitBatch(oversized.batch, limit) - for i := range chunks { - chunks[i].ID = uuid.NewString() - } - if err := enqueueJob(out, fatal, commitJob{batches: chunks, acks: []chan error{oversized.ack}}); err != nil { - return err - } - continue - } // A full batch, a lone request, or a request that cannot share the - // next batch needs no row copy through the group-commit buffer. - if firstRows == limit || len(requests) == 1 || firstRows+batchRows(requests[1].batch) > limit { + // next batch needs no row copy through the group-commit buffer. One + // oversized request remains one atomic directory; the limit is a + // group-commit target, not a durability boundary. + if firstRows >= limit || len(requests) == 1 || firstRows+batchRows(requests[1].batch) > limit { direct := requests[0] requests = requests[1:] direct.batch.ID = uuid.NewString() @@ -242,6 +237,7 @@ func (w *Writer) commitWorker(ctx context.Context, jobs <-chan commitJob, fatal func (w *Writer) commitJob(ctx context.Context, job commitJob) error { for _, batch := range job.batches { + started := time.Now() var lastErr error for attempt := 0; attempt < commitRetryLimit; attempt++ { if err := w.repository.Commit(batch); err == nil { @@ -271,8 +267,10 @@ func (w *Writer) commitJob(ctx context.Context, job commitJob) error { } if lastErr != nil { metrics.FlushErrors.WithLabelValues("failed").Inc() + recordDroppedRows(batch) return fmt.Errorf("commit telemetry batch %s after %d attempts: %w", batch.ID, commitRetryLimit, lastErr) } + recordFlushes(batch, time.Since(started).Seconds()) } return nil } @@ -282,32 +280,29 @@ func batchRows(batch Batch) int { } func (w *Writer) batchLimit() int { - limit := min(w.batchSize, maxBatchRows) + limit := min(w.batchSize, maxGroupBatchRows) if limit <= 0 { - return maxBatchRows + return maxGroupBatchRows } return limit } -func splitBatch(batch Batch, limit int) []Batch { - chunks := make([]Batch, 0, (batchRows(batch)+limit-1)/limit) - for batchRows(batch) > 0 { - chunk := Batch{} - remaining := limit - if count := min(remaining, len(batch.Spans)); count > 0 { - chunk.Spans, batch.Spans = batch.Spans[:count], batch.Spans[count:] - remaining -= count - } - if count := min(remaining, len(batch.Logs)); count > 0 { - chunk.Logs, batch.Logs = batch.Logs[:count], batch.Logs[count:] - remaining -= count - } - if count := min(remaining, len(batch.Metrics)); count > 0 { - chunk.Metrics, batch.Metrics = batch.Metrics[:count], batch.Metrics[count:] - } - chunks = append(chunks, chunk) +func recordFlushes(batch Batch, durationSec float64) { + if len(batch.Spans) > 0 { + metrics.RecordFlush("spans", durationSec) + } + if len(batch.Logs) > 0 { + metrics.RecordFlush("logs", durationSec) } - return chunks + if len(batch.Metrics) > 0 { + metrics.RecordFlush("metrics", durationSec) + } +} + +func recordDroppedRows(batch Batch) { + metrics.RowsDropped.WithLabelValues("spans").Add(float64(len(batch.Spans))) + metrics.RowsDropped.WithLabelValues("logs").Add(float64(len(batch.Logs))) + metrics.RowsDropped.WithLabelValues("metrics").Add(float64(len(batch.Metrics))) } func defaultCommitRetryDelay(attempt int) time.Duration { diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index eed5aad9..33e9a550 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "github.com/labstack/fanout/internal/metrics" "github.com/labstack/fanout/internal/telemetry" + "github.com/prometheus/client_golang/prometheus/testutil" ) type recordingCommitter struct { @@ -51,7 +53,7 @@ func (c *blockingCommitter) Commit(Batch) error { func TestWriterAcknowledgesOnlyAfterAtomicCommit(t *testing.T) { committer := &blockingCommitter{entered: make(chan struct{}), release: make(chan struct{})} - w := testWriter(committer, maxBatchRows) + w := testWriter(committer, maxGroupBatchRows) ctx, cancel := context.WithCancel(context.Background()) runDone := make(chan error, 1) go func() { runDone <- w.Run(ctx) }() @@ -81,7 +83,7 @@ func TestWriterAcknowledgesOnlyAfterAtomicCommit(t *testing.T) { func TestWriterGroupsQueuedSubmissions(t *testing.T) { committer := &recordingCommitter{} - w := testWriter(committer, maxBatchRows) + w := testWriter(committer, maxGroupBatchRows) results := make(chan error, 4) for i := range 4 { go func() { @@ -108,13 +110,13 @@ func TestWriterGroupsQueuedSubmissions(t *testing.T) { } } -func TestWriterSplitsOversizedSubmission(t *testing.T) { +func TestWriterCommitsOversizedSubmissionAtomically(t *testing.T) { committer := &recordingCommitter{} - w := testWriter(committer, maxBatchRows) + w := testWriter(committer, maxGroupBatchRows) ctx, cancel := context.WithCancel(context.Background()) runDone := make(chan error, 1) go func() { runDone <- w.Run(ctx) }() - if err := w.Submit(context.Background(), Batch{Spans: make([]telemetry.Span, maxBatchRows+1)}); err != nil { + if err := w.Submit(context.Background(), Batch{Spans: make([]telemetry.Span, maxGroupBatchRows+1)}); err != nil { t.Fatal(err) } cancel() @@ -123,16 +125,14 @@ func TestWriterSplitsOversizedSubmission(t *testing.T) { } committer.mu.Lock() defer committer.mu.Unlock() - if len(committer.batches) != 2 { - t.Fatalf("committed batches = %d, want 2", len(committer.batches)) + if len(committer.batches) != 1 { + t.Fatalf("committed batches = %d, want one atomic request", len(committer.batches)) } - if batchRows(committer.batches[0])+batchRows(committer.batches[1]) != maxBatchRows+1 { - t.Fatal("split lost rows") + if batchRows(committer.batches[0]) != maxGroupBatchRows+1 { + t.Fatal("atomic oversized commit lost rows") } - for _, batch := range committer.batches { - if batch.ID == "" || batchRows(batch) > maxBatchRows { - t.Fatalf("invalid chunk: id=%q rows=%d", batch.ID, batchRows(batch)) - } + if committer.batches[0].ID == "" { + t.Fatal("oversized batch has no commit ID") } } @@ -157,7 +157,38 @@ func TestWriterRetriesTransientCommitFailure(t *testing.T) { } } +func TestWriterRecordsLiveQueueAndFlushMetrics(t *testing.T) { + metrics.FlushTotal.Reset() + metrics.IngestQueueDepth.Reset() + committer := &recordingCommitter{} + w := testWriter(committer, 10) + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- w.Run(ctx) }() + batch := Batch{ + Spans: []telemetry.Span{{SpanID: "span"}}, + Logs: []telemetry.Log{{Body: "log"}}, + Metrics: []telemetry.Metric{{Name: "metric"}}, + } + if err := w.Submit(context.Background(), batch); err != nil { + t.Fatal(err) + } + cancel() + if err := <-runDone; err != nil { + t.Fatal(err) + } + for _, signal := range []string{"spans", "logs", "metrics"} { + if got := testutil.ToFloat64(metrics.FlushTotal.WithLabelValues(signal)); got != 1 { + t.Fatalf("flush total for %s = %v, want 1", signal, got) + } + } + if got := testutil.ToFloat64(metrics.IngestQueueDepth.WithLabelValues("batch")); got != 0 { + t.Fatalf("submission queue depth after shutdown = %v, want 0", got) + } +} + func TestWriterSurfacesPermanentFailureToSubmitAndRun(t *testing.T) { + metrics.RowsDropped.Reset() committer := &recordingCommitter{failures: commitRetryLimit + 1} w := testWriter(committer, 1) w.retryDelay = func(int) time.Duration { return 0 } @@ -174,6 +205,9 @@ func TestWriterSurfacesPermanentFailureToSubmitAndRun(t *testing.T) { if committer.calls != commitRetryLimit || len(committer.batches) != 0 { t.Fatalf("calls=%d committed=%d", committer.calls, len(committer.batches)) } + if got := testutil.ToFloat64(metrics.RowsDropped.WithLabelValues("spans")); got != 1 { + t.Fatalf("dropped span rows = %v, want 1", got) + } } type parallelCommitter struct { From 8f205d6b93d6f65820d3850919f997feef844f23 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 15:40:05 -0700 Subject: [PATCH 16/31] fix(storage): bound Parquet publication Keep validation and cleanup outside reader exclusion, preserve service availability after commit exhaustion, and prune trace indexes by batch event range. Batch metadata advances to version 2; version 1 is intentionally unsupported. --- internal/api/health.go | 12 +- internal/api/health_test.go | 28 ++++ internal/query/duck.go | 62 ++++--- internal/query/duck_test.go | 22 +++ internal/query/parquet_gate.go | 26 ++- internal/query/parquet_gate_test.go | 20 +++ internal/query/writegate/write_gate.go | 36 ++++- internal/query/writegate/write_gate_test.go | 20 ++- internal/telemetry/parquet.go | 171 +++++++++++--------- internal/telemetry/parquet_rows.go | 15 +- internal/telemetry/parquet_test.go | 49 ++++++ internal/telemetry/rows.go | 10 ++ internal/telemetry/store/compaction.go | 48 +++++- internal/telemetry/store/repository.go | 21 +-- internal/telemetry/store/repository_test.go | 80 ++++++++- internal/telemetry/store/writer.go | 44 ++--- internal/telemetry/store/writer_test.go | 22 ++- 17 files changed, 511 insertions(+), 175 deletions(-) diff --git a/internal/api/health.go b/internal/api/health.go index fb98a719..180ffddd 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -3,6 +3,7 @@ package api import ( "context" "database/sql" + "errors" "fmt" "log/slog" "net/http" @@ -177,6 +178,8 @@ func maintenanceStaleThreshold(maintEvery time.Duration) time.Duration { return stale } +var telemetryReadinessTimeout = 5 * time.Second + func (h *HealthHandler) checkTelemetry() CheckResult { if h.duck == nil { return CheckResult{ @@ -186,13 +189,20 @@ func (h *HealthHandler) checkTelemetry() CheckResult { } start := time.Now() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), telemetryReadinessTimeout) defer cancel() var one int // Use the same snapshot gate as public reads so a short compaction/retention // publication cannot race the Parquet scan and report a false outage. err := h.duck.QueryRowScan(ctx, []any{&one}, "SELECT 1 FROM telemetry.spans LIMIT 1") + if errors.Is(err, query.ErrParquetReadWait) { + return CheckResult{ + Status: "degraded", + LatencyMs: time.Since(start).Milliseconds(), + Error: err.Error(), + } + } if err != nil && err != sql.ErrNoRows { return CheckResult{ Status: "unhealthy", diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 5da7e158..f8ec87db 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "errors" "net/http" @@ -182,6 +183,33 @@ func TestReadiness_HealthyTelemetryAndRollups(t *testing.T) { } } +func TestTelemetryReadinessReportsPublicationContentionAsDegraded(t *testing.T) { + duck := &query.Duck{} + entered := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + done <- duck.PublishParquet(context.Background(), func() error { + close(entered) + <-release + return nil + }) + }() + <-entered + original := telemetryReadinessTimeout + telemetryReadinessTimeout = 20 * time.Millisecond + t.Cleanup(func() { telemetryReadinessTimeout = original }) + + result := NewHealthHandler(duck, config.Config{}).checkTelemetry() + if result.Status != "degraded" { + t.Fatalf("telemetry status = %q, want degraded: %+v", result.Status, result) + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } +} + // A maintenance pass that never executes must degrade once the startup grace // period elapses — the wedged-first-rollup case the check exists to expose. func TestCheckMaintenance_DegradedWhenNeverRanPastGrace(t *testing.T) { diff --git a/internal/query/duck.go b/internal/query/duck.go index f9f178e5..454360d4 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -446,15 +446,11 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() - unlockMaintenance := d.writeGate.Lock(writegate.WriteMaintenance) var pruneErr error if d.repository != nil { var parquetErr error if d.cfg.RetentionDays > 0 { - parquetErr = d.PublishParquet(func() error { - _, err := d.repository.PruneParquet(time.Now().Add(-time.Duration(d.cfg.RetentionDays) * 24 * time.Hour).UnixNano()) - return err - }) + _, parquetErr = d.repository.PruneParquet(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano()) } compactStart := time.Now() compacted, compactErr := d.repository.CompactParquetBacklog(ctx, d, 64) @@ -467,17 +463,24 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) pruneErr = errors.Join(pruneErr, parquetErr, compactErr) } - var cacheErr error - if d.cfg.RetentionDays > 0 { - for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { - if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { - cacheErr = errors.Join(cacheErr, fmt.Errorf("prune %s: %w", table, err)) + cacheErr := func() error { + unlock, err := d.writeGate.LockContext(ctx, writegate.WriteMaintenance) + if err != nil { + return err + } + defer unlock() + var errs []error + if d.cfg.RetentionDays > 0 { + for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { + if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { + errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) + } } } - } - _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") - unlockMaintenance() - err := errors.Join(pruneErr, cacheErr, checkpointErr) + _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") + return errors.Join(errors.Join(errs...), checkpointErr) + }() + err := errors.Join(pruneErr, cacheErr) maintenanceResult := metrics.TelemetrySuccess if err != nil { maintenanceResult = metrics.TelemetryError @@ -1425,9 +1428,14 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet } // MergeParquet executes the query-engine-specific half of compaction. The -// maintenance caller already holds writeGate, so this cannot contend with a -// rollup transaction while it uses the shared DuckDB pool. +// write is serialized with rollup-cache transactions, but only for this one +// merge so a large compaction backlog cannot starve rollups. func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { + unlock, err := d.writeGate.LockContext(ctx, writegate.WriteMaintenance) + if err != nil { + return err + } + defer unlock() quoted := make([]string, len(inputs)) for i, input := range inputs { quoted[i] = quoteDuckString(input) @@ -1437,13 +1445,24 @@ func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, query += " ORDER BY _trace_hash, start_unix_nano, span_id" } stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, quoteDuckString(output)) - _, err := d.DB.ExecContext(ctx, stmt) + _, err = d.DB.ExecContext(ctx, stmt) return err } // PublishParquet limits reader exclusion to the atomic directory swap. -func (d *Duck) PublishParquet(publish func() error) error { - d.parquetMu.Lock() +func (d *Duck) PublishParquet(ctx context.Context, publish func() error) error { + writeCtx, cancelWrite := context.WithTimeout(ctx, 2*defaultWriterGrace) + unlockWrite, err := d.writeGate.LockContext(writeCtx, writegate.WriteMaintenance) + cancelWrite() + if err != nil { + return fmt.Errorf("wait for DuckDB write gate: %w", err) + } + defer unlockWrite() + publishCtx, cancelPublish := context.WithTimeout(ctx, 2*defaultWriterGrace) + defer cancelPublish() + if err := d.parquetMu.LockContext(publishCtx); err != nil { + return fmt.Errorf("wait for Parquet readers: %w", err) + } defer d.parquetMu.Unlock() return publish() } @@ -1451,7 +1470,10 @@ func (d *Duck) PublishParquet(publish func() error) error { func quoteDuckString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } func (d *Duck) lockParquetRead(ctx context.Context) error { - return d.parquetMu.RLockContext(ctx) + if err := d.parquetMu.RLockContext(ctx); err != nil { + return errors.Join(ErrParquetReadWait, err) + } + return nil } // ---- Queries for API ---- diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 22da7a7c..07ed0f22 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -278,6 +278,28 @@ func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { } } +func TestPublishParquetHonorsContext(t *testing.T) { + d := &Duck{} + d.parquetMu.RLock() + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + called := false + err := d.PublishParquet(ctx, func() error { + called = true + return nil + }) + d.parquetMu.RUnlock() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("PublishParquet error = %v, want deadline exceeded", err) + } + if called { + t.Fatal("publication ran after its context expired") + } + if got := d.parquetMu.WaitingWriters(); got != 0 { + t.Fatalf("canceled publication remained queued: %d", got) + } +} + func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 6ae834d9..4db7b304 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -2,6 +2,7 @@ package query import ( "context" + "errors" "sync" "time" ) @@ -11,6 +12,9 @@ import ( // behind maintenance, short enough that retention and compaction always run. const defaultWriterGrace = 30 * time.Second +// ErrParquetReadWait distinguishes publication contention from a query error. +var ErrParquetReadWait = errors.New("wait for Parquet publication") + // parquetReadGate protects Parquet file publication without sync.RWMutex's // unconditional writer preference. Queuing maintenance must not stall // unrelated API reads or readiness probes, so readers continue to be admitted @@ -120,6 +124,12 @@ func (g *parquetReadGate) RUnlock() { } func (g *parquetReadGate) Lock() { + if err := g.LockContext(context.Background()); err != nil { + panic(err) + } +} + +func (g *parquetReadGate) LockContext(ctx context.Context) error { g.init() g.mu.Lock() g.nextWaiter++ @@ -131,7 +141,20 @@ func (g *parquetReadGate) Lock() { for g.writer || g.readers > 0 { changed := g.changed g.mu.Unlock() - <-changed + select { + case <-ctx.Done(): + g.mu.Lock() + for i, queued := range g.waiting { + if queued.id == waiter.id { + g.waiting = append(g.waiting[:i], g.waiting[i+1:]...) + break + } + } + g.notifyLocked() + g.mu.Unlock() + return ctx.Err() + case <-changed: + } g.mu.Lock() } for i, queued := range g.waiting { @@ -142,6 +165,7 @@ func (g *parquetReadGate) Lock() { } g.writer = true g.mu.Unlock() + return nil } func (g *parquetReadGate) Unlock() { diff --git a/internal/query/parquet_gate_test.go b/internal/query/parquet_gate_test.go index 3d020a62..0ec9211e 100644 --- a/internal/query/parquet_gate_test.go +++ b/internal/query/parquet_gate_test.go @@ -1,6 +1,8 @@ package query import ( + "context" + "errors" "sync" "testing" "time" @@ -12,6 +14,24 @@ type fakeClock struct { now time.Time } +func TestParquetGateRemovesCanceledPublisher(t *testing.T) { + var gate parquetReadGate + gate.RLock() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := gate.LockContext(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("LockContext error = %v, want deadline exceeded", err) + } + if got := gate.WaitingWriters(); got != 0 { + t.Fatalf("canceled publisher remained queued: %d", got) + } + gate.RUnlock() + if !gate.TryRLock() { + t.Fatal("canceled publisher continued blocking readers") + } + gate.RUnlock() +} + func (c *fakeClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/query/writegate/write_gate.go b/internal/query/writegate/write_gate.go index 94f797b5..dbeebfa9 100644 --- a/internal/query/writegate/write_gate.go +++ b/internal/query/writegate/write_gate.go @@ -3,6 +3,7 @@ package writegate import ( + "context" "sync" "time" @@ -22,11 +23,18 @@ const ( WriteMaintenance WriteOperation = "maintenance" ) -// WriteGate preserves the existing process-wide sync.Mutex acquisition -// semantics while measuring how long each operation waits for and holds the -// catalog write critical section. The zero value is ready for use. +// WriteGate serializes DuckDB writes while measuring how long each operation +// waits for and holds the critical section. The zero value is ready for use. type WriteGate struct { - mu sync.Mutex + once sync.Once + token chan struct{} +} + +func (g *WriteGate) init() { + g.once.Do(func() { + g.token = make(chan struct{}, 1) + g.token <- struct{}{} + }) } // observe is a package-level seam so a test can prove the observation happens @@ -39,15 +47,29 @@ var observe = metrics.RecordWriteGate // Callers must defer the returned function before acquiring a database // connection, transaction, or appender, and must call it exactly once. func (g *WriteGate) Lock(operation WriteOperation) func() { + unlock, err := g.LockContext(context.Background(), operation) + if err != nil { + panic(err) + } + return unlock +} + +// LockContext acquires the cache write gate, or returns when ctx expires. +func (g *WriteGate) LockContext(ctx context.Context, operation WriteOperation) (func(), error) { + g.init() waitStarted := time.Now() - g.mu.Lock() + select { + case <-g.token: + case <-ctx.Done(): + return nil, ctx.Err() + } acquired := time.Now() return func() { hold := time.Since(acquired) // Unlock before observing: a Prometheus histogram takes its own lock, // and holding the process's hottest critical section across that would // be a throughput regression in the code added to detect one. - g.mu.Unlock() + g.token <- struct{}{} observe(string(operation), acquired.Sub(waitStarted).Seconds(), hold.Seconds()) - } + }, nil } diff --git a/internal/query/writegate/write_gate_test.go b/internal/query/writegate/write_gate_test.go index a19f5c03..e0929fef 100644 --- a/internal/query/writegate/write_gate_test.go +++ b/internal/query/writegate/write_gate_test.go @@ -4,6 +4,8 @@ package writegate // repository lock and never pass through DuckDB. import ( + "context" + "errors" "testing" "time" @@ -11,6 +13,18 @@ import ( dto "github.com/prometheus/client_model/go" ) +func TestWriteGateLockContextHonorsCancellation(t *testing.T) { + var gate WriteGate + unlock := gate.Lock(WriteMaintenance) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := gate.LockContext(ctx, WriteMaintenance); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("LockContext error = %v, want deadline exceeded", err) + } + unlock() + gate.Lock(WriteMaintenance)() +} + func TestWriteGateSerializesHoldersInAcquisitionOrder(t *testing.T) { t.Parallel() @@ -92,9 +106,11 @@ func TestWriteGateObservesOutsideTheCriticalSection(t *testing.T) { var freeDuringObserve bool restore := swapObserver(t, func(string, float64, float64) { - if gate.mu.TryLock() { + select { + case <-gate.token: freeDuringObserve = true - gate.mu.Unlock() + gate.token <- struct{}{} + default: } }) defer restore() diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 0343fec5..f11f2ae9 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -22,21 +22,23 @@ import ( const ( BatchSuffix = ".batch" SchemaBatch = "_schema" + BatchSuffix - batchMetadataVersion = 1 + batchMetadataVersion = 2 parquetPageSize = 64 << 10 parquetRowGroupRows = 50_000 maxTraceQueryResults = 500 ) type BatchMetadata struct { - Version uint32 `json:"version"` - ID string `json:"id"` - MinIngestedNanos int64 `json:"min_ingested_nanos"` - MaxIngestedNanos int64 `json:"max_ingested_nanos"` - Generation uint32 `json:"generation"` - Spans int `json:"spans"` - Logs int `json:"logs"` - Metrics int `json:"metrics"` + Version uint32 `json:"version"` + ID string `json:"id"` + MinIngestedNanos int64 `json:"min_ingested_nanos"` + MaxIngestedNanos int64 `json:"max_ingested_nanos"` + MinSpanStartNanos int64 `json:"min_span_start_nanos"` + MaxSpanStartNanos int64 `json:"max_span_start_nanos"` + Generation uint32 `json:"generation"` + Spans int `json:"spans"` + Logs int `json:"logs"` + Metrics int `json:"metrics"` } type TraceQuery struct { @@ -177,6 +179,10 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] for i := range spans { rows[i] = makeSpanParquetRow(spans[i]) rows[i].TraceHash = xxh3.HashString(rows[i].TraceID) + if rows[i].StartUnixNano > 0 && (metadata.MinSpanStartNanos == 0 || rows[i].StartUnixNano < metadata.MinSpanStartNanos) { + metadata.MinSpanStartNanos = rows[i].StartUnixNano + } + metadata.MaxSpanStartNanos = max(metadata.MaxSpanStartNanos, rows[i].StartUnixNano) } sort.Slice(rows, func(i, j int) bool { if rows[i].TraceHash != rows[j].TraceHash { @@ -301,6 +307,10 @@ func (p *ParquetStore) Trace(ctx context.Context, query TraceQuery) ([]IndexedSp if err := ctx.Err(); err != nil { return nil, err } + if batch.metadata.MaxSpanStartNanos > 0 && + (batch.metadata.MaxSpanStartNanos < query.StartNanos || batch.metadata.MinSpanStartNanos >= query.EndNanos) { + continue + } match, found, err := batch.traces.Lookup(hash) if err != nil { return nil, err @@ -400,30 +410,34 @@ func indexedSpanEarlier(left, right IndexedSpan) bool { return left.SpanID < right.SpanID } -// PruneBefore hides complete batches before deleting them. The caller pins -// DuckDB readers while this method runs. -func (p *ParquetStore) PruneBefore(cutoff int64) (int, error) { - p.publishMu.Lock() - defer p.publishMu.Unlock() - p.mu.Lock() - defer p.mu.Unlock() +// PruneBefore hides complete batches while readers are pinned, then deletes +// the retired directories after publication. +func (p *ParquetStore) PruneBefore(cutoff int64, publish func(func() error) error) (int, error) { var retired []string var pruneErr error - for id, batch := range p.batches { - if batch.metadata.MaxIngestedNanos <= 0 || batch.metadata.MaxIngestedNanos >= cutoff { - continue + err := publish(func() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() + for id, batch := range p.batches { + if batch.metadata.MaxIngestedNanos <= 0 || batch.metadata.MaxIngestedNanos >= cutoff { + continue + } + path := filepath.Join(p.batchesDir, id+".retired") + if err := os.Rename(batch.dir, path); err != nil { + pruneErr = errors.Join(pruneErr, err) + continue + } + delete(p.batches, id) + retired = append(retired, path) } - path := filepath.Join(p.batchesDir, id+".retired") - if err := os.Rename(batch.dir, path); err != nil { - pruneErr = errors.Join(pruneErr, err) - continue + if len(retired) > 0 { + pruneErr = errors.Join(pruneErr, syncDirectory(p.batchesDir)) } - delete(p.batches, id) - retired = append(retired, path) - } - if len(retired) > 0 { - pruneErr = errors.Join(pruneErr, syncDirectory(p.batchesDir)) - } + return nil + }) + pruneErr = errors.Join(pruneErr, err) for _, path := range retired { pruneErr = errors.Join(pruneErr, os.RemoveAll(path)) } @@ -463,6 +477,8 @@ func (p *ParquetStore) Stats() (map[string]ParquetStats, error) { // files produced by the compactor. func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) error { metadata.Version = batchMetadataVersion + metadata.MinSpanStartNanos = 0 + metadata.MaxSpanStartNanos = 0 if metadata.Spans > 0 { f, err := os.Open(filepath.Join(dir, "spans.parquet")) if err != nil { @@ -480,6 +496,10 @@ func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) er for { n, readErr := reader.Read(buffer) for i := range n { + if start := buffer[i].StartUnixNano; start > 0 && (metadata.MinSpanStartNanos == 0 || start < metadata.MinSpanStartNanos) { + metadata.MinSpanStartNanos = start + } + metadata.MaxSpanStartNanos = max(metadata.MaxSpanStartNanos, buffer[i].StartUnixNano) if err := index.Append(buffer[i].TraceHash); err != nil { index.Abort() _ = reader.Close() @@ -522,11 +542,9 @@ func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) er return syncDirectory(dir) } -// PublishReplacement swaps a prepared compacted batch for its inputs while -// holding the store lock, so native trace readers cannot observe removed files. -func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string) error { - p.publishMu.Lock() - defer p.publishMu.Unlock() +// PublishReplacement validates a prepared compacted batch, atomically swaps it +// for its inputs while readers are pinned, then deletes retired inputs. +func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string, publish func(func() error) error) error { final := p.BatchPath(metadata.ID) source := stage if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { @@ -543,52 +561,58 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, replacement.traces.path = filepath.Join(final, "trace.fidx") } - p.mu.Lock() - defer p.mu.Unlock() retired := make([][2]string, 0, len(inputs)) - rollback := func() error { - var rollbackErr error - if source == final { - if _, statErr := os.Stat(stage); errors.Is(statErr, os.ErrNotExist) { - rollbackErr = errors.Join(rollbackErr, os.Rename(final, stage)) + err = publish(func() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() + rollback := func() error { + var rollbackErr error + if source == final { + if _, statErr := os.Stat(stage); errors.Is(statErr, os.ErrNotExist) { + rollbackErr = errors.Join(rollbackErr, os.Rename(final, stage)) + } + } + for i := len(retired) - 1; i >= 0; i-- { + rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) + } + return errors.Join(rollbackErr, syncDirectory(p.batchesDir)) + } + for _, id := range inputs { + active := p.BatchPath(id) + retiredPath := filepath.Join(p.batchesDir, id+".retired-"+metadata.ID) + if _, statErr := os.Stat(active); statErr == nil { + if err := os.Rename(active, retiredPath); err != nil { + return errors.Join(err, rollback()) + } + retired = append(retired, [2]string{active, retiredPath}) + } else if !errors.Is(statErr, os.ErrNotExist) { + return errors.Join(statErr, rollback()) + } else if _, retiredErr := os.Stat(retiredPath); retiredErr == nil { + retired = append(retired, [2]string{active, retiredPath}) + } else if !errors.Is(retiredErr, os.ErrNotExist) { + return errors.Join(retiredErr, rollback()) } } - for i := len(retired) - 1; i >= 0; i-- { - rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) - } - return errors.Join(rollbackErr, syncDirectory(p.batchesDir)) - } - for _, id := range inputs { - active := p.BatchPath(id) - retiredPath := filepath.Join(p.batchesDir, id+".retired-"+metadata.ID) - if _, statErr := os.Stat(active); statErr == nil { - if err := os.Rename(active, retiredPath); err != nil { + if source != final { + if err := os.Rename(stage, final); err != nil { return errors.Join(err, rollback()) } - retired = append(retired, [2]string{active, retiredPath}) - } else if !errors.Is(statErr, os.ErrNotExist) { - return errors.Join(statErr, rollback()) - } else if _, retiredErr := os.Stat(retiredPath); retiredErr == nil { - // Recovery can resume after inputs were retired but before the - // replacement or directory sync completed. - retired = append(retired, [2]string{active, retiredPath}) - } else if !errors.Is(retiredErr, os.ErrNotExist) { - return errors.Join(retiredErr, rollback()) - } - } - if source != final { - if err := os.Rename(stage, final); err != nil { + source = final + } + if err := syncDirectory(p.batchesDir); err != nil { return errors.Join(err, rollback()) } - source = final - } - if err := syncDirectory(p.batchesDir); err != nil { - return errors.Join(err, rollback()) - } - for _, id := range inputs { - delete(p.batches, id) + for _, id := range inputs { + delete(p.batches, id) + } + p.batches[metadata.ID] = replacement + return nil + }) + if err != nil { + return err } - p.batches[metadata.ID] = replacement var removeErr error for _, pair := range retired { removeErr = errors.Join(removeErr, os.RemoveAll(pair[1])) @@ -812,5 +836,6 @@ func validateBatchID(id string) error { } type traceParquetRow struct { - TraceHash uint64 `parquet:"_trace_hash"` + TraceHash uint64 `parquet:"_trace_hash"` + StartUnixNano int64 `parquet:"start_unix_nano"` } diff --git a/internal/telemetry/parquet_rows.go b/internal/telemetry/parquet_rows.go index fe922756..f275c15b 100644 --- a/internal/telemetry/parquet_rows.go +++ b/internal/telemetry/parquet_rows.go @@ -100,8 +100,8 @@ type logParquetRow struct { func makeLogParquetRow(r Log) logParquetRow { return logParquetRow{ - Namespace: r.Namespace, LogTime: firstPositive(r.TimeUnixNanos, r.ObservedTimeNanos, r.IngestedAt), - ObservedTime: firstPositive(r.ObservedTimeNanos, r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, + Namespace: r.Namespace, LogTime: FirstPositiveNanos(r.EventUnixNanos, r.TimeUnixNanos, r.ObservedTimeNanos, r.IngestedAt), + ObservedTime: FirstPositiveNanos(r.ObservedTimeNanos, r.EventUnixNanos, r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, ObservedTimeUnixNano: r.ObservedTimeNanos, Severity: r.Severity, SeverityNumber: int64(r.SeverityNumber), Body: r.Body, Service: r.ServiceName, TraceID: r.TraceID, SpanID: r.SpanID, Flags: int64(r.Flags), ResourceJSON: string(r.ResourceJSON), AttributesJSON: string(r.AttributesJSON), ScopeName: r.ScopeName, @@ -134,7 +134,7 @@ type metricParquetRow struct { func makeMetricParquetRow(r Metric) metricParquetRow { return metricParquetRow{ - Namespace: r.Namespace, MetricTime: firstPositive(r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, + Namespace: r.Namespace, MetricTime: FirstPositiveNanos(r.EventUnixNanos, r.TimeUnixNanos, r.IngestedAt), TimeUnixNano: r.TimeUnixNanos, Name: r.Name, Description: r.Description, Unit: r.Unit, MetricType: r.Type, Service: r.ServiceName, Value: r.Value, HistBoundsJSON: string(r.HistBoundsJSON), HistCountsJSON: string(r.HistCountsJSON), HistCount: r.HistCount, HistSum: r.HistSum, ExemplarsJSON: string(r.ExemplarsJSON), @@ -142,12 +142,3 @@ func makeMetricParquetRow(r Metric) metricParquetRow { ScopeVersion: r.ScopeVersion, IngestedAt: r.IngestedAt, IngestedUnixNano: r.IngestedAt, } } - -func firstPositive(values ...int64) int64 { - for _, value := range values { - if value > 0 { - return value - } - } - return 0 -} diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index aa6edf2e..cc1beec0 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -163,15 +163,64 @@ func TestParquetStorePreservesCompleteLogAndMetricRows(t *testing.T) { } batchDir := store.BatchPath("complete-signals") gotLog := readOneParquetRow[logParquetRow](t, filepath.Join(batchDir, "logs.parquet")) + if gotLog.LogTime != logRow.EventUnixNanos { + t.Fatalf("log event time = %d, want canonical %d", gotLog.LogTime, logRow.EventUnixNanos) + } if want := makeLogParquetRow(logRow); !reflect.DeepEqual(gotLog, want) { t.Fatalf("log row mismatch\n got: %#v\nwant: %#v", gotLog, want) } gotMetric := readOneParquetRow[metricParquetRow](t, filepath.Join(batchDir, "metrics.parquet")) + if gotMetric.MetricTime != metricRow.EventUnixNanos { + t.Fatalf("metric event time = %d, want canonical %d", gotMetric.MetricTime, metricRow.EventUnixNanos) + } if want := makeMetricParquetRow(metricRow); !reflect.DeepEqual(gotMetric, want) { t.Fatalf("metric row mismatch\n got: %#v\nwant: %#v", gotMetric, want) } } +func TestParquetStoreSkipsTraceIndexesOutsideTimeWindow(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(BatchMetadata{ID: "old-traces"}, []Span{{ + TraceID: "wanted", SpanID: "old", StartUnixNanos: 100, + }}, nil, nil); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(store.BatchPath("old-traces"), "trace.fidx")); err != nil { + t.Fatal(err) + } + got, err := store.Trace(context.Background(), TraceQuery{ + TraceID: "wanted", StartNanos: 1_000, EndNanos: 2_000, Limit: 10, + }) + if err != nil { + t.Fatalf("non-overlapping query opened old trace index: %v", err) + } + if len(got) != 0 { + t.Fatalf("non-overlapping trace query returned %#v", got) + } +} + +func TestPublishReplacementValidatesBeforePublication(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + stage := t.TempDir() + called := false + err = store.PublishReplacement(stage, BatchMetadata{ID: "replacement"}, nil, func(publish func() error) error { + called = true + return publish() + }) + if err == nil { + t.Fatal("invalid replacement was accepted") + } + if called { + t.Fatal("publication gate entered before replacement validation") + } +} + func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { diff --git a/internal/telemetry/rows.go b/internal/telemetry/rows.go index 1b403f66..09b7e4a7 100644 --- a/internal/telemetry/rows.go +++ b/internal/telemetry/rows.go @@ -102,3 +102,13 @@ func NormalizeNamespace(namespace string) string { } return namespace } + +// FirstPositiveNanos returns the first usable timestamp in priority order. +func FirstPositiveNanos(values ...int64) int64 { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index a9f43917..139b5c92 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -29,9 +29,13 @@ type compactionKey struct { // ParquetCompactor keeps DuckDB execution and publication locking in the query // layer while storage owns batch selection and crash-safe replacement state. +type ParquetPublisher interface { + PublishParquet(context.Context, func() error) error +} + type ParquetCompactor interface { + ParquetPublisher MergeParquet(context.Context, string, []string, string) error - PublishParquet(func() error) error } // CompactParquet combines one same-day, same-generation group. The output is @@ -46,7 +50,7 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac if exists, err := pathExists(markerPath); err != nil { return 0, err } else if exists { - if err := compactor.PublishParquet(r.recoverCompaction); err != nil { + if err := r.recoverCompaction(ctx, compactor.PublishParquet); err != nil { return 0, fmt.Errorf("recover pending Parquet compaction: %w", err) } } @@ -79,6 +83,9 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac if err := os.MkdirAll(stage, 0o755); err != nil { return 0, err } + if err := errors.Join(syncDirectory(filepath.Dir(stage)), syncDirectory(r.root)); err != nil { + return 0, err + } prepared := false defer func() { if !prepared { @@ -120,7 +127,7 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac return 0, err } prepared = true - if err := compactor.PublishParquet(func() error { return r.completeCompaction(marker) }); err != nil { + if err := r.completeCompaction(ctx, marker, compactor.PublishParquet); err != nil { return 0, err } return len(selected), nil @@ -169,7 +176,9 @@ func (r *Repository) CompactParquetBacklog(ctx context.Context, compactor Parque } } -func (r *Repository) recoverCompaction() error { +type parquetPublishFunc func(context.Context, func() error) error + +func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPublishFunc) error { data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) if errors.Is(err, os.ErrNotExist) { return nil @@ -181,11 +190,36 @@ func (r *Repository) recoverCompaction() error { if err := json.Unmarshal(data, &marker); err != nil { return err } - return r.completeCompaction(marker) + stageExists, err := pathExists(r.compactionStage(marker.Output.ID)) + if err != nil { + return err + } + finalExists, err := pathExists(r.Parquet.BatchPath(marker.Output.ID)) + if err != nil { + return err + } + if !stageExists && !finalExists { + for _, id := range marker.Inputs { + exists, statErr := pathExists(r.Parquet.BatchPath(id)) + if statErr != nil { + return statErr + } + if !exists { + return fmt.Errorf("compaction %s has no output and input %s is missing", marker.Output.ID, id) + } + } + if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(r.root) + } + return r.completeCompaction(ctx, marker, publish) } -func (r *Repository) completeCompaction(marker compactionMarker) error { - if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs); err != nil { +func (r *Repository) completeCompaction(ctx context.Context, marker compactionMarker, publish parquetPublishFunc) error { + if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs, func(swap func() error) error { + return publish(ctx, swap) + }); err != nil { return err } markerPath := filepath.Join(r.root, "COMPACTION.json") diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index d154c301..832e5273 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -38,7 +38,7 @@ func Open(root string) (*Repository, error) { return nil, err } r := &Repository{root: root, Parquet: parquetStore} - if err := r.recoverCompaction(); err != nil { + if err := r.recoverCompaction(context.Background(), func(_ context.Context, publish func() error) error { return publish() }); err != nil { _ = r.Close() return nil, fmt.Errorf("recover Parquet compaction: %w", err) } @@ -82,10 +82,12 @@ func (r *Repository) Trace(ctx context.Context, query telemetry.TraceQuery) ([]t func (r *Repository) RowCount() uint64 { return r.Parquet.RowCount() } -func (r *Repository) PruneParquet(cutoff int64) (int, error) { +func (r *Repository) PruneParquet(ctx context.Context, publisher ParquetPublisher, cutoff int64) (int, error) { r.compactionMu.Lock() defer r.compactionMu.Unlock() - return r.Parquet.PruneBefore(cutoff) + return r.Parquet.PruneBefore(cutoff, func(prune func() error) error { + return publisher.PublishParquet(ctx, prune) + }) } func validateBatch(batch Batch) error { @@ -115,7 +117,7 @@ func normalizeBatch(batch *Batch) { batch.Logs[i].IngestedAt = ingestedAt } if batch.Logs[i].EventUnixNanos == 0 { - batch.Logs[i].EventUnixNanos = firstNonzero(batch.Logs[i].TimeUnixNanos, batch.Logs[i].ObservedTimeNanos, batch.Logs[i].IngestedAt) + batch.Logs[i].EventUnixNanos = telemetry.FirstPositiveNanos(batch.Logs[i].TimeUnixNanos, batch.Logs[i].ObservedTimeNanos, batch.Logs[i].IngestedAt) } } for i := range batch.Metrics { @@ -124,7 +126,7 @@ func normalizeBatch(batch *Batch) { batch.Metrics[i].IngestedAt = ingestedAt } if batch.Metrics[i].EventUnixNanos == 0 { - batch.Metrics[i].EventUnixNanos = firstNonzero(batch.Metrics[i].TimeUnixNanos, batch.Metrics[i].IngestedAt) + batch.Metrics[i].EventUnixNanos = telemetry.FirstPositiveNanos(batch.Metrics[i].TimeUnixNanos, batch.Metrics[i].IngestedAt) } } } @@ -164,12 +166,3 @@ func batchMinIngestedNanos(batch Batch) int64 { } return value } - -func firstNonzero(values ...int64) int64 { - for _, value := range values { - if value != 0 { - return value - } - } - return 0 -} diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 7bec33e2..814957d1 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -18,6 +18,7 @@ import ( type testParquetCompactor struct { db *sql.DB publishErr error + afterSwap func() error } func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { @@ -33,11 +34,17 @@ func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, return err } -func (c *testParquetCompactor) PublishParquet(publish func() error) error { +func (c *testParquetCompactor) PublishParquet(_ context.Context, publish func() error) error { if c.publishErr != nil { return c.publishErr } - return publish() + if err := publish(); err != nil { + return err + } + if c.afterSwap != nil { + return c.afterSwap() + } + return nil } func sqlQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } @@ -190,7 +197,21 @@ func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { if err := repository.Commit(newer); err != nil { t.Fatal(err) } - removed, err := repository.PruneParquet(500) + publisher := &testParquetCompactor{afterSwap: func() error { + if repository.compactionMu.TryLock() { + repository.compactionMu.Unlock() + return errors.New("retention published without holding compaction lock") + } + retired, err := filepath.Glob(filepath.Join(repository.Parquet.BatchesDir(), "*.retired")) + if err != nil { + return err + } + if len(retired) != 1 { + return fmt.Errorf("retired directories during publication = %d, want 1", len(retired)) + } + return nil + }} + removed, err := repository.PruneParquet(context.Background(), publisher, 500) if err != nil || removed != 1 { t.Fatalf("prune = %d, %v", removed, err) } @@ -205,6 +226,45 @@ func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { } } +func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + batch := testBatch() + batch.ID = "intact-input" + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + marker := compactionMarker{ + Output: telemetry.BatchMetadata{ID: "missing-output", Generation: 1}, + Inputs: []string{batch.ID}, + } + data, err := json.Marshal(marker) + if err != nil { + t.Fatal(err) + } + if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + + recovered, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if got := recovered.RowCount(); got != 3 { + t.Fatalf("rows after marker recovery = %d, want 3", got) + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { + t.Fatalf("recoverable marker remains: %v", err) + } +} + func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { @@ -222,7 +282,7 @@ func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { if err := repository.Commit(batch); err != nil { t.Fatal(err) } - removed, err := repository.PruneParquet(500) + removed, err := repository.PruneParquet(context.Background(), &testParquetCompactor{}, 500) if err != nil || removed != 1 { t.Fatalf("prune future-dated events = %d, %v; want one batch expired by ingest time", removed, err) } @@ -246,7 +306,17 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } db := openTestDuckDB(t) defer db.Close() - compacted, err := repository.CompactParquet(context.Background(), &testParquetCompactor{db: db}, 64) + compactor := &testParquetCompactor{db: db, afterSwap: func() error { + retired, err := filepath.Glob(filepath.Join(repository.Parquet.BatchesDir(), "*.retired-*")) + if err != nil { + return err + } + if len(retired) != minCompactionInputs { + return fmt.Errorf("retired compaction inputs during publication = %d, want %d", len(retired), minCompactionInputs) + } + return nil + }} + compacted, err := repository.CompactParquet(context.Background(), compactor, 64) if err != nil { t.Fatal(err) } diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 8369dd33..c2a53e01 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -15,7 +15,7 @@ import ( const ( commitQueueDepth = 256 - commitRetryLimit = 3 + commitRetryLimit = 5 maxGroupBatchRows = 50_000 maxCommitWorkers = 4 submissionQueueDepth = 256 @@ -52,7 +52,9 @@ func NewWriter(repository *Repository, batchSize int) *Writer { func (w *Writer) Wait() { <-w.done } // Submit returns after every row in the request belongs to a durably published -// atomic Parquet batch directory. +// atomic Parquet batch directory. Delivery is at least once: callers that retry +// an ambiguous request may create another batch because OTLP has no idempotency +// key that survives across requests. func (w *Writer) Submit(ctx context.Context, batch Batch) error { if batchRows(batch) == 0 { return nil @@ -82,13 +84,12 @@ func (w *Writer) Run(ctx context.Context) error { jobs := make(chan commitJob, commitQueueDepth) workerCtx, cancelWorkers := context.WithCancel(context.Background()) defer cancelWorkers() - fatal := make(chan error, 1) var workers sync.WaitGroup for range min(maxCommitWorkers, max(1, runtime.GOMAXPROCS(0))) { workers.Add(1) go func() { defer workers.Done() - w.commitWorker(workerCtx, jobs, fatal) + w.commitWorker(workerCtx, jobs) }() } @@ -111,31 +112,23 @@ func (w *Writer) Run(ctx context.Context) error { cancelWorkers() <-finished } - select { - case err := <-fatal: - return err - default: - return nil - } + return nil } for { select { case request := <-w.submissions: metrics.UpdateQueueDepth("batch", len(w.submissions)) - if err := w.enqueueSubmissions(request, jobs, fatal); err != nil { + if err := w.enqueueSubmissions(ctx, request, jobs); err != nil { return errors.Join(err, finish(false)) } case <-ctx.Done(): return finish(true) - case err := <-fatal: - cancelWorkers() - return errors.Join(err, finish(false)) } } } -func (w *Writer) enqueueSubmissions(request submission, out chan<- commitJob, fatal <-chan error) error { +func (w *Writer) enqueueSubmissions(ctx context.Context, request submission, out chan<- commitJob) error { requests := []submission{request} for len(requests) < submissionQueueDepth { select { @@ -159,7 +152,7 @@ drained: direct := requests[0] requests = requests[1:] direct.batch.ID = uuid.NewString() - if err := enqueueJob(out, fatal, commitJob{batches: []Batch{direct.batch}, acks: []chan error{direct.ack}}); err != nil { + if err := enqueueJob(ctx, out, commitJob{batches: []Batch{direct.batch}, acks: []chan error{direct.ack}}); err != nil { return err } continue @@ -188,23 +181,23 @@ drained: for i := range group { acks[i] = group[i].ack } - if err := enqueueJob(out, fatal, commitJob{batches: []Batch{batch}, acks: acks}); err != nil { + if err := enqueueJob(ctx, out, commitJob{batches: []Batch{batch}, acks: acks}); err != nil { return err } } return nil } -func enqueueJob(out chan<- commitJob, fatal <-chan error, job commitJob) error { +func enqueueJob(ctx context.Context, out chan<- commitJob, job commitJob) error { select { case out <- job: return nil - case err := <-fatal: - return err + case <-ctx.Done(): + return ctx.Err() } } -func (w *Writer) commitWorker(ctx context.Context, jobs <-chan commitJob, fatal chan<- error) { +func (w *Writer) commitWorker(ctx context.Context, jobs <-chan commitJob) { for { select { case <-ctx.Done(): @@ -217,11 +210,8 @@ func (w *Writer) commitWorker(ctx context.Context, jobs <-chan commitJob, fatal for _, ack := range job.acks { ack <- err } - select { - case fatal <- err: - default: - } - return + slog.Error("telemetry batch commit exhausted retries", "error", err) + continue } for _, batch := range job.batches { metrics.RecordIngest("spans", len(batch.Spans)) @@ -306,5 +296,5 @@ func recordDroppedRows(batch Batch) { } func defaultCommitRetryDelay(attempt int) time.Duration { - return min(100*time.Millisecond*time.Duration(1< Date: Thu, 27 Aug 2026 17:33:06 -0700 Subject: [PATCH 17/31] fix(storage): isolate Parquet maintenance Keep reader exclusion and rollup-cache writes independent, bound retention swaps, and recover durable compaction state before retention can remove its inputs. Authoritative corruption remains fail-closed instead of being silently quarantined. --- internal/metrics/metrics.go | 16 +++ internal/query/duck.go | 46 ++++--- internal/query/duck_test.go | 94 +++++++++++++-- internal/telemetry/parquet.go | 125 +++++++++++++++++--- internal/telemetry/parquet_test.go | 66 +++++++++++ internal/telemetry/store/compaction.go | 16 +-- internal/telemetry/store/repository.go | 18 ++- internal/telemetry/store/repository_test.go | 38 +++++- 8 files changed, 359 insertions(+), 60 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1bdb1c4b..07e1c85a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -188,6 +188,22 @@ var ( Help: "Number of telemetry Parquet files per signal", }, []string{"signal"}) + ParquetPublishWaiters = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "fanout_parquet_publish_waiters", + Help: "Current Parquet publications waiting for active readers", + }) + + ParquetPublishWait = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_parquet_publish_wait_seconds", + Help: "Time spent waiting to publish a new Parquet file set", + Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, + }) + + ParquetPublishTimeouts = promauto.NewCounter(prometheus.CounterOpts{ + Name: "fanout_parquet_publish_timeouts_total", + Help: "Parquet publications abandoned after reader wait timeout", + }) + // HTTP metrics HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "fanout_http_requests_total", diff --git a/internal/query/duck.go b/internal/query/duck.go index 454360d4..905e9695 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -448,20 +448,24 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() var pruneErr error if d.repository != nil { - var parquetErr error - if d.cfg.RetentionDays > 0 { - _, parquetErr = d.repository.PruneParquet(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano()) - } + recoveryErr := d.repository.RecoverParquet(ctx, d) compactStart := time.Now() - compacted, compactErr := d.repository.CompactParquetBacklog(ctx, d, 64) + var parquetErr, compactErr error + compacted := 0 + if recoveryErr == nil { + if d.cfg.RetentionDays > 0 { + _, parquetErr = d.repository.PruneParquetBacklog(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), 64) + } + compacted, compactErr = d.repository.CompactParquetBacklog(ctx, d, 64) + } compactResult := metrics.TelemetryNoop - if compactErr != nil { + if recoveryErr != nil || compactErr != nil { compactResult = metrics.TelemetryError } else if compacted > 0 { compactResult = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) - pruneErr = errors.Join(pruneErr, parquetErr, compactErr) + pruneErr = errors.Join(pruneErr, recoveryErr, parquetErr, compactErr) } cacheErr := func() error { unlock, err := d.writeGate.LockContext(ctx, writegate.WriteMaintenance) @@ -1427,15 +1431,10 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet return d.repository.Trace(ctx, query) } -// MergeParquet executes the query-engine-specific half of compaction. The -// write is serialized with rollup-cache transactions, but only for this one -// merge so a large compaction backlog cannot starve rollups. +// MergeParquet executes the query-engine-specific half of compaction. It reads +// immutable inputs and writes an unpublished staging file, so it does not +// contend with DuckDB rollup-cache writes. func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { - unlock, err := d.writeGate.LockContext(ctx, writegate.WriteMaintenance) - if err != nil { - return err - } - defer unlock() quoted := make([]string, len(inputs)) for i, input := range inputs { quoted[i] = quoteDuckString(input) @@ -1445,22 +1444,21 @@ func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, query += " ORDER BY _trace_hash, start_unix_nano, span_id" } stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, quoteDuckString(output)) - _, err = d.DB.ExecContext(ctx, stmt) + _, err := d.DB.ExecContext(ctx, stmt) return err } // PublishParquet limits reader exclusion to the atomic directory swap. func (d *Duck) PublishParquet(ctx context.Context, publish func() error) error { - writeCtx, cancelWrite := context.WithTimeout(ctx, 2*defaultWriterGrace) - unlockWrite, err := d.writeGate.LockContext(writeCtx, writegate.WriteMaintenance) - cancelWrite() - if err != nil { - return fmt.Errorf("wait for DuckDB write gate: %w", err) - } - defer unlockWrite() publishCtx, cancelPublish := context.WithTimeout(ctx, 2*defaultWriterGrace) defer cancelPublish() - if err := d.parquetMu.LockContext(publishCtx); err != nil { + metrics.ParquetPublishWaiters.Inc() + waitStarted := time.Now() + err := d.parquetMu.LockContext(publishCtx) + metrics.ParquetPublishWaiters.Dec() + metrics.ParquetPublishWait.Observe(time.Since(waitStarted).Seconds()) + if err != nil { + metrics.ParquetPublishTimeouts.Inc() return fmt.Errorf("wait for Parquet readers: %w", err) } defer d.parquetMu.Unlock() diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 07ed0f22..7bb845ef 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "testing" "time" @@ -281,6 +282,7 @@ func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { func TestPublishParquetHonorsContext(t *testing.T) { d := &Duck{} d.parquetMu.RLock() + timeoutsBefore := testutil.ToFloat64(metrics.ParquetPublishTimeouts) ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() called := false @@ -298,6 +300,39 @@ func TestPublishParquetHonorsContext(t *testing.T) { if got := d.parquetMu.WaitingWriters(); got != 0 { t.Fatalf("canceled publication remained queued: %d", got) } + if got := testutil.ToFloat64(metrics.ParquetPublishTimeouts); got != timeoutsBefore+1 { + t.Fatalf("publication timeouts = %v, want %v", got, timeoutsBefore+1) + } + if got := testutil.ToFloat64(metrics.ParquetPublishWaiters); got != 0 { + t.Fatalf("publication waiters after timeout = %v, want 0", got) + } +} + +func TestParquetWorkDoesNotWaitForDuckDBWriteGate(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("COPY").WillReturnResult(sqlmock.NewResult(0, 1)) + d := &Duck{DB: db} + release := d.writeGate.Lock(writegate.WriteRollupService) + defer release() + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := d.MergeParquet(ctx, "logs", []string{"/tmp/input.parquet"}, "/tmp/output.parquet"); err != nil { + t.Fatalf("merge waited for unrelated DuckDB write gate: %v", err) + } + called := false + if err := d.PublishParquet(ctx, func() error { called = true; return nil }); err != nil { + t.Fatalf("publication waited for unrelated DuckDB write gate: %v", err) + } + if !called { + t.Fatal("publication callback did not run") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } } func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { @@ -382,7 +417,7 @@ func TestIndexedTraceReadHonorsParquetGateContext(t *testing.T) { } } -func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { +func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatal(err) @@ -393,6 +428,9 @@ func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { t.Fatal(err) } defer repository.Close() + if err := repository.Commit(telemetrystore.Batch{ID: "expired", Spans: []telemetry.Span{{TraceID: "trace", IngestedAt: 1}}}); err != nil { + t.Fatal(err) + } mock.ExpectExec("DELETE FROM service_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DELETE FROM endpoint_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DELETE FROM edge_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) @@ -402,16 +440,14 @@ func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { release := d.writeGate.Lock(writegate.WriteRollupService) done := make(chan error, 1) go func() { done <- d.runRepositoryMaintenance(context.Background()) }() - select { - case err := <-done: - release() - t.Fatalf("maintenance bypassed write gate: %v", err) - case <-time.After(25 * time.Millisecond): + deadline := time.Now().Add(time.Second) + for d.parquetMu.WaitingWriters() == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) } - if waiting := d.parquetMu.WaitingWriters(); waiting != 0 { + if waiting := d.parquetMu.WaitingWriters(); waiting != 1 { d.parquetMu.RUnlock() release() - t.Fatalf("maintenance queued Parquet publication before owning DuckDB write gate: %d", waiting) + t.Fatalf("publication waiters = %d, want 1 while DuckDB write gate is independently held", waiting) } d.parquetMu.RUnlock() release() @@ -423,6 +459,48 @@ func TestRepositoryMaintenanceSerializesDuckDBWrites(t *testing.T) { } } +type failingPublishCompactor struct{ *Duck } + +func (f failingPublishCompactor) PublishParquet(context.Context, func() error) error { + return errors.New("injected publication failure") +} + +func TestMaintenanceRecoversCompactionBeforeRetention(t *testing.T) { + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + db := openTestDuck(t) + defer db.Close() + for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { + if _, err := db.Exec("CREATE TABLE " + table + " (bucket TIMESTAMP)"); err != nil { + t.Fatal(err) + } + } + d := &Duck{DB: db, repository: repository, cfg: config.Config{RetentionDays: 1}} + for i := range 8 { + batch := telemetrystore.Batch{ + ID: fmt.Sprintf("expired-%d", i), + Spans: []telemetry.Span{{TraceID: "trace", SpanID: fmt.Sprintf("span-%d", i), StartUnixNanos: 1, IngestedAt: 1}}, + Logs: []telemetry.Log{{Body: "old", IngestedAt: 1}}, + Metrics: []telemetry.Metric{{Name: "old", IngestedAt: 1}}, + } + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + if _, err := repository.CompactParquet(context.Background(), failingPublishCompactor{d}, 64); err == nil { + t.Fatal("compaction unexpectedly published") + } + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatal(err) + } + if got := repository.RowCount(); got != 0 { + t.Fatalf("retention resurrected pending compaction rows: %d", got) + } +} + func TestDuckDSN(t *testing.T) { tests := []struct { name string diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index f11f2ae9..45687014 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -28,6 +28,8 @@ const ( maxTraceQueryResults = 500 ) +var syncPublishedDirectory = syncDirectory + type BatchMetadata struct { Version uint32 `json:"version"` ID string `json:"id"` @@ -279,6 +281,48 @@ func (p *ParquetStore) CleanupRetired() error { return cleanupErr } +// RestoreRetiredInputs rolls back a compaction whose durable output vanished +// before publication. Active inputs are left untouched; already-retired inputs +// are restored and registered again. +func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string) error { + if err := validateBatchID(replacementID); err != nil { + return err + } + p.publishMu.Lock() + defer p.publishMu.Unlock() + restored := false + for _, id := range inputs { + if err := validateBatchID(id); err != nil { + return err + } + active := p.BatchPath(id) + if _, err := os.Stat(active); errors.Is(err, os.ErrNotExist) { + retired := filepath.Join(p.batchesDir, id+".retired-"+replacementID) + if _, retiredErr := os.Stat(retired); retiredErr != nil { + if errors.Is(retiredErr, os.ErrNotExist) { + return fmt.Errorf("input %s is missing", id) + } + return retiredErr + } + if err := os.Rename(retired, active); err != nil { + return err + } + restored = true + } else if err != nil { + return err + } + if !p.hasBatch(id) { + if err := p.registerBatch(active); err != nil { + return err + } + } + } + if restored { + return syncDirectory(p.batchesDir) + } + return nil +} + // Trace reads only ranges selected by the persistent hash index. Scope filters // and the limit are applied while decoding so a pathological trace cannot grow // request memory without bound. @@ -412,16 +456,41 @@ func indexedSpanEarlier(left, right IndexedSpan) bool { // PruneBefore hides complete batches while readers are pinned, then deletes // the retired directories after publication. -func (p *ParquetStore) PruneBefore(cutoff int64, publish func(func() error) error) (int, error) { +func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(func() error) error) (int, error) { + if maxBatches <= 0 { + return 0, nil + } + p.publishMu.Lock() + locked := true + defer func() { + if locked { + p.publishMu.Unlock() + } + }() + p.mu.RLock() + candidates := make([]string, 0, maxBatches) + for id, batch := range p.batches { + if batch.metadata.MaxIngestedNanos > 0 && batch.metadata.MaxIngestedNanos < cutoff { + candidates = append(candidates, id) + if len(candidates) == maxBatches { + break + } + } + } + p.mu.RUnlock() + if len(candidates) == 0 { + p.publishMu.Unlock() + locked = false + return 0, nil + } var retired []string var pruneErr error err := publish(func() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() p.mu.Lock() defer p.mu.Unlock() - for id, batch := range p.batches { - if batch.metadata.MaxIngestedNanos <= 0 || batch.metadata.MaxIngestedNanos >= cutoff { + for _, id := range candidates { + batch, exists := p.batches[id] + if !exists { continue } path := filepath.Join(p.batchesDir, id+".retired") @@ -437,6 +506,8 @@ func (p *ParquetStore) PruneBefore(cutoff int64, publish func(func() error) erro } return nil }) + p.publishMu.Unlock() + locked = false pruneErr = errors.Join(pruneErr, err) for _, path := range retired { pruneErr = errors.Join(pruneErr, os.RemoveAll(path)) @@ -560,19 +631,39 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, if replacement.metadata.Spans > 0 { replacement.traces.path = filepath.Join(final, "trace.fidx") } + p.publishMu.Lock() + locked := true + defer func() { + if locked { + p.publishMu.Unlock() + } + }() retired := make([][2]string, 0, len(inputs)) err = publish(func() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() p.mu.Lock() defer p.mu.Unlock() + installReplacement := func() { + for _, id := range inputs { + delete(p.batches, id) + } + p.batches[metadata.ID] = replacement + } rollback := func() error { var rollbackErr error if source == final { - if _, statErr := os.Stat(stage); errors.Is(statErr, os.ErrNotExist) { - rollbackErr = errors.Join(rollbackErr, os.Rename(final, stage)) + if err := os.MkdirAll(filepath.Dir(stage), 0o755); err != nil { + installReplacement() + return err + } + if err := os.Rename(final, stage); err != nil { + // Restoring inputs while the output remains published would + // double-count every compacted row. Keep the output-only view. + installReplacement() + return err } + source = stage + delete(p.batches, metadata.ID) } for i := len(retired) - 1; i >= 0; i-- { rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) @@ -601,15 +692,17 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, } source = final } - if err := syncDirectory(p.batchesDir); err != nil { - return errors.Join(err, rollback()) - } - for _, id := range inputs { - delete(p.batches, id) + if err := syncPublishedDirectory(p.batchesDir); err != nil { + // The namespace already contains only the replacement. Keep the + // in-memory view consistent and let marker recovery retry the fsync. + installReplacement() + return err } - p.batches[metadata.ID] = replacement + installReplacement() return nil }) + p.publishMu.Unlock() + locked = false if err != nil { return err } @@ -659,7 +752,7 @@ func (p *ParquetStore) loadBatches() error { continue } if err := p.registerBatch(filepath.Join(p.batchesDir, entry.Name())); err != nil { - return err + return fmt.Errorf("load Parquet batch %s: %w", entry.Name(), err) } } return nil diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index cc1beec0..33b66961 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "reflect" "testing" + "time" "github.com/parquet-go/parquet-go" ) @@ -221,6 +222,71 @@ func TestPublishReplacementValidatesBeforePublication(t *testing.T) { } } +func TestPruneOwnsStoragePublicationBeforeExcludingReaders(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + t.Fatal(err) + } + store.publishMu.Lock() + entered := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := store.PruneBefore(2, 1, func(prune func() error) error { + close(entered) + return prune() + }) + done <- err + }() + select { + case <-entered: + store.publishMu.Unlock() + t.Fatal("reader exclusion began before storage publication ownership") + case <-time.After(20 * time.Millisecond): + } + store.publishMu.Unlock() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestPublishReplacementKeepsOutputOnlyAfterSyncFailure(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} + if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + t.Fatal(err) + } + output := BatchMetadata{ID: "output"} + if err := store.CommitBatch(output, span, nil, nil); err != nil { + t.Fatal(err) + } + originalSync := syncPublishedDirectory + syncPublishedDirectory = func(string) error { return errors.New("injected directory sync failure") } + t.Cleanup(func() { syncPublishedDirectory = originalSync }) + + err = store.PublishReplacement(filepath.Join(t.TempDir(), "missing-stage"), output, []string{"input"}, func(publish func() error) error { + return publish() + }) + if err == nil { + t.Fatal("replacement succeeded despite injected sync failure") + } + if _, err := os.Stat(store.BatchPath("input")); !os.IsNotExist(err) { + t.Fatalf("input was restored beside live output: %v", err) + } + if _, err := os.Stat(store.BatchPath("output")); err != nil { + t.Fatalf("replacement output missing: %v", err) + } + metadata := store.BatchMetadata() + if len(metadata) != 1 || metadata[0].ID != "output" { + t.Fatalf("live batches after sync failure = %#v, want output only", metadata) + } +} + func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 139b5c92..1542a3b2 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -178,6 +178,12 @@ func (r *Repository) CompactParquetBacklog(ctx context.Context, compactor Parque type parquetPublishFunc func(context.Context, func() error) error +func (r *Repository) RecoverParquet(ctx context.Context, publisher ParquetPublisher) error { + r.compactionMu.Lock() + defer r.compactionMu.Unlock() + return r.recoverCompaction(ctx, publisher.PublishParquet) +} + func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPublishFunc) error { data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) if errors.Is(err, os.ErrNotExist) { @@ -199,14 +205,8 @@ func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPubli return err } if !stageExists && !finalExists { - for _, id := range marker.Inputs { - exists, statErr := pathExists(r.Parquet.BatchPath(id)) - if statErr != nil { - return statErr - } - if !exists { - return fmt.Errorf("compaction %s has no output and input %s is missing", marker.Output.ID, id) - } + if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID); err != nil { + return fmt.Errorf("restore compaction %s inputs: %w", marker.Output.ID, err) } if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { return err diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 832e5273..02c966b1 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -82,14 +82,28 @@ func (r *Repository) Trace(ctx context.Context, query telemetry.TraceQuery) ([]t func (r *Repository) RowCount() uint64 { return r.Parquet.RowCount() } -func (r *Repository) PruneParquet(ctx context.Context, publisher ParquetPublisher, cutoff int64) (int, error) { +func (r *Repository) PruneParquet(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches int) (int, error) { r.compactionMu.Lock() defer r.compactionMu.Unlock() - return r.Parquet.PruneBefore(cutoff, func(prune func() error) error { + return r.Parquet.PruneBefore(cutoff, maxBatches, func(prune func() error) error { return publisher.PublishParquet(ctx, prune) }) } +func (r *Repository) PruneParquetBacklog(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches int) (int, error) { + if maxBatches <= 0 { + return 0, nil + } + total := 0 + for { + count, err := r.PruneParquet(ctx, publisher, cutoff, maxBatches) + total += count + if err != nil || count < maxBatches { + return total, err + } + } +} + func validateBatch(batch Batch) error { if batch.ID == "" || strings.ContainsAny(batch.ID, `/\\`) { return errors.New("telemetry batch requires a safe ID") diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 814957d1..d79038fa 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -211,7 +211,7 @@ func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { } return nil }} - removed, err := repository.PruneParquet(context.Background(), publisher, 500) + removed, err := repository.PruneParquet(context.Background(), publisher, 500, 64) if err != nil || removed != 1 { t.Fatalf("prune = %d, %v", removed, err) } @@ -248,6 +248,10 @@ func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { t.Fatal(err) } + retired := filepath.Join(repository.Parquet.BatchesDir(), batch.ID+".retired-"+marker.Output.ID) + if err := os.Rename(repository.Parquet.BatchPath(batch.ID), retired); err != nil { + t.Fatal(err) + } if err := repository.Close(); err != nil { t.Fatal(err) } @@ -265,6 +269,36 @@ func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { } } +func TestRepositoryPrunesBacklogInBoundedPublications(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range 5 { + batch := testBatch() + batch.ID = fmt.Sprintf("expired-%d", i) + batch.Spans[0].IngestedAt = 1 + batch.Logs[0].IngestedAt = 1 + batch.Metrics[0].IngestedAt = 1 + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + publications := 0 + publisher := &testParquetCompactor{afterSwap: func() error { + publications++ + return nil + }} + removed, err := repository.PruneParquetBacklog(context.Background(), publisher, 2, 2) + if err != nil { + t.Fatal(err) + } + if removed != 5 || publications != 3 { + t.Fatalf("removed=%d publications=%d, want 5 across 3 bounded swaps", removed, publications) + } +} + func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { @@ -282,7 +316,7 @@ func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { if err := repository.Commit(batch); err != nil { t.Fatal(err) } - removed, err := repository.PruneParquet(context.Background(), &testParquetCompactor{}, 500) + removed, err := repository.PruneParquet(context.Background(), &testParquetCompactor{}, 500, 64) if err != nil || removed != 1 { t.Fatalf("prune future-dated events = %d, %v; want one batch expired by ingest time", removed, err) } From 895c238a7e592b041f3fcd364d8eb0ab03106d7f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 18:12:17 -0700 Subject: [PATCH 18/31] fix(storage): bound Parquet publication Acquire reader exclusion before the storage publication lock and cap each maintenance pass so rollups cannot stall ingest. Keep recovery namespace swaps reader-atomic and escalate sustained maintenance failures. --- internal/api/health.go | 12 +- internal/api/health_test.go | 18 +- internal/query/duck.go | 61 ++++-- internal/query/duck_test.go | 70 ++++++- internal/query/parquet_gate.go | 13 +- internal/query/parquet_gate_test.go | 19 +- internal/telemetry/parquet.go | 218 +++++++++++++------- internal/telemetry/parquet_test.go | 113 +++++++++- internal/telemetry/store/compaction.go | 12 +- internal/telemetry/store/repository.go | 15 +- internal/telemetry/store/repository_test.go | 79 ++++++- 11 files changed, 486 insertions(+), 144 deletions(-) diff --git a/internal/api/health.go b/internal/api/health.go index 180ffddd..bb3850bf 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -239,8 +239,8 @@ func (h *HealthHandler) checkMaintenance() CheckResult { } } - lastOK, lastAt, lastErr := h.duck.MaintenanceHealth() - return maintenanceResult(lastOK, lastAt, lastErr, h.started, + lastOK, lastAt, failures, lastErr := h.duck.MaintenanceHealth() + return maintenanceResult(lastOK, lastAt, lastErr, failures, h.started, h.cfg.RollupInterval, h.cfg.MaintenanceInterval, time.Now()) @@ -249,16 +249,20 @@ func (h *HealthHandler) checkMaintenance() CheckResult { // maintenanceResult classifies the maintenance loop's health. now/started are // injected so every branch — a failing pass, never-ran-past-grace, and a loop // that stalled after a clean pass — is unit-testable without a running Duck. -func maintenanceResult(lastOK, lastAt time.Time, lastErr error, started time.Time, rollupEvery, maintEvery time.Duration, now time.Time) CheckResult { +func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailures int, started time.Time, rollupEvery, maintEvery time.Duration, now time.Time) CheckResult { res := CheckResult{Status: "ok"} if !lastAt.IsZero() { res.UpdatedAt = lastAt.UTC().Format(time.RFC3339) } if lastErr != nil { res.Status = "degraded" + if consecutiveFailures >= 3 { + res.Status = "unhealthy" + } res.Error = lastErr.Error() + res.Detail = fmt.Sprintf("%d consecutive failed passes", consecutiveFailures) if !lastOK.IsZero() { - res.Detail = "last clean pass: " + lastOK.UTC().Format(time.RFC3339) + res.Detail += "; last clean pass: " + lastOK.UTC().Format(time.RFC3339) } return res } diff --git a/internal/api/health_test.go b/internal/api/health_test.go index f8ec87db..b999d917 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -294,23 +294,25 @@ func TestMaintenanceResult(t *testing.T) { name string lastOK, lastAt time.Time lastErr error + failures int started time.Time maintEvery time.Duration wantStatus string }{ - {"clean recent pass", now.Add(-10 * time.Minute), now.Add(-10 * time.Minute), nil, now.Add(-2 * time.Hour), time.Hour, "ok"}, - {"failing pass", now.Add(-3 * time.Hour), now.Add(-time.Hour), errors.New("boom"), now.Add(-4 * time.Hour), time.Hour, "degraded"}, - {"never ran, past grace", time.Time{}, time.Time{}, nil, now.Add(-10 * time.Minute), time.Hour, "degraded"}, - {"never ran, within grace", time.Time{}, time.Time{}, nil, now.Add(-time.Minute), time.Hour, "ok"}, - {"stalled after clean pass", now.Add(-5 * time.Hour), now.Add(-5 * time.Hour), nil, now.Add(-6 * time.Hour), time.Hour, "degraded"}, + {"clean recent pass", now.Add(-10 * time.Minute), now.Add(-10 * time.Minute), nil, 0, now.Add(-2 * time.Hour), time.Hour, "ok"}, + {"failing pass", now.Add(-3 * time.Hour), now.Add(-time.Hour), errors.New("boom"), 1, now.Add(-4 * time.Hour), time.Hour, "degraded"}, + {"repeated failure", now.Add(-4 * time.Hour), now.Add(-time.Hour), errors.New("boom"), 3, now.Add(-5 * time.Hour), time.Hour, "unhealthy"}, + {"never ran, past grace", time.Time{}, time.Time{}, nil, 0, now.Add(-10 * time.Minute), time.Hour, "degraded"}, + {"never ran, within grace", time.Time{}, time.Time{}, nil, 0, now.Add(-time.Minute), time.Hour, "ok"}, + {"stalled after clean pass", now.Add(-5 * time.Hour), now.Add(-5 * time.Hour), nil, 0, now.Add(-6 * time.Hour), time.Hour, "degraded"}, // maintEvery=0 must still detect staleness: the loop floors 0→1h and keeps // running, so the check mirrors that (2h stale > 1h floored threshold). - {"stalled, interval unset (floored)", now.Add(-3 * time.Hour), now.Add(-3 * time.Hour), nil, now.Add(-4 * time.Hour), 0, "degraded"}, - {"recent pass, interval unset", now.Add(-10 * time.Minute), now.Add(-10 * time.Minute), nil, now.Add(-4 * time.Hour), 0, "ok"}, + {"stalled, interval unset (floored)", now.Add(-3 * time.Hour), now.Add(-3 * time.Hour), nil, 0, now.Add(-4 * time.Hour), 0, "degraded"}, + {"recent pass, interval unset", now.Add(-10 * time.Minute), now.Add(-10 * time.Minute), nil, 0, now.Add(-4 * time.Hour), 0, "ok"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - res := maintenanceResult(c.lastOK, c.lastAt, c.lastErr, c.started, rollupEvery, c.maintEvery, now) + res := maintenanceResult(c.lastOK, c.lastAt, c.lastErr, c.failures, c.started, rollupEvery, c.maintEvery, now) if res.Status != c.wantStatus { t.Errorf("status = %q, want %q (detail=%q err=%q)", res.Status, c.wantStatus, res.Detail, res.Error) } diff --git a/internal/query/duck.go b/internal/query/duck.go index 905e9695..752e9f6f 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -38,10 +38,11 @@ type Duck struct { parquetMu parquetReadGate // maintHealthMu guards the maintenance health fields below, which the // readiness probe reads while the maintenance pass writes them. - maintHealthMu sync.Mutex - lastMaintenanceOK time.Time - lastMaintenanceAt time.Time - lastMaintenanceErr error + maintHealthMu sync.Mutex + lastMaintenanceOK time.Time + lastMaintenanceAt time.Time + lastMaintenanceErr error + maintenanceFailures int } // MaintenanceHealth reports the maintenance loop's own health: when it last @@ -52,23 +53,25 @@ type Duck struct { // operator instead of only logging. lastAt distinguishes "failing right now" // from a stale error awaiting the hourly retry; a zero lastAt means no pass // has executed since the process started. -func (d *Duck) MaintenanceHealth() (lastOK, lastAt time.Time, lastErr error) { +func (d *Duck) MaintenanceHealth() (lastOK, lastAt time.Time, consecutiveFailures int, lastErr error) { d.maintHealthMu.Lock() defer d.maintHealthMu.Unlock() - return d.lastMaintenanceOK, d.lastMaintenanceAt, d.lastMaintenanceErr + return d.lastMaintenanceOK, d.lastMaintenanceAt, d.maintenanceFailures, d.lastMaintenanceErr } const ( - serviceRollupStateKey = "service_rollup_v2" - serviceRollupRawMaxKey = "service_rollup_v2_rawmax" - edgeRollupStateKey = "edge_rollup_v2" - edgeRollupRawMaxKey = "edge_rollup_v2_rawmax" - EndpointRollupStateKey = "endpoint_rollup_v1" - endpointRollupRawMaxKey = "endpoint_rollup_v1_rawmax" - endpointBackfillStateKey = "endpoint_rollup_v1_backfill_started" - EndpointReadyStateKey = "endpoint_rollup_v1_ready" - EndpointDisabledStateKey = "endpoint_rollup_v1_disabled" - defaultDuckDBPoolSize = 1 + serviceRollupStateKey = "service_rollup_v2" + serviceRollupRawMaxKey = "service_rollup_v2_rawmax" + edgeRollupStateKey = "edge_rollup_v2" + edgeRollupRawMaxKey = "edge_rollup_v2_rawmax" + EndpointRollupStateKey = "endpoint_rollup_v1" + endpointRollupRawMaxKey = "endpoint_rollup_v1_rawmax" + endpointBackfillStateKey = "endpoint_rollup_v1_backfill_started" + EndpointReadyStateKey = "endpoint_rollup_v1_ready" + EndpointDisabledStateKey = "endpoint_rollup_v1_disabled" + defaultDuckDBPoolSize = 1 + parquetMaintenanceBatchLimit = 64 + parquetMaintenancePublishLimit = 4 ) // rollupPublicationSafetyLag covers the maximum public SQL hold, publication @@ -448,15 +451,18 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() var pruneErr error if d.repository != nil { - recoveryErr := d.repository.RecoverParquet(ctx, d) + storageCtx, cancelStorage := context.WithTimeout(ctx, 2*defaultWriterGrace) + defer cancelStorage() + recoveryErr := d.repository.RecoverParquet(storageCtx, d) compactStart := time.Now() - var parquetErr, compactErr error + var cleanupErr, parquetErr, compactErr error compacted := 0 if recoveryErr == nil { + cleanupErr = d.repository.CleanupParquet() if d.cfg.RetentionDays > 0 { - _, parquetErr = d.repository.PruneParquetBacklog(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), 64) + _, parquetErr = d.repository.PruneParquetPass(storageCtx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePublishLimit) } - compacted, compactErr = d.repository.CompactParquetBacklog(ctx, d, 64) + compacted, compactErr = d.repository.CompactParquetPass(storageCtx, d, parquetMaintenanceBatchLimit, parquetMaintenancePublishLimit) } compactResult := metrics.TelemetryNoop if recoveryErr != nil || compactErr != nil { @@ -465,7 +471,7 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { compactResult = metrics.TelemetrySuccess } metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds()) - pruneErr = errors.Join(pruneErr, recoveryErr, parquetErr, compactErr) + pruneErr = errors.Join(pruneErr, recoveryErr, cleanupErr, parquetErr, compactErr) } cacheErr := func() error { unlock, err := d.writeGate.LockContext(ctx, writegate.WriteMaintenance) @@ -495,6 +501,9 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { d.lastMaintenanceAt = finished if err == nil { d.lastMaintenanceOK = finished + d.maintenanceFailures = 0 + } else { + d.maintenanceFailures++ } d.lastMaintenanceErr = err d.maintHealthMu.Unlock() @@ -524,6 +533,8 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { // consistent and deadlock-free. unlock := d.writeGate.Lock(writegate.WriteRollupService) defer unlock() + ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) + defer cancel() if err := d.lockParquetRead(ctx); err != nil { return 0, err } @@ -645,6 +656,8 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { }() unlock := d.writeGate.Lock(writegate.WriteRollupEndpoint) defer unlock() + ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) + defer cancel() if err := d.lockParquetRead(ctx); err != nil { return 0, err } @@ -768,6 +781,8 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { }() unlock := d.writeGate.Lock(writegate.WriteRollupEdge) defer unlock() + ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) + defer cancel() if err := d.lockParquetRead(ctx); err != nil { return 0, err } @@ -890,7 +905,9 @@ WHERE ingested_unix_nano > ? // backlog in a single statement. Unbounded catch-up is what took prod down on // 2026-06-13 (UTC): the edge rollup's first pass covered 12 days of spans, // spilled 375 GiB to temp, filled the disk, and never committed. -const rollupChunkNanos = int64(time.Hour) +// Ten-minute chunks also keep the rebuildable read lease short enough for +// retention and compaction to acquire the Parquet publication gate. +const rollupChunkNanos = int64(10 * time.Minute) // edgeStartChunkNanos bounds how wide a start_time range one edge-rollup // DELETE+INSERT processes, so the call_edges self-join over a wide backlog diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 7bb845ef..e1b742d0 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -5,6 +5,8 @@ import ( "database/sql" "errors" "fmt" + "os" + "path/filepath" "testing" "time" @@ -297,7 +299,7 @@ func TestPublishParquetHonorsContext(t *testing.T) { if called { t.Fatal("publication ran after its context expired") } - if got := d.parquetMu.WaitingWriters(); got != 0 { + if got := waitingParquetWriters(&d.parquetMu); got != 0 { t.Fatalf("canceled publication remained queued: %d", got) } if got := testutil.ToFloat64(metrics.ParquetPublishTimeouts); got != timeoutsBefore+1 { @@ -355,7 +357,7 @@ func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { close(writerDone) }() deadline := time.Now().Add(time.Second) - for d.parquetMu.WaitingWriters() == 0 { + for waitingParquetWriters(&d.parquetMu) == 0 { if time.Now().After(deadline) { d.parquetMu.RUnlock() t.Fatal("maintenance writer did not begin waiting") @@ -441,10 +443,10 @@ func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { done := make(chan error, 1) go func() { done <- d.runRepositoryMaintenance(context.Background()) }() deadline := time.Now().Add(time.Second) - for d.parquetMu.WaitingWriters() == 0 && time.Now().Before(deadline) { + for waitingParquetWriters(&d.parquetMu) == 0 && time.Now().Before(deadline) { time.Sleep(time.Millisecond) } - if waiting := d.parquetMu.WaitingWriters(); waiting != 1 { + if waiting := waitingParquetWriters(&d.parquetMu); waiting != 1 { d.parquetMu.RUnlock() release() t.Fatalf("publication waiters = %d, want 1 while DuckDB write gate is independently held", waiting) @@ -459,6 +461,66 @@ func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { } } +func TestMaintenanceRetriesRetiredDirectoryCleanup(t *testing.T) { + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + retired := filepath.Join(repository.Parquet.BatchesDir(), "stale.retired-old") + if err := os.Mkdir(retired, 0o755); err != nil { + t.Fatal(err) + } + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db, repository: repository} + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(retired); !os.IsNotExist(err) { + t.Fatalf("retired directory remains after maintenance: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + for range 3 { + mock.ExpectExec("CHECKPOINT").WillReturnError(errors.New("injected checkpoint failure")) + } + mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) + d := &Duck{DB: db} + for range 3 { + if err := d.runRepositoryMaintenance(context.Background()); err == nil { + t.Fatal("maintenance succeeded despite checkpoint failure") + } + } + _, _, failures, _ := d.MaintenanceHealth() + if failures != 3 { + t.Fatalf("consecutive failures = %d, want 3", failures) + } + if err := d.runRepositoryMaintenance(context.Background()); err != nil { + t.Fatal(err) + } + _, _, failures, _ = d.MaintenanceHealth() + if failures != 0 { + t.Fatalf("consecutive failures after recovery = %d, want 0", failures) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + type failingPublishCompactor struct{ *Duck } func (f failingPublishCompactor) PublishParquet(context.Context, func() error) error { diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 4db7b304..2dff745b 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -12,6 +12,11 @@ import ( // behind maintenance, short enough that retention and compaction always run. const defaultWriterGrace = 30 * time.Second +// rollupReaderLease leaves publication enough headroom after the admission +// grace expires. Rollups are rebuildable cache work and may be canceled; user +// queries retain their caller-provided deadlines. +const rollupReaderLease = defaultWriterGrace / 2 + // ErrParquetReadWait distinguishes publication contention from a query error. var ErrParquetReadWait = errors.New("wait for Parquet publication") @@ -179,11 +184,3 @@ func (g *parquetReadGate) Unlock() { g.notifyLocked() g.mu.Unlock() } - -// WaitingWriters reports how many publishers are queued behind active readers. -func (g *parquetReadGate) WaitingWriters() int { - g.init() - g.mu.Lock() - defer g.mu.Unlock() - return len(g.waiting) -} diff --git a/internal/query/parquet_gate_test.go b/internal/query/parquet_gate_test.go index 0ec9211e..0ec51f13 100644 --- a/internal/query/parquet_gate_test.go +++ b/internal/query/parquet_gate_test.go @@ -14,6 +14,13 @@ type fakeClock struct { now time.Time } +func waitingParquetWriters(g *parquetReadGate) int { + g.init() + g.mu.Lock() + defer g.mu.Unlock() + return len(g.waiting) +} + func TestParquetGateRemovesCanceledPublisher(t *testing.T) { var gate parquetReadGate gate.RLock() @@ -22,7 +29,7 @@ func TestParquetGateRemovesCanceledPublisher(t *testing.T) { if err := gate.LockContext(ctx); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("LockContext error = %v, want deadline exceeded", err) } - if got := gate.WaitingWriters(); got != 0 { + if got := waitingParquetWriters(&gate); got != 0 { t.Fatalf("canceled publisher remained queued: %d", got) } gate.RUnlock() @@ -47,7 +54,7 @@ func (c *fakeClock) Advance(d time.Duration) { func waitForQueuedWriter(t *testing.T, gate *parquetReadGate) { t.Helper() deadline := time.Now().Add(time.Second) - for gate.WaitingWriters() == 0 { + for waitingParquetWriters(gate) == 0 { if time.Now().After(deadline) { t.Fatal("publisher never queued") } @@ -58,9 +65,9 @@ func waitForQueuedWriter(t *testing.T, gate *parquetReadGate) { func waitForQueuedWriters(t *testing.T, gate *parquetReadGate, count int) { t.Helper() deadline := time.Now().Add(time.Second) - for gate.WaitingWriters() < count { + for waitingParquetWriters(gate) < count { if time.Now().After(deadline) { - t.Fatalf("publishers queued = %d, want %d", gate.WaitingWriters(), count) + t.Fatalf("publishers queued = %d, want %d", waitingParquetWriters(gate), count) } time.Sleep(time.Millisecond) } @@ -149,9 +156,9 @@ func TestParquetGateDistinguishesWritersQueuedAtSameInstant(t *testing.T) { } } deadline := time.Now().Add(time.Second) - for gate.WaitingWriters() != 0 { + for waitingParquetWriters(gate) != 0 { if time.Now().After(deadline) { - t.Fatalf("stale publisher remained queued: %d", gate.WaitingWriters()) + t.Fatalf("stale publisher remained queued: %d", waitingParquetWriters(gate)) } time.Sleep(time.Millisecond) } diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 45687014..7382f965 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -254,10 +254,6 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] // publication. It is called only after compaction recovery has consumed its // durable marker, so no rollback can still need these directories. func (p *ParquetStore) CleanupRetired() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() - p.mu.Lock() - defer p.mu.Unlock() entries, err := os.ReadDir(p.batchesDir) if err != nil { return err @@ -281,46 +277,85 @@ func (p *ParquetStore) CleanupRetired() error { return cleanupErr } -// RestoreRetiredInputs rolls back a compaction whose durable output vanished -// before publication. Active inputs are left untouched; already-retired inputs -// are restored and registered again. -func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string) error { +// RestoreRetiredInputs rolls back a compaction whose durable output vanished. +// The complete namespace change is hidden from readers by publish. +func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func() error) error) error { if err := validateBatchID(replacementID); err != nil { return err } - p.publishMu.Lock() - defer p.publishMu.Unlock() - restored := false for _, id := range inputs { if err := validateBatchID(id); err != nil { return err } + } + type restoredInput struct { + id string + active string + retired string + batch *storedBatch + move bool + } + prepared := make([]restoredInput, 0, len(inputs)) + for _, id := range inputs { active := p.BatchPath(id) + path := active + move := false if _, err := os.Stat(active); errors.Is(err, os.ErrNotExist) { - retired := filepath.Join(p.batchesDir, id+".retired-"+replacementID) - if _, retiredErr := os.Stat(retired); retiredErr != nil { - if errors.Is(retiredErr, os.ErrNotExist) { - return fmt.Errorf("input %s is missing", id) - } - return retiredErr - } - if err := os.Rename(retired, active); err != nil { - return err - } - restored = true + path = filepath.Join(p.batchesDir, id+".retired-"+replacementID) + move = true } else if err != nil { return err } - if !p.hasBatch(id) { - if err := p.registerBatch(active); err != nil { - return err + batch, err := loadStoredBatch(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("input %s is missing", id) + } + return fmt.Errorf("load input %s: %w", id, err) + } + batch.dir = active + if batch.metadata.Spans > 0 { + batch.traces.path = filepath.Join(active, "trace.fidx") + } + prepared = append(prepared, restoredInput{id: id, active: active, retired: path, batch: batch, move: move}) + } + return publish(func() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() + installActive := func() { + p.mu.Lock() + defer p.mu.Unlock() + for _, input := range prepared { + if _, err := os.Stat(input.active); err == nil { + p.batches[input.id] = input.batch + } else { + delete(p.batches, input.id) + } } } - } - if restored { - return syncDirectory(p.batchesDir) - } - return nil + + moved := make([]restoredInput, 0, len(prepared)) + for _, input := range prepared { + if !input.move { + continue + } + if err := os.Rename(input.retired, input.active); err != nil { + var rollbackErr error + for i := len(moved) - 1; i >= 0; i-- { + rollbackErr = errors.Join(rollbackErr, os.Rename(moved[i].active, moved[i].retired)) + } + installActive() + return errors.Join(err, rollbackErr, syncDirectory(p.batchesDir)) + } + moved = append(moved, input) + } + var syncErr error + if len(moved) > 0 { + syncErr = syncDirectory(p.batchesDir) + } + installActive() + return syncErr + }) } // Trace reads only ranges selected by the persistent hash index. Scope filters @@ -460,45 +495,48 @@ func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(fu if maxBatches <= 0 { return 0, nil } - p.publishMu.Lock() - locked := true - defer func() { - if locked { - p.publishMu.Unlock() - } - }() + type candidate struct { + id string + max int64 + } p.mu.RLock() - candidates := make([]string, 0, maxBatches) + candidates := make([]candidate, 0, len(p.batches)) for id, batch := range p.batches { if batch.metadata.MaxIngestedNanos > 0 && batch.metadata.MaxIngestedNanos < cutoff { - candidates = append(candidates, id) - if len(candidates) == maxBatches { - break - } + candidates = append(candidates, candidate{id: id, max: batch.metadata.MaxIngestedNanos}) } } p.mu.RUnlock() + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].max != candidates[j].max { + return candidates[i].max < candidates[j].max + } + return candidates[i].id < candidates[j].id + }) + if len(candidates) > maxBatches { + candidates = candidates[:maxBatches] + } if len(candidates) == 0 { - p.publishMu.Unlock() - locked = false return 0, nil } var retired []string var pruneErr error err := publish(func() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() p.mu.Lock() defer p.mu.Unlock() - for _, id := range candidates { - batch, exists := p.batches[id] + for _, candidate := range candidates { + batch, exists := p.batches[candidate.id] if !exists { continue } - path := filepath.Join(p.batchesDir, id+".retired") + path := filepath.Join(p.batchesDir, candidate.id+".retired") if err := os.Rename(batch.dir, path); err != nil { pruneErr = errors.Join(pruneErr, err) continue } - delete(p.batches, id) + delete(p.batches, candidate.id) retired = append(retired, path) } if len(retired) > 0 { @@ -506,8 +544,6 @@ func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(fu } return nil }) - p.publishMu.Unlock() - locked = false pruneErr = errors.Join(pruneErr, err) for _, path := range retired { pruneErr = errors.Join(pruneErr, os.RemoveAll(path)) @@ -616,6 +652,14 @@ func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) er // PublishReplacement validates a prepared compacted batch, atomically swaps it // for its inputs while readers are pinned, then deletes retired inputs. func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string, publish func(func() error) error) error { + if err := validateBatchID(metadata.ID); err != nil { + return err + } + for _, id := range inputs { + if err := validateBatchID(id); err != nil { + return err + } + } final := p.BatchPath(metadata.ID) source := stage if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { @@ -631,16 +675,39 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, if replacement.metadata.Spans > 0 { replacement.traces.path = filepath.Join(final, "trace.fidx") } - p.publishMu.Lock() - locked := true - defer func() { - if locked { - p.publishMu.Unlock() + inputBatches := make(map[string]*storedBatch, len(inputs)) + for _, id := range inputs { + p.mu.RLock() + batch, exists := p.batches[id] + p.mu.RUnlock() + if exists { + inputBatches[id] = batch + continue } - }() - + active := p.BatchPath(id) + path := active + if _, statErr := os.Stat(path); errors.Is(statErr, os.ErrNotExist) { + path = filepath.Join(p.batchesDir, id+".retired-"+metadata.ID) + } else if statErr != nil { + return statErr + } + batch, loadErr := loadStoredBatch(path) + if errors.Is(loadErr, os.ErrNotExist) { + continue + } + if loadErr != nil { + return fmt.Errorf("load compaction input %s: %w", id, loadErr) + } + batch.dir = active + if batch.metadata.Spans > 0 { + batch.traces.path = filepath.Join(active, "trace.fidx") + } + inputBatches[id] = batch + } retired := make([][2]string, 0, len(inputs)) err = publish(func() error { + p.publishMu.Lock() + defer p.publishMu.Unlock() p.mu.Lock() defer p.mu.Unlock() installReplacement := func() { @@ -651,25 +718,32 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, } rollback := func() error { var rollbackErr error - if source == final { - if err := os.MkdirAll(filepath.Dir(stage), 0o755); err != nil { - installReplacement() - return err - } - if err := os.Rename(final, stage); err != nil { - // Restoring inputs while the output remains published would - // double-count every compacted row. Keep the output-only view. - installReplacement() - return err - } - source = stage - delete(p.batches, metadata.ID) - } for i := len(retired) - 1; i >= 0; i-- { rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) } + delete(p.batches, metadata.ID) + for id, batch := range inputBatches { + if _, err := os.Stat(batch.dir); err == nil { + p.batches[id] = batch + } else { + delete(p.batches, id) + } + } return errors.Join(rollbackErr, syncDirectory(p.batchesDir)) } + // Recovery may resume with the output already published. Move it back + // to staging before touching inputs, so rollback can only expose the + // complete old set or the complete replacement, never both. + if source == final { + if err := os.MkdirAll(filepath.Dir(stage), 0o755); err != nil { + return err + } + if err := os.Rename(final, stage); err != nil { + return err + } + source = stage + delete(p.batches, metadata.ID) + } for _, id := range inputs { active := p.BatchPath(id) retiredPath := filepath.Join(p.batchesDir, id+".retired-"+metadata.ID) @@ -701,8 +775,6 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, installReplacement() return nil }) - p.publishMu.Unlock() - locked = false if err != nil { return err } diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index 33b66961..f1209ee6 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -222,7 +222,7 @@ func TestPublishReplacementValidatesBeforePublication(t *testing.T) { } } -func TestPruneOwnsStoragePublicationBeforeExcludingReaders(t *testing.T) { +func TestPruneReaderWaitDoesNotBlockCommit(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { t.Fatal(err) @@ -230,23 +230,82 @@ func TestPruneOwnsStoragePublicationBeforeExcludingReaders(t *testing.T) { if err := store.CommitBatch(BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { t.Fatal(err) } - store.publishMu.Lock() entered := make(chan struct{}) + release := make(chan struct{}) done := make(chan error, 1) go func() { _, err := store.PruneBefore(2, 1, func(prune func() error) error { close(entered) + <-release return prune() }) done <- err }() select { case <-entered: - store.publishMu.Unlock() - t.Fatal("reader exclusion began before storage publication ownership") - case <-time.After(20 * time.Millisecond): + case <-time.After(time.Second): + t.Fatal("prune did not begin waiting for reader exclusion") } - store.publishMu.Unlock() + commitDone := make(chan error, 1) + go func() { + commitDone <- store.CommitBatch(BatchMetadata{ID: "concurrent", MaxIngestedNanos: 3}, []Span{{TraceID: "trace-2"}}, nil, nil) + }() + select { + case err := <-commitDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("commit blocked behind prune's reader wait") + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestReplacementReaderWaitDoesNotBlockCommit(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} + if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + t.Fatal(err) + } + output := BatchMetadata{ID: "output"} + if err := store.CommitBatch(output, span, nil, nil); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + stage := filepath.Join(t.TempDir(), "missing-stage") + go func() { + done <- store.PublishReplacement(stage, output, []string{"input"}, func(publish func() error) error { + close(entered) + <-release + return publish() + }) + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("replacement did not begin waiting for reader exclusion") + } + commitDone := make(chan error, 1) + go func() { + commitDone <- store.CommitBatch(BatchMetadata{ID: "concurrent"}, []Span{{TraceID: "trace-2"}}, nil, nil) + }() + select { + case err := <-commitDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("commit blocked behind replacement's reader wait") + } + close(release) if err := <-done; err != nil { t.Fatal(err) } @@ -287,6 +346,48 @@ func TestPublishReplacementKeepsOutputOnlyAfterSyncFailure(t *testing.T) { } } +func TestPublishReplacementUnpublishesRecoveredOutputBeforeRetiringInputs(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} + if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + t.Fatal(err) + } + output := BatchMetadata{ID: "output"} + if err := store.CommitBatch(output, span, nil, nil); err != nil { + t.Fatal(err) + } + blockedRetired := filepath.Join(store.BatchesDir(), "input.retired-output") + if err := os.Mkdir(blockedRetired, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(blockedRetired, "block"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + stage := filepath.Join(t.TempDir(), "recovery", "output") + err = store.PublishReplacement(stage, output, []string{"input"}, func(publish func() error) error { + return publish() + }) + if err == nil { + t.Fatal("replacement succeeded despite blocked input retirement") + } + if _, err := os.Stat(store.BatchPath("output")); !os.IsNotExist(err) { + t.Fatalf("recovered output remained visible beside active input: %v", err) + } + if _, err := os.Stat(store.BatchPath("input")); err != nil { + t.Fatalf("input was not left queryable after rollback: %v", err) + } + if _, err := os.Stat(stage); err != nil { + t.Fatalf("output was not preserved for the next recovery attempt: %v", err) + } + metadata := store.BatchMetadata() + if len(metadata) != 1 || metadata[0].ID != "input" { + t.Fatalf("live batches after rollback = %#v, want input only", metadata) + } +} + func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 1542a3b2..53260acb 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -165,15 +165,19 @@ func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) return selected } -func (r *Repository) CompactParquetBacklog(ctx context.Context, compactor ParquetCompactor, maxBatches int) (int, error) { +func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCompactor, maxBatches, maxPublications int) (int, error) { + if maxBatches <= 0 || maxPublications <= 0 { + return 0, nil + } total := 0 - for { + for range maxPublications { count, err := r.CompactParquet(ctx, compactor, maxBatches) total += count if err != nil || count == 0 { return total, err } } + return total, nil } type parquetPublishFunc func(context.Context, func() error) error @@ -205,7 +209,9 @@ func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPubli return err } if !stageExists && !finalExists { - if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID); err != nil { + if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID, func(swap func() error) error { + return publish(ctx, swap) + }); err != nil { return fmt.Errorf("restore compaction %s inputs: %w", marker.Output.ID, err) } if err := os.Remove(filepath.Join(r.root, "COMPACTION.json")); err != nil && !errors.Is(err, os.ErrNotExist) { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 02c966b1..9f48bb87 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -65,6 +65,14 @@ func (r *Repository) cleanupCompactionArtifacts() error { func (r *Repository) Close() error { return r.Parquet.Close() } +// CleanupParquet removes retired inputs only while no retention or compaction +// transaction can still need them for rollback. +func (r *Repository) CleanupParquet() error { + r.compactionMu.Lock() + defer r.compactionMu.Unlock() + return r.Parquet.CleanupRetired() +} + func (r *Repository) Commit(batch Batch) error { normalizeBatch(&batch) if err := validateBatch(batch); err != nil { @@ -90,18 +98,19 @@ func (r *Repository) PruneParquet(ctx context.Context, publisher ParquetPublishe }) } -func (r *Repository) PruneParquetBacklog(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches int) (int, error) { - if maxBatches <= 0 { +func (r *Repository) PruneParquetPass(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches, maxPublications int) (int, error) { + if maxBatches <= 0 || maxPublications <= 0 { return 0, nil } total := 0 - for { + for range maxPublications { count, err := r.PruneParquet(ctx, publisher, cutoff, maxBatches) total += count if err != nil || count < maxBatches { return total, err } } + return total, nil } func validateBatch(batch Batch) error { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index d79038fa..4a0d5b55 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" _ "github.com/duckdb/duckdb-go/v2" "github.com/labstack/fanout/internal/telemetry" @@ -21,6 +22,12 @@ type testParquetCompactor struct { afterSwap func() error } +type testParquetPublisherFunc func(context.Context, func() error) error + +func (f testParquetPublisherFunc) PublishParquet(ctx context.Context, publish func() error) error { + return f(ctx, publish) +} + func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { quoted := make([]string, len(inputs)) for i, input := range inputs { @@ -269,7 +276,61 @@ func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { } } -func TestRepositoryPrunesBacklogInBoundedPublications(t *testing.T) { +func TestRepositoryRestoresInputsThroughPublicationGate(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + batch := testBatch() + batch.ID = "retired-input" + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + marker := compactionMarker{ + Output: telemetry.BatchMetadata{ID: "missing-output", Generation: 1}, + Inputs: []string{batch.ID}, + } + data, err := json.Marshal(marker) + if err != nil { + t.Fatal(err) + } + if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { + t.Fatal(err) + } + retired := filepath.Join(repository.Parquet.BatchesDir(), batch.ID+".retired-"+marker.Output.ID) + if err := os.Rename(repository.Parquet.BatchPath(batch.ID), retired); err != nil { + t.Fatal(err) + } + + entered := make(chan struct{}) + release := make(chan struct{}) + publisher := testParquetPublisherFunc(func(_ context.Context, publish func() error) error { + close(entered) + <-release + return publish() + }) + done := make(chan error, 1) + go func() { done <- repository.RecoverParquet(context.Background(), publisher) }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("recovery did not enter the publication gate") + } + if _, err := os.Stat(repository.Parquet.BatchPath(batch.ID)); !os.IsNotExist(err) { + t.Fatalf("input became visible before publication: %v", err) + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if _, err := os.Stat(repository.Parquet.BatchPath(batch.ID)); err != nil { + t.Fatalf("input was not restored after publication: %v", err) + } +} + +func TestRepositoryPrunePassIsBoundedAndOldestFirst(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) @@ -278,9 +339,9 @@ func TestRepositoryPrunesBacklogInBoundedPublications(t *testing.T) { for i := range 5 { batch := testBatch() batch.ID = fmt.Sprintf("expired-%d", i) - batch.Spans[0].IngestedAt = 1 - batch.Logs[0].IngestedAt = 1 - batch.Metrics[0].IngestedAt = 1 + batch.Spans[0].IngestedAt = int64(i + 1) + batch.Logs[0].IngestedAt = int64(i + 1) + batch.Metrics[0].IngestedAt = int64(i + 1) if err := repository.Commit(batch); err != nil { t.Fatal(err) } @@ -290,12 +351,16 @@ func TestRepositoryPrunesBacklogInBoundedPublications(t *testing.T) { publications++ return nil }} - removed, err := repository.PruneParquetBacklog(context.Background(), publisher, 2, 2) + removed, err := repository.PruneParquetPass(context.Background(), publisher, 10, 2, 2) if err != nil { t.Fatal(err) } - if removed != 5 || publications != 3 { - t.Fatalf("removed=%d publications=%d, want 5 across 3 bounded swaps", removed, publications) + if removed != 4 || publications != 2 { + t.Fatalf("removed=%d publications=%d, want 4 across 2 bounded swaps", removed, publications) + } + metadata := repository.Parquet.BatchMetadata() + if len(metadata) != 1 || metadata[0].ID != "expired-4" { + t.Fatalf("remaining batches = %#v, want newest expired-4", metadata) } } From fbc5604401226705daa8cf73620e3555e30f666f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 19:22:08 -0700 Subject: [PATCH 19/31] fix(storage): guarantee maintenance progress Apply maintenance budgets only between completed publications so slow compactions make durable progress. Bound rollup admission and publication waits without canceling admitted cache work or removing ingest from rotation. --- internal/api/health.go | 3 - internal/api/health_test.go | 4 +- internal/query/duck.go | 61 ++++++------ internal/query/duck_test.go | 34 ++++++- internal/query/parquet_gate.go | 9 +- internal/telemetry/parquet.go | 103 +++++++++++--------- internal/telemetry/parquet_test.go | 69 ++++++------- internal/telemetry/store/compaction.go | 21 ++-- internal/telemetry/store/repository.go | 45 +++++++-- internal/telemetry/store/repository_test.go | 79 ++++++++++++--- 10 files changed, 276 insertions(+), 152 deletions(-) diff --git a/internal/api/health.go b/internal/api/health.go index bb3850bf..06466f77 100644 --- a/internal/api/health.go +++ b/internal/api/health.go @@ -256,9 +256,6 @@ func maintenanceResult(lastOK, lastAt time.Time, lastErr error, consecutiveFailu } if lastErr != nil { res.Status = "degraded" - if consecutiveFailures >= 3 { - res.Status = "unhealthy" - } res.Error = lastErr.Error() res.Detail = fmt.Sprintf("%d consecutive failed passes", consecutiveFailures) if !lastOK.IsZero() { diff --git a/internal/api/health_test.go b/internal/api/health_test.go index b999d917..e50a5ef7 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -189,7 +189,7 @@ func TestTelemetryReadinessReportsPublicationContentionAsDegraded(t *testing.T) release := make(chan struct{}) done := make(chan error, 1) go func() { - done <- duck.PublishParquet(context.Background(), func() error { + done <- duck.PublishParquet(context.Background(), func(context.Context) error { close(entered) <-release return nil @@ -301,7 +301,7 @@ func TestMaintenanceResult(t *testing.T) { }{ {"clean recent pass", now.Add(-10 * time.Minute), now.Add(-10 * time.Minute), nil, 0, now.Add(-2 * time.Hour), time.Hour, "ok"}, {"failing pass", now.Add(-3 * time.Hour), now.Add(-time.Hour), errors.New("boom"), 1, now.Add(-4 * time.Hour), time.Hour, "degraded"}, - {"repeated failure", now.Add(-4 * time.Hour), now.Add(-time.Hour), errors.New("boom"), 3, now.Add(-5 * time.Hour), time.Hour, "unhealthy"}, + {"repeated failure remains routable", now.Add(-4 * time.Hour), now.Add(-time.Hour), errors.New("boom"), 3, now.Add(-5 * time.Hour), time.Hour, "degraded"}, {"never ran, past grace", time.Time{}, time.Time{}, nil, 0, now.Add(-10 * time.Minute), time.Hour, "degraded"}, {"never ran, within grace", time.Time{}, time.Time{}, nil, 0, now.Add(-time.Minute), time.Hour, "ok"}, {"stalled after clean pass", now.Add(-5 * time.Hour), now.Add(-5 * time.Hour), nil, 0, now.Add(-6 * time.Hour), time.Hour, "degraded"}, diff --git a/internal/query/duck.go b/internal/query/duck.go index 752e9f6f..2b54dfbd 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -60,18 +60,18 @@ func (d *Duck) MaintenanceHealth() (lastOK, lastAt time.Time, consecutiveFailure } const ( - serviceRollupStateKey = "service_rollup_v2" - serviceRollupRawMaxKey = "service_rollup_v2_rawmax" - edgeRollupStateKey = "edge_rollup_v2" - edgeRollupRawMaxKey = "edge_rollup_v2_rawmax" - EndpointRollupStateKey = "endpoint_rollup_v1" - endpointRollupRawMaxKey = "endpoint_rollup_v1_rawmax" - endpointBackfillStateKey = "endpoint_rollup_v1_backfill_started" - EndpointReadyStateKey = "endpoint_rollup_v1_ready" - EndpointDisabledStateKey = "endpoint_rollup_v1_disabled" - defaultDuckDBPoolSize = 1 - parquetMaintenanceBatchLimit = 64 - parquetMaintenancePublishLimit = 4 + serviceRollupStateKey = "service_rollup_v2" + serviceRollupRawMaxKey = "service_rollup_v2_rawmax" + edgeRollupStateKey = "edge_rollup_v2" + edgeRollupRawMaxKey = "edge_rollup_v2_rawmax" + EndpointRollupStateKey = "endpoint_rollup_v1" + endpointRollupRawMaxKey = "endpoint_rollup_v1_rawmax" + endpointBackfillStateKey = "endpoint_rollup_v1_backfill_started" + EndpointReadyStateKey = "endpoint_rollup_v1_ready" + EndpointDisabledStateKey = "endpoint_rollup_v1_disabled" + defaultDuckDBPoolSize = 1 + parquetMaintenanceBatchLimit = 64 + parquetMaintenancePhaseBudget = 10 * time.Minute ) // rollupPublicationSafetyLag covers the maximum public SQL hold, publication @@ -451,18 +451,16 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() var pruneErr error if d.repository != nil { - storageCtx, cancelStorage := context.WithTimeout(ctx, 2*defaultWriterGrace) - defer cancelStorage() - recoveryErr := d.repository.RecoverParquet(storageCtx, d) + recoveryErr := d.repository.RecoverParquet(ctx, d) compactStart := time.Now() var cleanupErr, parquetErr, compactErr error compacted := 0 if recoveryErr == nil { cleanupErr = d.repository.CleanupParquet() if d.cfg.RetentionDays > 0 { - _, parquetErr = d.repository.PruneParquetPass(storageCtx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePublishLimit) + _, parquetErr = d.repository.PruneParquetPass(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) } - compacted, compactErr = d.repository.CompactParquetPass(storageCtx, d, parquetMaintenanceBatchLimit, parquetMaintenancePublishLimit) + compacted, compactErr = d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget) } compactResult := metrics.TelemetryNoop if recoveryErr != nil || compactErr != nil { @@ -533,9 +531,7 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { // consistent and deadlock-free. unlock := d.writeGate.Lock(writegate.WriteRollupService) defer unlock() - ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) - defer cancel() - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockRollupParquetRead(ctx); err != nil { return 0, err } defer d.parquetMu.RUnlock() @@ -656,9 +652,7 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { }() unlock := d.writeGate.Lock(writegate.WriteRollupEndpoint) defer unlock() - ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) - defer cancel() - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockRollupParquetRead(ctx); err != nil { return 0, err } defer d.parquetMu.RUnlock() @@ -781,9 +775,7 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { }() unlock := d.writeGate.Lock(writegate.WriteRollupEdge) defer unlock() - ctx, cancel := context.WithTimeout(ctx, rollupReaderLease) - defer cancel() - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockRollupParquetRead(ctx); err != nil { return 0, err } defer d.parquetMu.RUnlock() @@ -905,9 +897,10 @@ WHERE ingested_unix_nano > ? // backlog in a single statement. Unbounded catch-up is what took prod down on // 2026-06-13 (UTC): the edge rollup's first pass covered 12 days of spans, // spilled 375 GiB to temp, filled the disk, and never committed. -// Ten-minute chunks also keep the rebuildable read lease short enough for -// retention and compaction to acquire the Parquet publication gate. -const rollupChunkNanos = int64(10 * time.Minute) +// One-hour chunks preserve fast catch-up while bounding the work admitted to a +// transaction. Publication may wait for the current chunk, but does not cancel +// it and force the same cache work to restart indefinitely. +const rollupChunkNanos = int64(time.Hour) // edgeStartChunkNanos bounds how wide a start_time range one edge-rollup // DELETE+INSERT processes, so the call_edges self-join over a wide backlog @@ -1466,7 +1459,7 @@ func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, } // PublishParquet limits reader exclusion to the atomic directory swap. -func (d *Duck) PublishParquet(ctx context.Context, publish func() error) error { +func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) error) error { publishCtx, cancelPublish := context.WithTimeout(ctx, 2*defaultWriterGrace) defer cancelPublish() metrics.ParquetPublishWaiters.Inc() @@ -1479,7 +1472,7 @@ func (d *Duck) PublishParquet(ctx context.Context, publish func() error) error { return fmt.Errorf("wait for Parquet readers: %w", err) } defer d.parquetMu.Unlock() - return publish() + return publish(publishCtx) } func quoteDuckString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } @@ -1491,6 +1484,12 @@ func (d *Duck) lockParquetRead(ctx context.Context) error { return nil } +func (d *Duck) lockRollupParquetRead(ctx context.Context) error { + waitCtx, cancel := context.WithTimeout(ctx, rollupReaderLease) + defer cancel() + return d.lockParquetRead(waitCtx) +} + // ---- Queries for API ---- type LatencyRow struct { diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index e1b742d0..68739ac6 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -288,7 +288,7 @@ func TestPublishParquetHonorsContext(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() called := false - err := d.PublishParquet(ctx, func() error { + err := d.PublishParquet(ctx, func(context.Context) error { called = true return nil }) @@ -326,7 +326,7 @@ func TestParquetWorkDoesNotWaitForDuckDBWriteGate(t *testing.T) { t.Fatalf("merge waited for unrelated DuckDB write gate: %v", err) } called := false - if err := d.PublishParquet(ctx, func() error { called = true; return nil }); err != nil { + if err := d.PublishParquet(ctx, func(context.Context) error { called = true; return nil }); err != nil { t.Fatalf("publication waited for unrelated DuckDB write gate: %v", err) } if !called { @@ -402,6 +402,34 @@ func TestRollupReadLockHonorsContext(t *testing.T) { } } +func TestRollupAdmissionLeaseDoesNotCancelAdmittedWork(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + originalLease := rollupReaderLease + rollupReaderLease = 5 * time.Millisecond + t.Cleanup(func() { rollupReaderLease = originalLease }) + + mock.ExpectBegin() + mock.ExpectQuery("FROM rollup_state").WithArgs(serviceRollupStateKey). + WillReturnRows(sqlmock.NewRows([]string{"last_ingested_unix_nano"}).AddRow(100)) + mock.ExpectQuery("FROM rollup_state").WithArgs(serviceRollupRawMaxKey). + WillReturnRows(sqlmock.NewRows([]string{"last_ingested_unix_nano"}).AddRow(100)) + mock.ExpectQuery("FROM \\(").WillDelayFor(20 * time.Millisecond). + WillReturnRows(sqlmock.NewRows([]string{"watermark"}).AddRow(100)) + mock.ExpectCommit() + + d := &Duck{DB: db} + if _, err := d.refreshServiceRollup(context.Background()); err != nil { + t.Fatalf("admitted rollup was canceled by its admission lease: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + func TestIndexedTraceReadHonorsParquetGateContext(t *testing.T) { repository, err := telemetrystore.Open(t.TempDir()) if err != nil { @@ -523,7 +551,7 @@ func TestMaintenanceTracksConsecutiveFailures(t *testing.T) { type failingPublishCompactor struct{ *Duck } -func (f failingPublishCompactor) PublishParquet(context.Context, func() error) error { +func (f failingPublishCompactor) PublishParquet(context.Context, func(context.Context) error) error { return errors.New("injected publication failure") } diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 2dff745b..3ddc1997 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -12,10 +12,11 @@ import ( // behind maintenance, short enough that retention and compaction always run. const defaultWriterGrace = 30 * time.Second -// rollupReaderLease leaves publication enough headroom after the admission -// grace expires. Rollups are rebuildable cache work and may be canceled; user -// queries retain their caller-provided deadlines. -const rollupReaderLease = defaultWriterGrace / 2 +// rollupReaderLease bounds only how long rebuildable rollup work waits to enter +// the Parquet snapshot. Once admitted, the bounded rollup chunk runs under its +// caller context so a slow but healthy pass can commit instead of retrying the +// same chunk forever. +var rollupReaderLease = defaultWriterGrace / 2 // ErrParquetReadWait distinguishes publication contention from a query error. var ErrParquetReadWait = errors.New("wait for Parquet publication") diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 7382f965..b90abe44 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -58,12 +58,12 @@ type storedBatch struct { } type ParquetStore struct { - dir string - batchesDir string - stagingDir string - mu sync.RWMutex - publishMu sync.Mutex - batches map[string]*storedBatch + dir string + batchesDir string + stagingDir string + mu sync.RWMutex + publishGate chan struct{} + batches map[string]*storedBatch } type ParquetStats struct { @@ -74,8 +74,9 @@ type ParquetStats struct { func OpenParquetStore(dir string) (*ParquetStore, error) { p := &ParquetStore{ dir: dir, batchesDir: filepath.Join(dir, "batches"), stagingDir: filepath.Join(dir, "staging"), - batches: make(map[string]*storedBatch), + publishGate: make(chan struct{}, 1), batches: make(map[string]*storedBatch), } + p.publishGate <- struct{}{} for _, path := range []string{p.dir, p.batchesDir} { if err := os.MkdirAll(path, 0o755); err != nil { return nil, err @@ -154,6 +155,10 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] return nil } if info, err := os.Stat(final); err == nil && info.IsDir() { + if err := p.lockPublish(context.Background()); err != nil { + return err + } + defer p.unlockPublish() if err := syncDirectory(p.batchesDir); err != nil { return err } @@ -226,9 +231,22 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] if err := syncDirectory(stage); err != nil { return err } + prepared, err := loadStoredBatch(stage) + if err != nil { + return fmt.Errorf("validate staged Parquet batch: %w", err) + } + if prepared.metadata.ID != metadata.ID { + return fmt.Errorf("staged Parquet metadata ID %q does not match batch ID %q", prepared.metadata.ID, metadata.ID) + } + prepared.dir = final + if prepared.metadata.Spans > 0 { + prepared.traces.path = filepath.Join(final, "trace.fidx") + } - p.publishMu.Lock() - defer p.publishMu.Unlock() + if err := p.lockPublish(context.Background()); err != nil { + return err + } + defer p.unlockPublish() if p.hasBatch(metadata.ID) { complete = true return os.RemoveAll(stage) @@ -247,39 +265,26 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] if err := syncDirectory(p.batchesDir); err != nil { return err } - return p.registerBatch(final) + p.mu.Lock() + p.batches[metadata.ID] = prepared + p.mu.Unlock() + return nil } -// CleanupRetired removes inputs hidden by a completed retention or compaction -// publication. It is called only after compaction recovery has consumed its -// durable marker, so no rollback can still need these directories. -func (p *ParquetStore) CleanupRetired() error { - entries, err := os.ReadDir(p.batchesDir) - if err != nil { - return err - } - removed := false - var cleanupErr error - for _, entry := range entries { - name := entry.Name() - if !entry.IsDir() || strings.HasSuffix(name, BatchSuffix) || !strings.Contains(name, ".retired") { - continue - } - if err := os.RemoveAll(filepath.Join(p.batchesDir, name)); err != nil { - cleanupErr = errors.Join(cleanupErr, err) - } else { - removed = true - } - } - if removed { - cleanupErr = errors.Join(cleanupErr, syncDirectory(p.batchesDir)) +func (p *ParquetStore) lockPublish(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.publishGate: + return nil } - return cleanupErr } +func (p *ParquetStore) unlockPublish() { p.publishGate <- struct{}{} } + // RestoreRetiredInputs rolls back a compaction whose durable output vanished. // The complete namespace change is hidden from readers by publish. -func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func() error) error) error { +func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func(context.Context) error) error) error { if err := validateBatchID(replacementID); err != nil { return err } @@ -319,9 +324,11 @@ func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID strin } prepared = append(prepared, restoredInput{id: id, active: active, retired: path, batch: batch, move: move}) } - return publish(func() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() + return publish(func(ctx context.Context) error { + if err := p.lockPublish(ctx); err != nil { + return err + } + defer p.unlockPublish() installActive := func() { p.mu.Lock() defer p.mu.Unlock() @@ -491,7 +498,7 @@ func indexedSpanEarlier(left, right IndexedSpan) bool { // PruneBefore hides complete batches while readers are pinned, then deletes // the retired directories after publication. -func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(func() error) error) (int, error) { +func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(func(context.Context) error) error) (int, error) { if maxBatches <= 0 { return 0, nil } @@ -521,9 +528,11 @@ func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(fu } var retired []string var pruneErr error - err := publish(func() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() + err := publish(func(ctx context.Context) error { + if err := p.lockPublish(ctx); err != nil { + return err + } + defer p.unlockPublish() p.mu.Lock() defer p.mu.Unlock() for _, candidate := range candidates { @@ -651,7 +660,7 @@ func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) er // PublishReplacement validates a prepared compacted batch, atomically swaps it // for its inputs while readers are pinned, then deletes retired inputs. -func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string, publish func(func() error) error) error { +func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string, publish func(func(context.Context) error) error) error { if err := validateBatchID(metadata.ID); err != nil { return err } @@ -705,9 +714,11 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputBatches[id] = batch } retired := make([][2]string, 0, len(inputs)) - err = publish(func() error { - p.publishMu.Lock() - defer p.publishMu.Unlock() + err = publish(func(ctx context.Context) error { + if err := p.lockPublish(ctx); err != nil { + return err + } + defer p.unlockPublish() p.mu.Lock() defer p.mu.Unlock() installReplacement := func() { diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index f1209ee6..10f2e14f 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -210,9 +210,9 @@ func TestPublishReplacementValidatesBeforePublication(t *testing.T) { } stage := t.TempDir() called := false - err = store.PublishReplacement(stage, BatchMetadata{ID: "replacement"}, nil, func(publish func() error) error { + err = store.PublishReplacement(stage, BatchMetadata{ID: "replacement"}, nil, func(publish func(context.Context) error) error { called = true - return publish() + return publish(context.Background()) }) if err == nil { t.Fatal("invalid replacement was accepted") @@ -234,10 +234,10 @@ func TestPruneReaderWaitDoesNotBlockCommit(t *testing.T) { release := make(chan struct{}) done := make(chan error, 1) go func() { - _, err := store.PruneBefore(2, 1, func(prune func() error) error { + _, err := store.PruneBefore(2, 1, func(prune func(context.Context) error) error { close(entered) <-release - return prune() + return prune(context.Background()) }) done <- err }() @@ -264,6 +264,32 @@ func TestPruneReaderWaitDoesNotBlockCommit(t *testing.T) { } } +func TestPrunePublicationContextBoundsStorageLockWait(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + t.Fatal(err) + } + if err := store.lockPublish(context.Background()); err != nil { + t.Fatal(err) + } + defer store.unlockPublish() + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + removed, err := store.PruneBefore(2, 1, func(prune func(context.Context) error) error { + return prune(ctx) + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("PruneBefore error = %v, want deadline exceeded", err) + } + if removed != 0 || len(store.BatchMetadata()) != 1 { + t.Fatalf("timed-out publication removed=%d batches=%d, want the input untouched", removed, len(store.BatchMetadata())) + } +} + func TestReplacementReaderWaitDoesNotBlockCommit(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { @@ -282,10 +308,10 @@ func TestReplacementReaderWaitDoesNotBlockCommit(t *testing.T) { done := make(chan error, 1) stage := filepath.Join(t.TempDir(), "missing-stage") go func() { - done <- store.PublishReplacement(stage, output, []string{"input"}, func(publish func() error) error { + done <- store.PublishReplacement(stage, output, []string{"input"}, func(publish func(context.Context) error) error { close(entered) <-release - return publish() + return publish(context.Background()) }) }() select { @@ -328,8 +354,8 @@ func TestPublishReplacementKeepsOutputOnlyAfterSyncFailure(t *testing.T) { syncPublishedDirectory = func(string) error { return errors.New("injected directory sync failure") } t.Cleanup(func() { syncPublishedDirectory = originalSync }) - err = store.PublishReplacement(filepath.Join(t.TempDir(), "missing-stage"), output, []string{"input"}, func(publish func() error) error { - return publish() + err = store.PublishReplacement(filepath.Join(t.TempDir(), "missing-stage"), output, []string{"input"}, func(publish func(context.Context) error) error { + return publish(context.Background()) }) if err == nil { t.Fatal("replacement succeeded despite injected sync failure") @@ -367,8 +393,8 @@ func TestPublishReplacementUnpublishesRecoveredOutputBeforeRetiringInputs(t *tes t.Fatal(err) } stage := filepath.Join(t.TempDir(), "recovery", "output") - err = store.PublishReplacement(stage, output, []string{"input"}, func(publish func() error) error { - return publish() + err = store.PublishReplacement(stage, output, []string{"input"}, func(publish func(context.Context) error) error { + return publish(context.Background()) }) if err == nil { t.Fatal("replacement succeeded despite blocked input retirement") @@ -400,29 +426,6 @@ func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { } } -func TestParquetStoreCleansOnlyRetiredDirectories(t *testing.T) { - store, err := OpenParquetStore(t.TempDir()) - if err != nil { - t.Fatal(err) - } - retired := filepath.Join(store.BatchesDir(), "old.retired-compacted") - if err := os.Mkdir(retired, 0o755); err != nil { - t.Fatal(err) - } - if err := store.CommitBatch(BatchMetadata{ID: "contains.retired"}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { - t.Fatal(err) - } - if err := store.CleanupRetired(); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(retired); !os.IsNotExist(err) { - t.Fatalf("retired directory remains: %v", err) - } - if _, err := os.Stat(store.BatchPath("contains.retired")); err != nil { - t.Fatalf("published batch with retired in its ID was removed: %v", err) - } -} - func TestParquetStoreRejectsMetadataRowCountMismatchOnOpen(t *testing.T) { dir := t.TempDir() store, err := OpenParquetStore(dir) diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 53260acb..a8632a99 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -30,7 +30,7 @@ type compactionKey struct { // ParquetCompactor keeps DuckDB execution and publication locking in the query // layer while storage owns batch selection and crash-safe replacement state. type ParquetPublisher interface { - PublishParquet(context.Context, func() error) error + PublishParquet(context.Context, func(context.Context) error) error } type ParquetCompactor interface { @@ -165,22 +165,25 @@ func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) return selected } -func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCompactor, maxBatches, maxPublications int) (int, error) { - if maxBatches <= 0 || maxPublications <= 0 { +// CompactParquetPass starts complete compactions until the phase budget +// expires. An in-flight merge keeps the caller context so slow, valid work +// commits instead of restarting the same input group on every pass. +func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCompactor, maxBatches int, budget time.Duration) (int, error) { + if maxBatches <= 0 || budget <= 0 { return 0, nil } + deadline := time.Now().Add(budget) total := 0 - for range maxPublications { + for { count, err := r.CompactParquet(ctx, compactor, maxBatches) total += count - if err != nil || count == 0 { + if err != nil || count == 0 || !time.Now().Before(deadline) { return total, err } } - return total, nil } -type parquetPublishFunc func(context.Context, func() error) error +type parquetPublishFunc func(context.Context, func(context.Context) error) error func (r *Repository) RecoverParquet(ctx context.Context, publisher ParquetPublisher) error { r.compactionMu.Lock() @@ -209,7 +212,7 @@ func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPubli return err } if !stageExists && !finalExists { - if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID, func(swap func() error) error { + if err := r.Parquet.RestoreRetiredInputs(marker.Inputs, marker.Output.ID, func(swap func(context.Context) error) error { return publish(ctx, swap) }); err != nil { return fmt.Errorf("restore compaction %s inputs: %w", marker.Output.ID, err) @@ -223,7 +226,7 @@ func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPubli } func (r *Repository) completeCompaction(ctx context.Context, marker compactionMarker, publish parquetPublishFunc) error { - if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs, func(swap func() error) error { + if err := r.Parquet.PublishReplacement(r.compactionStage(marker.Output.ID), marker.Output, marker.Inputs, func(swap func(context.Context) error) error { return publish(ctx, swap) }); err != nil { return err diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 9f48bb87..e008e08d 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -38,7 +38,7 @@ func Open(root string) (*Repository, error) { return nil, err } r := &Repository{root: root, Parquet: parquetStore} - if err := r.recoverCompaction(context.Background(), func(_ context.Context, publish func() error) error { return publish() }); err != nil { + if err := r.recoverCompaction(context.Background(), func(ctx context.Context, publish func(context.Context) error) error { return publish(ctx) }); err != nil { _ = r.Close() return nil, fmt.Errorf("recover Parquet compaction: %w", err) } @@ -46,7 +46,7 @@ func Open(root string) (*Repository, error) { _ = r.Close() return nil, fmt.Errorf("clean Parquet compaction staging: %w", err) } - if err := r.Parquet.CleanupRetired(); err != nil { + if err := r.cleanupRetired(); err != nil { _ = r.Close() return nil, fmt.Errorf("clean retired Parquet batches: %w", err) } @@ -70,7 +70,31 @@ func (r *Repository) Close() error { return r.Parquet.Close() } func (r *Repository) CleanupParquet() error { r.compactionMu.Lock() defer r.compactionMu.Unlock() - return r.Parquet.CleanupRetired() + return r.cleanupRetired() +} + +func (r *Repository) cleanupRetired() error { + entries, err := os.ReadDir(r.Parquet.BatchesDir()) + if err != nil { + return err + } + removed := false + var cleanupErr error + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() || strings.HasSuffix(name, telemetry.BatchSuffix) || !strings.Contains(name, ".retired") { + continue + } + if err := os.RemoveAll(filepath.Join(r.Parquet.BatchesDir(), name)); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } else { + removed = true + } + } + if removed { + cleanupErr = errors.Join(cleanupErr, syncDirectory(r.Parquet.BatchesDir())) + } + return cleanupErr } func (r *Repository) Commit(batch Batch) error { @@ -93,24 +117,27 @@ func (r *Repository) RowCount() uint64 { return r.Parquet.RowCount() } func (r *Repository) PruneParquet(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches int) (int, error) { r.compactionMu.Lock() defer r.compactionMu.Unlock() - return r.Parquet.PruneBefore(cutoff, maxBatches, func(prune func() error) error { + return r.Parquet.PruneBefore(cutoff, maxBatches, func(prune func(context.Context) error) error { return publisher.PublishParquet(ctx, prune) }) } -func (r *Repository) PruneParquetPass(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches, maxPublications int) (int, error) { - if maxBatches <= 0 || maxPublications <= 0 { +// PruneParquetPass starts bounded publications until the phase budget expires. +// The budget is checked between publications so an atomic swap is never +// canceled halfway through. +func (r *Repository) PruneParquetPass(ctx context.Context, publisher ParquetPublisher, cutoff int64, maxBatches int, budget time.Duration) (int, error) { + if maxBatches <= 0 || budget <= 0 { return 0, nil } + deadline := time.Now().Add(budget) total := 0 - for range maxPublications { + for { count, err := r.PruneParquet(ctx, publisher, cutoff, maxBatches) total += count - if err != nil || count < maxBatches { + if err != nil || count < maxBatches || !time.Now().Before(deadline) { return total, err } } - return total, nil } func validateBatch(batch Batch) error { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 4a0d5b55..22020348 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -22,9 +22,9 @@ type testParquetCompactor struct { afterSwap func() error } -type testParquetPublisherFunc func(context.Context, func() error) error +type testParquetPublisherFunc func(context.Context, func(context.Context) error) error -func (f testParquetPublisherFunc) PublishParquet(ctx context.Context, publish func() error) error { +func (f testParquetPublisherFunc) PublishParquet(ctx context.Context, publish func(context.Context) error) error { return f(ctx, publish) } @@ -41,11 +41,11 @@ func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, return err } -func (c *testParquetCompactor) PublishParquet(_ context.Context, publish func() error) error { +func (c *testParquetCompactor) PublishParquet(ctx context.Context, publish func(context.Context) error) error { if c.publishErr != nil { return c.publishErr } - if err := publish(); err != nil { + if err := publish(ctx); err != nil { return err } if c.afterSwap != nil { @@ -306,10 +306,10 @@ func TestRepositoryRestoresInputsThroughPublicationGate(t *testing.T) { entered := make(chan struct{}) release := make(chan struct{}) - publisher := testParquetPublisherFunc(func(_ context.Context, publish func() error) error { + publisher := testParquetPublisherFunc(func(ctx context.Context, publish func(context.Context) error) error { close(entered) <-release - return publish() + return publish(ctx) }) done := make(chan error, 1) go func() { done <- repository.RecoverParquet(context.Background(), publisher) }() @@ -330,7 +330,7 @@ func TestRepositoryRestoresInputsThroughPublicationGate(t *testing.T) { } } -func TestRepositoryPrunePassIsBoundedAndOldestFirst(t *testing.T) { +func TestRepositoryPrunePassDrainsWithinBudgetAndOldestFirst(t *testing.T) { repository, err := Open(t.TempDir()) if err != nil { t.Fatal(err) @@ -351,16 +351,71 @@ func TestRepositoryPrunePassIsBoundedAndOldestFirst(t *testing.T) { publications++ return nil }} - removed, err := repository.PruneParquetPass(context.Background(), publisher, 10, 2, 2) + removed, err := repository.PruneParquetPass(context.Background(), publisher, 10, 2, time.Second) if err != nil { t.Fatal(err) } - if removed != 4 || publications != 2 { - t.Fatalf("removed=%d publications=%d, want 4 across 2 bounded swaps", removed, publications) + if removed != 5 || publications != 3 { + t.Fatalf("removed=%d publications=%d, want 5 across 3 bounded swaps", removed, publications) } metadata := repository.Parquet.BatchMetadata() - if len(metadata) != 1 || metadata[0].ID != "expired-4" { - t.Fatalf("remaining batches = %#v, want newest expired-4", metadata) + if len(metadata) != 0 { + t.Fatalf("remaining batches = %#v, want the backlog drained", metadata) + } +} + +func TestRepositoryPrunePassFinishesPublicationAfterBudget(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + for i := range 4 { + batch := testBatch() + batch.ID = fmt.Sprintf("expired-%d", i) + batch.Spans[0].IngestedAt = int64(i + 1) + batch.Logs[0].IngestedAt = int64(i + 1) + batch.Metrics[0].IngestedAt = int64(i + 1) + if err := repository.Commit(batch); err != nil { + t.Fatal(err) + } + } + publications := 0 + publisher := &testParquetCompactor{afterSwap: func() error { + publications++ + time.Sleep(10 * time.Millisecond) + return nil + }} + removed, err := repository.PruneParquetPass(context.Background(), publisher, 10, 2, time.Millisecond) + if err != nil { + t.Fatal(err) + } + if removed != 2 || publications != 1 { + t.Fatalf("removed=%d publications=%d, want the in-flight 2-batch publication completed once", removed, publications) + } +} + +func TestRepositoryCleanupOwnsRetiredDirectories(t *testing.T) { + repository, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + retired := filepath.Join(repository.Parquet.BatchesDir(), "old.retired-compacted") + if err := os.Mkdir(retired, 0o755); err != nil { + t.Fatal(err) + } + if err := repository.Commit(Batch{ID: "contains.retired", Spans: []telemetry.Span{{TraceID: "trace"}}}); err != nil { + t.Fatal(err) + } + if err := repository.CleanupParquet(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(retired); !os.IsNotExist(err) { + t.Fatalf("retired directory remains: %v", err) + } + if _, err := os.Stat(repository.Parquet.BatchPath("contains.retired")); err != nil { + t.Fatalf("published batch with retired in its ID was removed: %v", err) } } From 7de118181610b4de9b9a074011e12c09573270e4 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 20:10:22 -0700 Subject: [PATCH 20/31] fix(storage): bound maintenance work by construction Five review rounds relocated the process-wide serialization point four times (parquetMu, writeGate, publishMu, back to parquetMu) without the finding rate falling, because each fix bolted a timeout or count onto a call graph where no constant could see the cost of the work it budgeted. This bounds that work at admission instead. The edge rollup stops after maxEdgeSubWindowsPerPass sub-windows, persists a resume cursor, and leaves the ingested watermark behind until the window finishes. A seed load or backfill compresses a wide start_time range into a narrow ingested window, so chunking ingested time alone let one pass expand into an unbounded number of sub-windows and hold the read gate for as long as that took. Cancelling mid-transaction only discarded the work and retried it forever, so the pass now yields instead, and hold time is a function of the constant rather than the shape of the data. PublishParquet budgets the reader drain and the directory swap separately. Sharing one deadline meant a publisher that spent most of it waiting entered the exclusive window with almost nothing left and cancelled its own renames after paying the full cost of excluding every reader. CommitBatch takes a context, threaded from the writer through Repository.Commit. It was the last publish-gate waiter using context.Background(), so a stalled publication blocked ingest indefinitely and OTLP clients lost rows to their own timeouts. A compaction marker that fails recovery maxCompactionRecoveryAttempts times is set aside as COMPACTION.json.failed, and Open sets it aside and boots rather than exiting. Recovery correctly gates retention, compaction, and retired cleanup, so without a give-up path one bad marker latched all maintenance off for the process lifetime and refusing to boot left no way to run the cleanup that would clear it. Nothing is deleted: the staged output survives and cleanupRetired now protects the .retired- sets named by any live or set-aside marker. Cancelled passes do not count toward giving up. PruneBefore holds p.mu only to mutate the in-memory set, not across up to 64 renames and an fsync. CommitBatch's first action reads that same mutex, which made it a second undeadlined serialization point on the ingest path, invisible to the publish gate's accounting. Adopting a directory left by an interrupted commit now validates it before taking the gate rather than inside it. Removes Repository.Trace and the panicking parquetReadGate.RLock/Lock wrappers: both were duplicate paths, and Repository.Trace bypassed the read gate entirely. Adds invariant tests for the bounded rollup hold, the marker give-up and its rollback-set protection, booting past an unrecoverable marker, and ingest surviving a stalled publication, so a future relocation fails CI instead of waiting for another review round. --- internal/observability/namespace_test.go | 2 +- .../observability/performance_rollup_test.go | 2 +- internal/observability/service_test.go | 67 +++--- internal/query/duck.go | 123 +++++++++-- internal/query/duck_test.go | 22 +- internal/query/edge_backlog_test.go | 137 +++++++++++- internal/query/parquet_gate.go | 12 -- internal/query/parquet_gate_test.go | 27 ++- internal/telemetry/parquet.go | 81 ++++++- internal/telemetry/parquet_test.go | 69 ++++-- internal/telemetry/store/compaction.go | 24 ++- internal/telemetry/store/repository.go | 117 ++++++++-- internal/telemetry/store/repository_test.go | 203 ++++++++++++++++-- internal/telemetry/store/writer.go | 4 +- internal/telemetry/store/writer_test.go | 6 +- 15 files changed, 751 insertions(+), 145 deletions(-) diff --git a/internal/observability/namespace_test.go b/internal/observability/namespace_test.go index 03c2940d..db74c7a0 100644 --- a/internal/observability/namespace_test.go +++ b/internal/observability/namespace_test.go @@ -38,7 +38,7 @@ CREATE TABLE service_rollup ( t.Fatalf("insert service rollups: %v", err) } - svc := New(SQLDB(db), newTestRepository(t)) + svc := New(SQLDB(db), newTestRepository(t).Parquet) result, err := svc.Overview(context.Background(), Scope{Start: stamp.Add(-time.Minute), End: stamp.Add(time.Minute)}, 100) if err != nil { t.Fatalf("Overview: %v", err) diff --git a/internal/observability/performance_rollup_test.go b/internal/observability/performance_rollup_test.go index 8d56837f..5d13e3e6 100644 --- a/internal/observability/performance_rollup_test.go +++ b/internal/observability/performance_rollup_test.go @@ -90,7 +90,7 @@ FROM (VALUES ` + seed.values + `) t(ms)` t.Fatalf("seed endpoint rollup state: %v", err) } - svc := New(SQLDB(db), newTestRepository(t)) + svc := New(SQLDB(db), newTestRepository(t).Parquet) svc.endpointMature.Store(true) var cachedCalls, totalCachedCalls int64 var minBucket, maxBucket time.Time diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 63666573..b397f502 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -24,20 +24,21 @@ func newTestRepository(t *testing.T) *telemetrystore.Repository { return repository } -func newMockService(t *testing.T) (*Service, sqlmock.Sqlmock) { +func newMockService(t *testing.T) (*Service, sqlmock.Sqlmock, *telemetrystore.Repository) { t.Helper() db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New: %v", err) } t.Cleanup(func() { _ = db.Close() }) - svc := New(SQLDB(db), newTestRepository(t)) + repository := newTestRepository(t) + svc := New(SQLDB(db), repository.Parquet) svc.now = func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) } - return svc, mock + return svc, mock, repository } func TestNormalizeScopeDefaultsAndBounds(t *testing.T) { - svc, _ := newMockService(t) + svc, _, _ := newMockService(t) scope, err := svc.normalizeScope(Scope{}) if err != nil { t.Fatalf("normalizeScope: %v", err) @@ -56,7 +57,7 @@ func TestNormalizeScopeDefaultsAndBounds(t *testing.T) { } func TestOverviewReturnsCanonicalEnvelope(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) rows := sqlmock.NewRows([]string{"service", "spans", "error_rate", "p50_ms", "p95_ms", "log_count", "metric_count"}). @@ -88,7 +89,7 @@ func TestOverviewReturnsCanonicalEnvelope(t *testing.T) { } func TestTopologyUsesSharedNodesAndTypedEdges(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) mock.ExpectQuery(regexp.QuoteMeta(overviewQuery)). @@ -117,7 +118,7 @@ func TestTopologyUsesSharedNodesAndTypedEdges(t *testing.T) { } func TestPerformanceReturnsAllVisualizationDatasets(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, _ := newMockService(t) svc.endpointMature.Store(true) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) @@ -163,7 +164,7 @@ func TestPerformanceReturnsAllVisualizationDatasets(t *testing.T) { } func TestQueryEndpointsFallsBackToRawUntilBackfillReady(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) @@ -187,7 +188,7 @@ func TestQueryEndpointsFallsBackToRawUntilBackfillReady(t *testing.T) { } func TestQueryEndpointsFallsBackToRawWhenRollupProbeFails(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, _ := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) @@ -211,13 +212,13 @@ func TestQueryEndpointsFallsBackToRawWhenRollupProbeFails(t *testing.T) { } func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) mock.ExpectQuery(regexp.QuoteMeta(recentTraceQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout"). WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "ERROR", StatusMsg: "declined"}, {Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"}, }, Logs: []telemetry.Log{ @@ -251,10 +252,10 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { } func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "logs-fixture", Logs: []telemetry.Log{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "logs-fixture", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "payment declined", TraceID: "trace-1", SpanID: "root"}, {Namespace: "prod", TimeUnixNanos: start.Add(time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "card declined: token=abc123", TraceID: "trace-2", SpanID: "root2"}, {Namespace: "prod", TimeUnixNanos: start.Add(2 * time.Millisecond).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: `auth declined: {"password":"hunter2"}`, TraceID: "trace-3", SpanID: "root3"}, @@ -292,13 +293,13 @@ func TestLogsAppliesFiltersAndBuildsHistogram(t *testing.T) { } func TestLogsRetainsOnlyNewestLimit(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) logs := make([]telemetry.Log, 100) for i := range logs { logs[i] = telemetry.Log{Namespace: "prod", TimeUnixNanos: start.Add(time.Duration(i) * time.Millisecond).UnixNano(), Severity: "INFO", Body: "entry"} } - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "bounded-logs", Logs: logs}); err != nil { t.Fatal(err) } entryRows := sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}) @@ -321,10 +322,10 @@ func TestLogsRetainsOnlyNewestLimit(t *testing.T) { } func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "parquet-logs", Logs: []telemetry.Log{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "parquet-logs", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "token=secret", TraceID: "trace-parquet", }}}); err != nil { @@ -351,16 +352,16 @@ func TestLogsAlwaysUseAuthoritativeParquet(t *testing.T) { } func TestLogsQueryParquetAcrossBatches(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Second) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "overlap-newer", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.Add(150 * time.Millisecond).UnixNano(), Body: "newer-old", Severity: "INFO"}, {Namespace: "prod", TimeUnixNanos: start.Add(300 * time.Millisecond).UnixNano(), Body: "newer-batch", Severity: "INFO"}, }}); err != nil { t.Fatal(err) } - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "overlap-late", Logs: []telemetry.Log{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "overlap-late", Logs: []telemetry.Log{ {Namespace: "prod", TimeUnixNanos: start.Add(100 * time.Millisecond).UnixNano(), Body: "late-old", Severity: "INFO"}, {Namespace: "prod", TimeUnixNanos: start.Add(200 * time.Millisecond).UnixNano(), Body: "late-boundary", Severity: "INFO"}, }}); err != nil { @@ -393,7 +394,7 @@ func TestLogsQueryParquetAcrossBatches(t *testing.T) { } func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) batches := []telemetrystore.Batch{ {ID: "trace-latest", Spans: []telemetry.Span{{Namespace: "prod", TraceID: "trace-order", SpanID: "root", StartUnixNanos: start.UnixNano(), DurationMS: 1}}, Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(30 * time.Millisecond).UnixNano(), Body: "latest"}}}, @@ -402,7 +403,7 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { {ID: "trace-outside-span", Logs: []telemetry.Log{{Namespace: "prod", TraceID: "trace-order", TimeUnixNanos: start.Add(2 * time.Minute).UnixNano(), Body: "unrelated later event"}}}, } for _, batch := range batches { - if err := svc.repository.(*telemetrystore.Repository).Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -423,10 +424,10 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { } func TestTraceUsesIndexedParquet(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "indexed-trace", Spans: []telemetry.Span{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "indexed-trace", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "parquet-trace", SpanID: "root", ServiceName: "checkout", Name: "pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 25, StatusCode: "ERROR", StatusMsg: "declined", }}}); err != nil { @@ -452,16 +453,16 @@ func TestTraceUsesIndexedParquet(t *testing.T) { } func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-old-root", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "root", ServiceName: "frontend", StartUnixNanos: start.Add(10 * time.Minute).UnixNano(), DurationMS: 100, StatusCode: "ERROR", }}}); err != nil { t.Fatal(err) } - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-newer-child", Spans: []telemetry.Span{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-newer-child", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "split-trace", SpanID: "child", ParentSpanID: "root", ServiceName: "backend", StartUnixNanos: start.Add(50 * time.Minute).UnixNano(), DurationMS: 25, StatusCode: "OK", }}}); err != nil { @@ -483,11 +484,11 @@ func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { } func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-after-rebuild", Spans: []telemetry.Span{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-after-rebuild", Spans: []telemetry.Span{{ Namespace: "prod", TraceID: "new-trace", SpanID: "root", ServiceName: "frontend", StartUnixNanos: cutoff.Add(time.Minute).UnixNano(), DurationMS: 10, StatusCode: "OK", }}}); err != nil { @@ -509,11 +510,11 @@ func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { } func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) cutoff := start.Add(30 * time.Minute) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "trace-cross-namespace", Spans: []telemetry.Span{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-cross-namespace", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "shared-trace", SpanID: "child", ParentSpanID: "old-root", StartUnixNanos: cutoff.Add(time.Minute).UnixNano()}, {Namespace: "staging", TraceID: "shared-trace", SpanID: "root", StartUnixNanos: cutoff.Add(2 * time.Minute).UnixNano()}, }}); err != nil { @@ -537,10 +538,10 @@ func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { var _ DB = queryrows.SQLAdapter{} func TestLogsBoundsParquetQueryWithLimitAndAggregatedBuckets(t *testing.T) { - svc, mock := newMockService(t) + svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - if err := svc.repository.(*telemetrystore.Repository).Commit(telemetrystore.Batch{ID: "bounded-parquet", Logs: []telemetry.Log{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "bounded-parquet", Logs: []telemetry.Log{{ Namespace: "prod", TimeUnixNanos: start.Add(time.Minute).UnixNano(), Severity: "ERROR", ServiceName: "checkout", Body: "hello", TraceID: "trace-parquet", }}}); err != nil { diff --git a/internal/query/duck.go b/internal/query/duck.go index 2b54dfbd..cfeefda6 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -64,6 +64,8 @@ const ( serviceRollupRawMaxKey = "service_rollup_v2_rawmax" edgeRollupStateKey = "edge_rollup_v2" edgeRollupRawMaxKey = "edge_rollup_v2_rawmax" + edgeRollupSubCursorKey = "edge_rollup_v2_substart" + edgeRollupSubWindowKey = "edge_rollup_v2_subwindow_end" EndpointRollupStateKey = "endpoint_rollup_v1" endpointRollupRawMaxKey = "endpoint_rollup_v1_rawmax" endpointBackfillStateKey = "endpoint_rollup_v1_backfill_started" @@ -74,6 +76,16 @@ const ( parquetMaintenancePhaseBudget = 10 * time.Minute ) +// parquetDrainBudget bounds how long a publisher waits for readers already +// inside the snapshot to leave. parquetSwapBudget bounds the exclusive window +// that follows — the renames and fsync of at most parquetMaintenanceBatchLimit +// directories — which must be allowed to finish rather than be abandoned with +// every reader already excluded. +const ( + parquetDrainBudget = 2 * defaultWriterGrace + parquetSwapBudget = 30 * time.Second +) + // rollupPublicationSafetyLag covers the maximum public SQL hold, publication // grace, bounded commit retries, and queued Parquet encoding with headroom for // a busy disk. Rows stamped at request receipt remain inside the recomputed @@ -278,6 +290,11 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { {serviceRollupRawMaxKey, svc}, {edgeRollupStateKey, edge}, {edgeRollupRawMaxKey, edge}, + // Skipping discards any half-aggregated window, so its resume point + // must go too or the next pass would reopen a window behind the + // watermark it just advanced. + {edgeRollupSubCursorKey, 0}, + {edgeRollupSubWindowKey, 0}, // Endpoint queries remain on their raw-span fallback when the operator // explicitly skips historical rollups. Mark this cache disabled instead of // later declaring a new-only, incomplete endpoint cache ready. @@ -801,7 +818,22 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { return 0, err } sourceMax = rawWatermark - if rawWatermark <= lastWatermark { + + // A previous pass that ran out of sub-window budget recorded where to + // resume and left the ingested watermark untouched. Reuse that window + // verbatim so its unprocessed start_time tail is finished before the + // watermark is allowed past it. + subCursor, err := rollupWatermark(ctx, tx, edgeRollupSubCursorKey) + if err != nil { + return 0, err + } + resumeEnd, err := rollupWatermark(ctx, tx, edgeRollupSubWindowKey) + if err != nil { + return 0, err + } + resuming := subCursor > 0 && resumeEnd > lastWatermark + + if rawWatermark <= lastWatermark && !resuming { err := tx.Commit() if err == nil { result = metrics.RollupNoop @@ -816,6 +848,10 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { } } windowStart, windowEnd, chunked := rollupWindow(lastWatermark, minIngested, rawWatermark) + if resuming { + windowEnd = resumeEnd + chunked = windowEnd < rawWatermark + } // Same chunking + trailing-window logic as the service rollup (see // refreshServiceRollup): never advance into the lag window below an @@ -853,10 +889,23 @@ WHERE ingested_unix_nano > ? } var totalAffected int64 + completed := true + var nextCursor int64 if minStartT.Valid { subLo := minStartT.Time + if subCursor > 0 { + if resumeAt := time.Unix(0, subCursor).In(subLo.Location()); resumeAt.After(subLo) { + subLo = resumeAt + } + } maxT := maxStartT.Time + processed := 0 for !subLo.After(maxT) { + if processed == maxEdgeSubWindowsPerPass { + completed = false + nextCursor = subLo.UnixNano() + break + } subHi := subLo.Add(time.Duration(edgeStartChunkNanos)) if _, err := tx.ExecContext(ctx, edgeRollupDeleteSQL, windowStart, windowEnd, subLo, subHi); err != nil { return 0, err @@ -872,14 +921,32 @@ WHERE ingested_unix_nano > ? totalAffected += rows } subLo = subHi + processed++ } } - if err := storeRollupWatermark(ctx, tx, edgeRollupStateKey, newWatermark); err != nil { - return 0, err - } - if err := storeRollupWatermark(ctx, tx, edgeRollupRawMaxKey, rawMaxProcessed); err != nil { - return 0, err + // An unfinished window keeps the ingested watermark where it was: the + // sub-window cursor is the only durable record that part of this window is + // already aggregated, and advancing past it would drop the remainder. + storedWatermark := newWatermark + if completed { + if err := storeRollupWatermark(ctx, tx, edgeRollupStateKey, newWatermark); err != nil { + return 0, err + } + if err := storeRollupWatermark(ctx, tx, edgeRollupRawMaxKey, rawMaxProcessed); err != nil { + return 0, err + } + if err := clearEdgeRollupCursor(ctx, tx); err != nil { + return 0, err + } + } else { + storedWatermark = lastWatermark + if err := storeRollupWatermark(ctx, tx, edgeRollupSubCursorKey, nextCursor); err != nil { + return 0, err + } + if err := storeRollupWatermark(ctx, tx, edgeRollupSubWindowKey, windowEnd); err != nil { + return 0, err + } } if err := tx.Commit(); err != nil { return 0, err @@ -887,10 +954,19 @@ WHERE ingested_unix_nano > ? result = metrics.RollupSuccess recordedRows = totalAffected - updateRollupProgress(metrics.RollupEdge, true, newWatermark, rawWatermark) + updateRollupProgress(metrics.RollupEdge, true, storedWatermark, rawWatermark) return totalAffected, nil } +// clearEdgeRollupCursor drops the resume point once its window is fully +// aggregated, so the next pass opens a fresh ingested window. +func clearEdgeRollupCursor(ctx context.Context, tx *sql.Tx) error { + if err := storeRollupWatermark(ctx, tx, edgeRollupSubCursorKey, 0); err != nil { + return err + } + return storeRollupWatermark(ctx, tx, edgeRollupSubWindowKey, 0) +} + // rollupChunkNanos caps how much ingest history one rollup pass scans. A pass // that starts far behind — first run on an existing dataset, or recovery after // an outage — catches up one chunk per tick instead of aggregating the whole @@ -907,6 +983,20 @@ const rollupChunkNanos = int64(time.Hour) // (catch-up/bulk-load) can't exhaust memory. The pass loops over sub-windows. const edgeStartChunkNanos = int64(30 * time.Minute) +// maxEdgeSubWindowsPerPass bounds how many start_time sub-windows one pass +// processes, and with it how long that pass holds the Parquet read gate. +// +// Chunking ingested time alone does not bound the work: a seed load or a +// backfill compresses a wide start_time range into a narrow ingested window, +// so a single one-hour chunk can expand to hundreds of sub-windows and hold +// the gate for as long as that takes. No timeout can bound that from outside — +// cancelling mid-transaction only discards the work and retries it forever. So +// the pass instead stops at a fixed number of sub-windows, persists where to +// resume, and leaves the ingested watermark where it was. Hold time is then a +// function of this constant rather than of the dataset's shape, and every pass +// makes durable forward progress. +const maxEdgeSubWindowsPerPass = 8 + // rollupWindow bounds one pass's scan to (start, end]. start falls back to // just before the oldest ingested row when there's no stored watermark, so a // first pass doesn't open the window at the epoch. @@ -1438,7 +1528,7 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet return nil, err } defer d.parquetMu.RUnlock() - return d.repository.Trace(ctx, query) + return d.repository.Parquet.Trace(ctx, query) } // MergeParquet executes the query-engine-specific half of compaction. It reads @@ -1459,12 +1549,19 @@ func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, } // PublishParquet limits reader exclusion to the atomic directory swap. +// +// The drain and the swap get separate budgets. Sharing one deadline meant a +// publisher that spent most of it waiting for readers entered the exclusive +// window with almost nothing left, cancelled its own renames, and failed the +// pass — having already paid the full cost of excluding every reader. The swap +// is renames plus fsync against a bounded batch count, so it is budgeted for +// what that work costs rather than for whatever the drain happened to leave. func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) error) error { - publishCtx, cancelPublish := context.WithTimeout(ctx, 2*defaultWriterGrace) - defer cancelPublish() + drainCtx, cancelDrain := context.WithTimeout(ctx, parquetDrainBudget) + defer cancelDrain() metrics.ParquetPublishWaiters.Inc() waitStarted := time.Now() - err := d.parquetMu.LockContext(publishCtx) + err := d.parquetMu.LockContext(drainCtx) metrics.ParquetPublishWaiters.Dec() metrics.ParquetPublishWait.Observe(time.Since(waitStarted).Seconds()) if err != nil { @@ -1472,7 +1569,9 @@ func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) return fmt.Errorf("wait for Parquet readers: %w", err) } defer d.parquetMu.Unlock() - return publish(publishCtx) + swapCtx, cancelSwap := context.WithTimeout(ctx, parquetSwapBudget) + defer cancelSwap() + return publish(swapCtx) } func quoteDuckString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 68739ac6..c2a7b228 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -176,7 +176,7 @@ func TestNewDuckUsesUTCForEveryConnection(t *testing.T) { } defer d.Close() eventTime := time.Date(2026, 8, 27, 16, 0, 30, 0, time.UTC) - if err := repository.Commit(telemetrystore.Batch{ID: "timezone-window", Spans: []telemetry.Span{{ + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "timezone-window", Spans: []telemetry.Span{{ Namespace: "default", TraceID: "trace-timezone", SpanID: "span-timezone", ServiceName: "checkout", StartUnixNanos: eventTime.UnixNano(), DurationMS: 5, StatusCode: "STATUS_CODE_OK", IngestedAt: eventTime.UnixNano(), @@ -236,7 +236,7 @@ func TestQueryContextHoldsParquetLockUntilRowsFinish(t *testing.T) { } lockAcquired := make(chan struct{}) go func() { - d.parquetMu.Lock() + mustLock(&d.parquetMu) close(lockAcquired) d.parquetMu.Unlock() }() @@ -267,7 +267,7 @@ func TestQueryContextHoldsParquetLockUntilRowsFinish(t *testing.T) { func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { d := &Duck{} - d.parquetMu.Lock() + mustLock(&d.parquetMu) ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() started := time.Now() @@ -283,7 +283,7 @@ func TestQueryRowScanCancelsWhileMaintenanceWaitsForReaders(t *testing.T) { func TestPublishParquetHonorsContext(t *testing.T) { d := &Duck{} - d.parquetMu.RLock() + mustRLock(t, &d.parquetMu) timeoutsBefore := testutil.ToFloat64(metrics.ParquetPublishTimeouts) ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() @@ -345,12 +345,12 @@ func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { defer db.Close() mock.ExpectQuery("SELECT 1").WillReturnRows(sqlmock.NewRows([]string{"value"}).AddRow(1)) d := &Duck{DB: db} - d.parquetMu.RLock() + mustRLock(t, &d.parquetMu) writerAcquired := make(chan struct{}) releaseWriter := make(chan struct{}) writerDone := make(chan struct{}) go func() { - d.parquetMu.Lock() + mustLock(&d.parquetMu) close(writerAcquired) <-releaseWriter d.parquetMu.Unlock() @@ -392,7 +392,7 @@ func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { func TestRollupReadLockHonorsContext(t *testing.T) { d := &Duck{} - d.parquetMu.Lock() + mustLock(&d.parquetMu) ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() _, err := d.refreshServiceRollup(ctx) @@ -437,7 +437,7 @@ func TestIndexedTraceReadHonorsParquetGateContext(t *testing.T) { } defer repository.Close() d := &Duck{repository: repository} - d.parquetMu.Lock() + mustLock(&d.parquetMu) ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) defer cancel() _, err = d.Trace(ctx, telemetry.TraceQuery{TraceID: "trace", StartNanos: 1, EndNanos: 2, Limit: 1}) @@ -458,7 +458,7 @@ func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { t.Fatal(err) } defer repository.Close() - if err := repository.Commit(telemetrystore.Batch{ID: "expired", Spans: []telemetry.Span{{TraceID: "trace", IngestedAt: 1}}}); err != nil { + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "expired", Spans: []telemetry.Span{{TraceID: "trace", IngestedAt: 1}}}); err != nil { t.Fatal(err) } mock.ExpectExec("DELETE FROM service_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) @@ -466,7 +466,7 @@ func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { mock.ExpectExec("DELETE FROM edge_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CHECKPOINT").WillReturnResult(sqlmock.NewResult(0, 0)) d := &Duck{DB: db, repository: repository, cfg: config.Config{MaintenanceInterval: time.Nanosecond, RetentionDays: 1}} - d.parquetMu.RLock() + mustRLock(t, &d.parquetMu) release := d.writeGate.Lock(writegate.WriteRollupService) done := make(chan error, 1) go func() { done <- d.runRepositoryMaintenance(context.Background()) }() @@ -576,7 +576,7 @@ func TestMaintenanceRecoversCompactionBeforeRetention(t *testing.T) { Logs: []telemetry.Log{{Body: "old", IngestedAt: 1}}, Metrics: []telemetry.Metric{{Name: "old", IngestedAt: 1}}, } - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index de4363e1..5e83ccf9 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -2,6 +2,7 @@ package query import ( "context" + "fmt" "testing" "time" @@ -155,7 +156,7 @@ func TestSkipRollupToLatest(t *testing.T) { for i := range spans { spans[i] = telemetry.Span{Namespace: "default", TraceID: "backlog", SpanID: string(rune(i + 1)), ServiceName: "svc", Kind: "SPAN_KIND_CLIENT", StartUnixNanos: now - int64(i)*int64(time.Minute), DurationMS: 10, StatusCode: "STATUS_CODE_OK", IngestedAt: now} } - if err := repository.Commit(telemetrystore.Batch{ID: "skip-backlog", Spans: spans}); err != nil { + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "skip-backlog", Spans: spans}); err != nil { t.Fatalf("commit backlog: %v", err) } d, err := NewDuck(ctx, cfg, repository) @@ -231,7 +232,7 @@ ON CONFLICT (cache_key) DO UPDATE SET last_ingested_unix_nano = 1, updated_at = // rollupOnce picks it up — proves the watermark didn't over-advance and // swallow data that arrived after the skip. liveTime := time.Now().Add(time.Millisecond).UnixNano() - if err := repository.Commit(telemetrystore.Batch{ID: "skip-live", Spans: []telemetry.Span{{Namespace: "default", TraceID: "tr-live-1", SpanID: "sp-live-1", ServiceName: "svc-live", Kind: "SPAN_KIND_SERVER", StartUnixNanos: liveTime, DurationMS: 5, StatusCode: "STATUS_CODE_OK", IngestedAt: liveTime}}}); err != nil { + if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "skip-live", Spans: []telemetry.Span{{Namespace: "default", TraceID: "tr-live-1", SpanID: "sp-live-1", ServiceName: "svc-live", Kind: "SPAN_KIND_SERVER", StartUnixNanos: liveTime, DurationMS: 5, StatusCode: "STATUS_CODE_OK", IngestedAt: liveTime}}}); err != nil { t.Fatalf("commit live span: %v", err) } @@ -243,3 +244,135 @@ ON CONFLICT (cache_key) DO UPDATE SET last_ingested_unix_nano = 1, updated_at = t.Error("rollupOnce processed 0 rows for live span after skip-to-latest; watermark may have over-advanced") } } + +// readEdgeRollupState returns the edge rollup's persisted watermark and its +// sub-window resume point. +func readEdgeRollupState(t *testing.T, d *Duck) (watermark, cursor int64) { + t.Helper() + ctx := context.Background() + tx, err := d.DB.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + if watermark, err = rollupWatermark(ctx, tx, edgeRollupStateKey); err != nil { + t.Fatal(err) + } + if cursor, err = rollupWatermark(ctx, tx, edgeRollupSubCursorKey); err != nil { + t.Fatal(err) + } + return watermark, cursor +} + +// TestEdgeRollupBoundsSubWindowsPerPass pins the invariant that one rollup pass +// holds the Parquet read gate for an amount of work fixed by configuration, +// never by the shape of the data. +// +// Chunking ingested time does not provide that bound: a seed load or backfill +// compresses a wide start_time range into a narrow ingested window, so a single +// one-hour chunk can expand into an unbounded number of sub-windows and hold +// the gate for as long as that takes — starving every retention and compaction +// publication behind it. No timeout can fix that from the outside; cancelling +// mid-transaction only discards the work and retries it forever. The pass must +// instead stop at a fixed sub-window count, persist where to resume, and leave +// the ingested watermark behind until the window is finished. +func TestEdgeRollupBoundsSubWindowsPerPass(t *testing.T) { + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatalf("CreateTables: %v", err) + } + if err := CreateViews(db); err != nil { + t.Fatalf("CreateViews: %v", err) + } + d := &Duck{DB: db, cfg: config.Config{RetentionDays: 30, DuckDBMemory: "1GB"}} + ctx := context.Background() + + // One ingested instant, but start_time spread over 600 minutes: 20 + // sub-windows of edgeStartChunkNanos, well past maxEdgeSubWindowsPerPass. + const ( + rows = 12_000 + spreadMins = 600 + wantWindows = spreadMins / 30 + ) + baseTime := time.Now().UTC().Truncate(time.Minute) + ingestedNano := baseTime.UnixNano() + for _, spec := range []struct{ kind, span, parent, service string }{ + {"SPAN_KIND_SERVER", "parent-%d", "", "svc-a"}, + {"SPAN_KIND_CLIENT", "child-%d", "parent-%d", "svc-b"}, + } { + parent := "NULL" + if spec.parent != "" { + parent = fmt.Sprintf("printf('%s', i)", spec.parent) + } + _, err := db.ExecContext(ctx, fmt.Sprintf(` +WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) +INSERT INTO telemetry.spans ( + namespace, trace_id, span_id, parent_span_id, service, operation, kind, + start_time, end_time, start_unix_nano, end_unix_nano, duration_ms, + status, ingested_at, ingested_unix_nano +) +SELECT 'default', printf('trace-%%d', i %% 2000), printf('%s', i), %s, '%s', 'op', '%s', + base_time - ((i %% %d) * INTERVAL '1' MINUTE), + base_time - ((i %% %d) * INTERVAL '1' MINUTE) + INTERVAL '10' MILLISECOND, + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)), + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)) + 10000000, + 10.0, 'STATUS_CODE_OK', base_time, ? +FROM range(?, ?) t(i), input`, + spec.span, parent, spec.service, spec.kind, spreadMins, spreadMins, spreadMins, spreadMins), + baseTime, ingestedNano, 0, rows) + if err != nil { + t.Fatalf("insert %s: %v", spec.kind, err) + } + } + + // The first pass must stop short and record where to resume, leaving the + // ingested watermark untouched so nothing in the window can be skipped. + if _, err := d.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("first pass: %v", err) + } + watermark, cursor := readEdgeRollupState(t, d) + if cursor == 0 { + t.Fatal("first pass consumed the whole window: the per-pass sub-window bound did not engage") + } + if watermark != 0 { + t.Fatalf("ingested watermark advanced to %d over an unfinished window", watermark) + } + + passes := 1 + for { + if passes > wantWindows+2 { + t.Fatal("edge rollup did not converge: passes are not making forward progress") + } + if _, err := d.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("pass %d: %v", passes+1, err) + } + passes++ + if _, cursor = readEdgeRollupState(t, d); cursor == 0 { + break + } + } + if passes < 2 { + t.Fatalf("converged in %d passes, want the work spread over several", passes) + } + if watermark, _ = readEdgeRollupState(t, d); watermark == 0 { + t.Fatal("ingested watermark never advanced after the window completed") + } + + // Resuming must neither drop nor double-count a bucket: the aggregate has + // to match what a single unbounded pass would have produced. + var edgeBuckets, spanBuckets int64 + if err := db.QueryRowContext(ctx, `SELECT count(DISTINCT bucket) FROM edge_rollup`).Scan(&edgeBuckets); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, ` +SELECT count(DISTINCT date_trunc('minute', start_time)) FROM telemetry.spans`).Scan(&spanBuckets); err != nil { + t.Fatal(err) + } + if spanBuckets != spreadMins { + t.Fatalf("spans cover %d minute buckets, want %d", spanBuckets, spreadMins) + } + if edgeBuckets != spanBuckets { + t.Fatalf("edge_rollup covers %d buckets, spans cover %d — resuming dropped or duplicated work", edgeBuckets, spanBuckets) + } + t.Logf("converged in %d passes over %d sub-windows; buckets edge=%d spans=%d", passes, wantWindows, edgeBuckets, spanBuckets) +} diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 3ddc1997..9945de1a 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -91,12 +91,6 @@ func (g *parquetReadGate) TryRLock() bool { return true } -func (g *parquetReadGate) RLock() { - if err := g.RLockContext(context.Background()); err != nil { - panic(err) - } -} - func (g *parquetReadGate) RLockContext(ctx context.Context) error { g.init() g.mu.Lock() @@ -129,12 +123,6 @@ func (g *parquetReadGate) RUnlock() { g.mu.Unlock() } -func (g *parquetReadGate) Lock() { - if err := g.LockContext(context.Background()); err != nil { - panic(err) - } -} - func (g *parquetReadGate) LockContext(ctx context.Context) error { g.init() g.mu.Lock() diff --git a/internal/query/parquet_gate_test.go b/internal/query/parquet_gate_test.go index 0ec51f13..db1c260f 100644 --- a/internal/query/parquet_gate_test.go +++ b/internal/query/parquet_gate_test.go @@ -23,7 +23,7 @@ func waitingParquetWriters(g *parquetReadGate) int { func TestParquetGateRemovesCanceledPublisher(t *testing.T) { var gate parquetReadGate - gate.RLock() + mustRLock(t, &gate) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() if err := gate.LockContext(ctx); !errors.Is(err, context.DeadlineExceeded) { @@ -79,7 +79,7 @@ func TestParquetGateAdmitsReadersWhileWriterGraceRuns(t *testing.T) { if !gate.TryRLock() { t.Fatal("first reader was not admitted") } - go gate.Lock() + go mustLock(gate) waitForQueuedWriter(t, gate) clock.Advance(defaultWriterGrace / 2) if !gate.TryRLock() { @@ -95,7 +95,7 @@ func TestParquetGateQueuesReadersOnceWriterGraceExpires(t *testing.T) { if !gate.TryRLock() { t.Fatal("first reader was not admitted") } - go gate.Lock() + go mustLock(gate) waitForQueuedWriter(t, gate) clock.Advance(defaultWriterGrace + time.Second) if gate.TryRLock() { @@ -109,10 +109,10 @@ func TestParquetGateQueuesReadersOnceWriterGraceExpires(t *testing.T) { func TestParquetGatePublishesAfterOverlappingReadersDrain(t *testing.T) { clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} gate := &parquetReadGate{now: clock.Now} - gate.RLock() + mustRLock(t, gate) published := make(chan struct{}) go func() { - gate.Lock() + mustLock(gate) close(published) gate.Unlock() }() @@ -134,12 +134,12 @@ func TestParquetGatePublishesAfterOverlappingReadersDrain(t *testing.T) { func TestParquetGateDistinguishesWritersQueuedAtSameInstant(t *testing.T) { clock := &fakeClock{now: time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)} gate := &parquetReadGate{now: clock.Now} - gate.RLock() + mustRLock(t, gate) acquired := make(chan struct{}, 2) release := make(chan struct{}, 2) for range 2 { go func() { - gate.Lock() + mustLock(gate) acquired <- struct{}{} <-release gate.Unlock() @@ -171,3 +171,16 @@ func TestParquetGateDistinguishesWritersQueuedAtSameInstant(t *testing.T) { } gate.RUnlock() } + +func mustRLock(t *testing.T, gate *parquetReadGate) { + t.Helper() + if err := gate.RLockContext(context.Background()); err != nil { + t.Fatalf("RLockContext: %v", err) + } +} + +func mustLock(gate *parquetReadGate) { + if err := gate.LockContext(context.Background()); err != nil { + panic(err) + } +} diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index b90abe44..af8c0d35 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/parquet-go/parquet-go" "github.com/parquet-go/parquet-go/compress/zstd" @@ -28,6 +29,12 @@ const ( maxTraceQueryResults = 500 ) +// commitPublishWait caps how long one ingest commit waits for the publish +// gate. It sits above the publisher's own swap budget so it only fires when a +// publication is genuinely stuck rather than merely slow, and it keeps a stuck +// publication from being indistinguishable from a hung ingest path. +const commitPublishWait = 60 * time.Second + var syncPublishedDirectory = syncDirectory type BatchMetadata struct { @@ -144,7 +151,14 @@ func (p *ParquetStore) RowCount() uint64 { return count } -func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error { +// CommitBatch publishes one atomic batch directory. ctx bounds the wait for +// the publish gate: ingest is the only caller that used to wait on it without +// a deadline, so a publication that stalled took the ingest path down with it +// and OTLP clients lost rows to their own timeouts. A commit that cannot get +// the gate now fails, retries, and is counted, instead of hanging. +func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error { + ctx, cancel := context.WithTimeout(ctx, commitPublishWait) + defer cancel() if err := validateBatchID(metadata.ID); err != nil { return err } @@ -155,14 +169,23 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] return nil } if info, err := os.Stat(final); err == nil && info.IsDir() { - if err := p.lockPublish(context.Background()); err != nil { + // Adopting a directory left by an interrupted commit means validating + // it — Parquet footers plus a full trace-index walk. That is done + // before taking the gate, so re-registering one batch cannot stall + // every other publisher and ingest worker behind it. + adopted, err := loadRegisteredBatch(final) + if err != nil { + return err + } + if err := p.lockPublish(ctx); err != nil { return err } defer p.unlockPublish() if err := syncDirectory(p.batchesDir); err != nil { return err } - return p.registerBatch(final) + p.installBatch(adopted) + return nil } else if err != nil && !errors.Is(err, os.ErrNotExist) { return err } @@ -243,7 +266,7 @@ func (p *ParquetStore) CommitBatch(metadata BatchMetadata, spans []Span, logs [] prepared.traces.path = filepath.Join(final, "trace.fidx") } - if err := p.lockPublish(context.Background()); err != nil { + if err := p.lockPublish(ctx); err != nil { return err } defer p.unlockPublish() @@ -533,10 +556,26 @@ func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(fu return err } defer p.unlockPublish() - p.mu.Lock() - defer p.mu.Unlock() + // p.mu is taken only to mutate the in-memory set, never across the + // renames and fsync. Holding it for filesystem work made it a second, + // undeadlined serialization point on the ingest path — CommitBatch's + // first action is a hasBatch read of this same mutex — and one + // invisible to the publish gate's accounting. The renames themselves + // need no additional exclusion: readers are already outside the + // snapshot for the length of this callback, and the publish gate + // serializes this against every other publisher. + p.mu.RLock() + planned := make(map[string]*storedBatch, len(candidates)) + for _, candidate := range candidates { + if batch, exists := p.batches[candidate.id]; exists { + planned[candidate.id] = batch + } + } + p.mu.RUnlock() + + retiredIDs := make([]string, 0, len(planned)) for _, candidate := range candidates { - batch, exists := p.batches[candidate.id] + batch, exists := planned[candidate.id] if !exists { continue } @@ -545,12 +584,17 @@ func (p *ParquetStore) PruneBefore(cutoff int64, maxBatches int, publish func(fu pruneErr = errors.Join(pruneErr, err) continue } - delete(p.batches, candidate.id) + retiredIDs = append(retiredIDs, candidate.id) retired = append(retired, path) } if len(retired) > 0 { pruneErr = errors.Join(pruneErr, syncDirectory(p.batchesDir)) } + p.mu.Lock() + for _, id := range retiredIDs { + delete(p.batches, id) + } + p.mu.Unlock() return nil }) pruneErr = errors.Join(pruneErr, err) @@ -842,17 +886,32 @@ func (p *ParquetStore) loadBatches() error { } func (p *ParquetStore) registerBatch(dir string) error { - batch, err := loadStoredBatch(dir) + batch, err := loadRegisteredBatch(dir) if err != nil { return err } + p.installBatch(batch) + return nil +} + +// loadRegisteredBatch validates a published directory and is safe to call +// outside the publish gate: it only reads files that are already immutable. +func loadRegisteredBatch(dir string) (*storedBatch, error) { + batch, err := loadStoredBatch(dir) + if err != nil { + return nil, err + } if filepath.Base(dir) != batch.metadata.ID+BatchSuffix { - return fmt.Errorf("parquet batch directory %q does not match metadata ID %q", filepath.Base(dir), batch.metadata.ID) + return nil, fmt.Errorf("parquet batch directory %q does not match metadata ID %q", filepath.Base(dir), batch.metadata.ID) } + return batch, nil +} + +// installBatch publishes a validated batch into the queryable set. +func (p *ParquetStore) installBatch(batch *storedBatch) { p.mu.Lock() p.batches[batch.metadata.ID] = batch p.mu.Unlock() - return nil } func loadStoredBatch(dir string) (*storedBatch, error) { diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index 10f2e14f..61cf4bc6 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -24,7 +24,7 @@ func TestParquetStorePublishesCompleteBatchAndRecovers(t *testing.T) { } span := completeTestSpan() metadata := BatchMetadata{ID: "batch-1", MinIngestedNanos: span.IngestedAt, MaxIngestedNanos: span.IngestedAt} - if err := store.CommitBatch(metadata, []Span{span}, []Log{{Namespace: "tenant", Body: "ready", TimeUnixNanos: 12}}, []Metric{{Namespace: "tenant", Name: "requests", TimeUnixNanos: 13}}); err != nil { + if err := store.CommitBatch(context.Background(), metadata, []Span{span}, []Log{{Namespace: "tenant", Body: "ready", TimeUnixNanos: 12}}, []Metric{{Namespace: "tenant", Name: "requests", TimeUnixNanos: 13}}); err != nil { t.Fatal(err) } @@ -37,7 +37,7 @@ func TestParquetStorePublishesCompleteBatchAndRecovers(t *testing.T) { if got := store.RowCount(); got != 3 { t.Fatalf("row count = %d, want 3", got) } - if err := store.CommitBatch(metadata, []Span{span}, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), metadata, []Span{span}, nil, nil); err != nil { t.Fatalf("idempotent commit: %v", err) } if got := store.RowCount(); got != 3 { @@ -93,7 +93,7 @@ func TestParquetStoreTraceUsesExactIDAndEventOrder(t *testing.T) { {TraceID: "other", SpanID: "other", StartUnixNanos: 20}, {TraceID: "wanted", SpanID: "earlier", StartUnixNanos: 10}, } - if err := store.CommitBatch(BatchMetadata{ID: "batch"}, spans, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "batch"}, spans, nil, nil); err != nil { t.Fatal(err) } got, err := traceAll(store, "wanted") @@ -125,7 +125,7 @@ func TestParquetStoreTraceFiltersAndBoundsResultsDuringRead(t *testing.T) { Span{Namespace: "staging", TraceID: "large-trace", SpanID: "wrong-namespace", StartUnixNanos: 11}, Span{Namespace: "prod", TraceID: "large-trace", SpanID: "outside-window", StartUnixNanos: 200}, ) - if err := store.CommitBatch(BatchMetadata{ID: "large"}, spans, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "large"}, spans, nil, nil); err != nil { t.Fatal(err) } @@ -159,7 +159,7 @@ func TestParquetStorePreservesCompleteLogAndMetricRows(t *testing.T) { ExemplarsJSON: []byte(`[{"trace_id":"trace"}]`), AttributesJSON: []byte(`{"route":"/pay"}`), ResourceJSON: []byte(`{"host":"one"}`), ScopeName: "scope", ScopeVersion: "1.2.3", IngestedAt: 24, } - if err := store.CommitBatch(BatchMetadata{ID: "complete-signals"}, nil, []Log{logRow}, []Metric{metricRow}); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "complete-signals"}, nil, []Log{logRow}, []Metric{metricRow}); err != nil { t.Fatal(err) } batchDir := store.BatchPath("complete-signals") @@ -184,7 +184,7 @@ func TestParquetStoreSkipsTraceIndexesOutsideTimeWindow(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.CommitBatch(BatchMetadata{ID: "old-traces"}, []Span{{ + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "old-traces"}, []Span{{ TraceID: "wanted", SpanID: "old", StartUnixNanos: 100, }}, nil, nil); err != nil { t.Fatal(err) @@ -227,7 +227,7 @@ func TestPruneReaderWaitDoesNotBlockCommit(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.CommitBatch(BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { t.Fatal(err) } entered := make(chan struct{}) @@ -248,7 +248,7 @@ func TestPruneReaderWaitDoesNotBlockCommit(t *testing.T) { } commitDone := make(chan error, 1) go func() { - commitDone <- store.CommitBatch(BatchMetadata{ID: "concurrent", MaxIngestedNanos: 3}, []Span{{TraceID: "trace-2"}}, nil, nil) + commitDone <- store.CommitBatch(context.Background(), BatchMetadata{ID: "concurrent", MaxIngestedNanos: 3}, []Span{{TraceID: "trace-2"}}, nil, nil) }() select { case err := <-commitDone: @@ -269,7 +269,7 @@ func TestPrunePublicationContextBoundsStorageLockWait(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.CommitBatch(BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "expired", MaxIngestedNanos: 1}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { t.Fatal(err) } if err := store.lockPublish(context.Background()); err != nil { @@ -296,11 +296,11 @@ func TestReplacementReaderWaitDoesNotBlockCommit(t *testing.T) { t.Fatal(err) } span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} - if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "input"}, span, nil, nil); err != nil { t.Fatal(err) } output := BatchMetadata{ID: "output"} - if err := store.CommitBatch(output, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), output, span, nil, nil); err != nil { t.Fatal(err) } entered := make(chan struct{}) @@ -321,7 +321,7 @@ func TestReplacementReaderWaitDoesNotBlockCommit(t *testing.T) { } commitDone := make(chan error, 1) go func() { - commitDone <- store.CommitBatch(BatchMetadata{ID: "concurrent"}, []Span{{TraceID: "trace-2"}}, nil, nil) + commitDone <- store.CommitBatch(context.Background(), BatchMetadata{ID: "concurrent"}, []Span{{TraceID: "trace-2"}}, nil, nil) }() select { case err := <-commitDone: @@ -343,11 +343,11 @@ func TestPublishReplacementKeepsOutputOnlyAfterSyncFailure(t *testing.T) { t.Fatal(err) } span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} - if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "input"}, span, nil, nil); err != nil { t.Fatal(err) } output := BatchMetadata{ID: "output"} - if err := store.CommitBatch(output, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), output, span, nil, nil); err != nil { t.Fatal(err) } originalSync := syncPublishedDirectory @@ -378,11 +378,11 @@ func TestPublishReplacementUnpublishesRecoveredOutputBeforeRetiringInputs(t *tes t.Fatal(err) } span := []Span{{TraceID: "trace", SpanID: "span", StartUnixNanos: 1}} - if err := store.CommitBatch(BatchMetadata{ID: "input"}, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "input"}, span, nil, nil); err != nil { t.Fatal(err) } output := BatchMetadata{ID: "output"} - if err := store.CommitBatch(output, span, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), output, span, nil, nil); err != nil { t.Fatal(err) } blockedRetired := filepath.Join(store.BatchesDir(), "input.retired-output") @@ -420,7 +420,7 @@ func TestParquetStoreRejectsUnsafeBatchID(t *testing.T) { t.Fatal(err) } for _, id := range []string{"", ".hidden", "../escape", "has space"} { - if err := store.CommitBatch(BatchMetadata{ID: id}, []Span{{TraceID: "t"}}, nil, nil); err == nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: id}, []Span{{TraceID: "t"}}, nil, nil); err == nil { t.Fatalf("CommitBatch accepted unsafe ID %q", id) } } @@ -432,7 +432,7 @@ func TestParquetStoreRejectsMetadataRowCountMismatchOnOpen(t *testing.T) { if err != nil { t.Fatal(err) } - if err := store.CommitBatch(BatchMetadata{ID: "mismatch"}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "mismatch"}, []Span{{TraceID: "trace"}}, nil, nil); err != nil { t.Fatal(err) } metadataPath := filepath.Join(store.BatchPath("mismatch"), "metadata.json") @@ -488,3 +488,36 @@ func readOneParquetRow[T any](t *testing.T, path string) T { } return rows[0] } + +// TestCommitBatchDoesNotBlockIndefinitelyOnStalledPublication pins the +// invariant that ingest is never held hostage by a publication. CommitBatch +// used to wait on the publish gate with context.Background(), so a publisher +// stuck inside the gate blocked every OTLP commit until it finished — the +// clients then dropped rows on their own timeouts, turning a latency problem +// into permanent data loss. +func TestCommitBatchDoesNotBlockIndefinitelyOnStalledPublication(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.lockPublish(context.Background()); err != nil { + t.Fatal(err) + } + defer store.unlockPublish() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + span := completeTestSpan() + metadata := BatchMetadata{ID: "batch-stalled", MinIngestedNanos: span.IngestedAt, MaxIngestedNanos: span.IngestedAt} + + done := make(chan error, 1) + go func() { done <- store.CommitBatch(ctx, metadata, []Span{span}, nil, nil) }() + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("commit against a stalled publication = %v, want context.DeadlineExceeded", err) + } + case <-time.After(10 * time.Second): + t.Fatal("commit blocked on a stalled publication instead of honouring its context") + } +} diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index a8632a99..97e97ca8 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "math" "os" "path/filepath" @@ -185,10 +186,31 @@ func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCo type parquetPublishFunc func(context.Context, func(context.Context) error) error +// RecoverParquet resolves a pending compaction marker. A marker that keeps +// failing is set aside after maxCompactionRecoveryAttempts so it stops gating +// the rest of maintenance; see quarantineCompactionMarker for why that +// preserves every input and output it touched. func (r *Repository) RecoverParquet(ctx context.Context, publisher ParquetPublisher) error { r.compactionMu.Lock() defer r.compactionMu.Unlock() - return r.recoverCompaction(ctx, publisher.PublishParquet) + err := r.recoverCompaction(ctx, publisher.PublishParquet) + if err == nil { + r.recoveryFailures = 0 + return nil + } + // A cancelled pass says nothing about the marker: shutdown and publication + // contention must not count toward giving up on it. + if ctx.Err() != nil { + return err + } + r.recoveryFailures++ + if r.recoveryFailures < maxCompactionRecoveryAttempts { + return err + } + slog.Error("Parquet compaction recovery failed repeatedly; setting marker aside", + "attempts", r.recoveryFailures, "err", err, "marker", quarantinedMarkerName) + r.recoveryFailures = 0 + return errors.Join(err, r.quarantineCompactionMarker()) } func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPublishFunc) error { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index e008e08d..235ce8ca 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -2,8 +2,10 @@ package store import ( "context" + "encoding/json" "errors" "fmt" + "log/slog" "math" "os" "path/filepath" @@ -24,11 +26,25 @@ type Batch struct { // Repository publishes self-contained Parquet batch directories. The // directory rename is the transaction and the filesystem is the catalog. type Repository struct { - root string - Parquet *telemetry.ParquetStore - compactionMu sync.Mutex + root string + Parquet *telemetry.ParquetStore + // compactionMu guards the compaction marker and the retired directories a + // pending marker may still need, along with recoveryFailures. + compactionMu sync.Mutex + recoveryFailures int } +// maxCompactionRecoveryAttempts bounds how many passes a marker may fail +// recovery before it is set aside. A marker that cannot be recovered gates +// retention, compaction, and retired-directory cleanup — correctly, since all +// three could destroy what a rollback needs — so without a give-up path one +// bad marker latches every form of maintenance off for the process lifetime +// and storage grows unreclaimed behind a healthy-looking probe. +const maxCompactionRecoveryAttempts = 3 + +// quarantinedMarkerName is the marker set aside by quarantineCompactionMarker. +const quarantinedMarkerName = "COMPACTION.json.failed" + func Open(root string) (*Repository, error) { if err := os.MkdirAll(root, 0o755); err != nil { return nil, err @@ -38,11 +54,23 @@ func Open(root string) (*Repository, error) { return nil, err } r := &Repository{root: root, Parquet: parquetStore} + // A marker that cannot be recovered must not keep the process from + // booting: refusing to start leaves the operator with no way to run the + // cleanup that would clear it, so the only recovery was a manual rm -rf of + // live storage. Set it aside instead and come up degraded. Nothing is + // deleted — the staged output and the retired inputs both survive — so the + // compaction can still be completed or rolled back by hand. + quarantined := false if err := r.recoverCompaction(context.Background(), func(ctx context.Context, publish func(context.Context) error) error { return publish(ctx) }); err != nil { - _ = r.Close() - return nil, fmt.Errorf("recover Parquet compaction: %w", err) + slog.Error("Parquet compaction recovery failed at open; setting marker aside", + "err", err, "marker", quarantinedMarkerName) + if quarantineErr := r.quarantineCompactionMarker(); quarantineErr != nil { + _ = r.Close() + return nil, errors.Join(fmt.Errorf("recover Parquet compaction: %w", err), quarantineErr) + } + quarantined = true } - if err := r.cleanupCompactionArtifacts(); err != nil { + if err := r.cleanupCompactionArtifacts(quarantined); err != nil { _ = r.Close() return nil, fmt.Errorf("clean Parquet compaction staging: %w", err) } @@ -53,9 +81,14 @@ func Open(root string) (*Repository, error) { return r, nil } -func (r *Repository) cleanupCompactionArtifacts() error { - if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { - return err +// cleanupCompactionArtifacts drops staging left by an interrupted compaction. +// preserveStage keeps the staged output for a marker that was set aside, which +// is the only copy of that compaction's merged rows. +func (r *Repository) cleanupCompactionArtifacts(preserveStage bool) error { + if !preserveStage { + if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { + return err + } } if err := os.Remove(filepath.Join(r.root, "COMPACTION.json.tmp")); err != nil && !errors.Is(err, os.ErrNotExist) { return err @@ -63,6 +96,24 @@ func (r *Repository) cleanupCompactionArtifacts() error { return syncDirectory(r.root) } +// quarantineCompactionMarker sets aside a marker whose recovery keeps failing +// so it stops gating retention, compaction, and retired-directory cleanup. +// +// Nothing is deleted: the staged output stays, and the retired inputs are +// protected from cleanup by protectedRetiredSuffixes, so the operator can +// still complete or roll back the compaction by hand. This is not the +// batch-level quarantine that was rejected — no authoritative telemetry is +// discarded, and no batch becomes unreadable that was readable before. +func (r *Repository) quarantineCompactionMarker() error { + if err := os.Rename(filepath.Join(r.root, "COMPACTION.json"), filepath.Join(r.root, quarantinedMarkerName)); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + return syncDirectory(r.root) +} + func (r *Repository) Close() error { return r.Parquet.Close() } // CleanupParquet removes retired inputs only while no retention or compaction @@ -74,6 +125,10 @@ func (r *Repository) CleanupParquet() error { } func (r *Repository) cleanupRetired() error { + protected, err := r.protectedRetiredSuffixes() + if err != nil { + return err + } entries, err := os.ReadDir(r.Parquet.BatchesDir()) if err != nil { return err @@ -85,6 +140,9 @@ func (r *Repository) cleanupRetired() error { if !entry.IsDir() || strings.HasSuffix(name, telemetry.BatchSuffix) || !strings.Contains(name, ".retired") { continue } + if protectedRetired(name, protected) { + continue + } if err := os.RemoveAll(filepath.Join(r.Parquet.BatchesDir(), name)); err != nil { cleanupErr = errors.Join(cleanupErr, err) } else { @@ -97,7 +155,40 @@ func (r *Repository) cleanupRetired() error { return cleanupErr } -func (r *Repository) Commit(batch Batch) error { +func protectedRetired(name string, protected map[string]bool) bool { + for suffix := range protected { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + +// protectedRetiredSuffixes names the retired-input sets that a live or +// set-aside compaction marker may still need in order to roll back. Deleting +// one of those is unrecoverable: the input is gone and its rows were never +// published under the replacement. An unreadable marker therefore fails +// closed rather than protecting nothing. +func (r *Repository) protectedRetiredSuffixes() (map[string]bool, error) { + protected := make(map[string]bool) + for _, name := range []string{"COMPACTION.json", quarantinedMarkerName} { + data, err := os.ReadFile(filepath.Join(r.root, name)) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + var marker compactionMarker + if err := json.Unmarshal(data, &marker); err != nil { + return nil, fmt.Errorf("read compaction marker %s: %w", name, err) + } + protected[".retired-"+marker.Output.ID] = true + } + return protected, nil +} + +func (r *Repository) Commit(ctx context.Context, batch Batch) error { normalizeBatch(&batch) if err := validateBatch(batch); err != nil { return err @@ -105,11 +196,7 @@ func (r *Repository) Commit(batch Batch) error { metadata := telemetry.BatchMetadata{ ID: batch.ID, MinIngestedNanos: batchMinIngestedNanos(batch), MaxIngestedNanos: batchMaxIngestedNanos(batch), } - return r.Parquet.CommitBatch(metadata, batch.Spans, batch.Logs, batch.Metrics) -} - -func (r *Repository) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) { - return r.Parquet.Trace(ctx, query) + return r.Parquet.CommitBatch(ctx, metadata, batch.Spans, batch.Logs, batch.Metrics) } func (r *Repository) RowCount() uint64 { return r.Parquet.RowCount() } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 22020348..8ad199aa 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -75,10 +75,10 @@ func TestRepositoryCommitIsIdempotentDurableAndQueryable(t *testing.T) { t.Fatal(err) } batch := testBatch() - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatalf("idempotent commit: %v", err) } if got := repository.RowCount(); got != 3 { @@ -127,7 +127,7 @@ func TestRepositoryCommitsOversizedRequestAsOneBatch(t *testing.T) { StartUnixNanos: int64(i + 1), EndUnixNanos: int64(i + 2), IngestedAt: 1, } } - if err := repository.Commit(Batch{ID: "oversized", Spans: spans}); err != nil { + if err := repository.Commit(context.Background(), Batch{ID: "oversized", Spans: spans}); err != nil { t.Fatal(err) } metadata := repository.Parquet.BatchMetadata() @@ -143,7 +143,7 @@ func TestRepositoryHasOneAuthoritativeStorageLayout(t *testing.T) { t.Fatal(err) } defer repository.Close() - if err := repository.Commit(testBatch()); err != nil { + if err := repository.Commit(context.Background(), testBatch()); err != nil { t.Fatal(err) } for _, removed := range []string{"wal", "hot", "MANIFEST.json", "MANIFEST.log", "ducklake.sqlite"} { @@ -198,10 +198,10 @@ func TestRepositoryPrunesOnlyExpiredBatches(t *testing.T) { newer.Spans[0].StartUnixNanos, newer.Spans[0].IngestedAt = 1_000, 1_000 newer.Logs[0].TimeUnixNanos, newer.Logs[0].IngestedAt = 1_000, 1_000 newer.Metrics[0].TimeUnixNanos, newer.Metrics[0].IngestedAt = 1_000, 1_000 - if err := repository.Commit(old); err != nil { + if err := repository.Commit(context.Background(), old); err != nil { t.Fatal(err) } - if err := repository.Commit(newer); err != nil { + if err := repository.Commit(context.Background(), newer); err != nil { t.Fatal(err) } publisher := &testParquetCompactor{afterSwap: func() error { @@ -241,7 +241,7 @@ func TestRepositoryDiscardsRecoverableMarkerWithoutOutput(t *testing.T) { } batch := testBatch() batch.ID = "intact-input" - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } marker := compactionMarker{ @@ -285,7 +285,7 @@ func TestRepositoryRestoresInputsThroughPublicationGate(t *testing.T) { defer repository.Close() batch := testBatch() batch.ID = "retired-input" - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } marker := compactionMarker{ @@ -342,7 +342,7 @@ func TestRepositoryPrunePassDrainsWithinBudgetAndOldestFirst(t *testing.T) { batch.Spans[0].IngestedAt = int64(i + 1) batch.Logs[0].IngestedAt = int64(i + 1) batch.Metrics[0].IngestedAt = int64(i + 1) - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -376,7 +376,7 @@ func TestRepositoryPrunePassFinishesPublicationAfterBudget(t *testing.T) { batch.Spans[0].IngestedAt = int64(i + 1) batch.Logs[0].IngestedAt = int64(i + 1) batch.Metrics[0].IngestedAt = int64(i + 1) - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -405,7 +405,7 @@ func TestRepositoryCleanupOwnsRetiredDirectories(t *testing.T) { if err := os.Mkdir(retired, 0o755); err != nil { t.Fatal(err) } - if err := repository.Commit(Batch{ID: "contains.retired", Spans: []telemetry.Span{{TraceID: "trace"}}}); err != nil { + if err := repository.Commit(context.Background(), Batch{ID: "contains.retired", Spans: []telemetry.Span{{TraceID: "trace"}}}); err != nil { t.Fatal(err) } if err := repository.CleanupParquet(); err != nil { @@ -433,7 +433,7 @@ func TestRepositoryRetentionUsesIngestTimeNotEventTime(t *testing.T) { batch.Spans[0].IngestedAt = 100 batch.Logs[0].IngestedAt = 100 batch.Metrics[0].IngestedAt = 100 - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } removed, err := repository.PruneParquet(context.Background(), &testParquetCompactor{}, 500, 64) @@ -454,7 +454,7 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { batch.ID = fmt.Sprintf("batch-%d", i) batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) batch.Spans[0].StartUnixNanos = int64(100 + i) - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -513,7 +513,7 @@ func TestRepositoryRecoversPendingCompactionWithoutRestart(t *testing.T) { batch := testBatch() batch.ID = fmt.Sprintf("pending-%d", i) batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -548,7 +548,7 @@ func TestRepositoryRecoversInterruptedCompactionSwap(t *testing.T) { batch := testBatch() batch.ID = fmt.Sprintf("recover-%d", i) batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) - if err := repository.Commit(batch); err != nil { + if err := repository.Commit(context.Background(), batch); err != nil { t.Fatal(err) } } @@ -626,7 +626,178 @@ func openTestDuckDB(t *testing.T) *sql.DB { } func traceAll(repository *Repository, traceID string) ([]telemetry.IndexedSpan, error) { - return repository.Trace(context.Background(), telemetry.TraceQuery{ + return repository.Parquet.Trace(context.Background(), telemetry.TraceQuery{ TraceID: traceID, StartNanos: -1 << 63, EndNanos: 1<<63 - 1, Limit: 500, }) } + +// seedFailedCompaction leaves a live COMPACTION.json whose recovery always +// fails, which is the state that used to latch maintenance off permanently. +func seedFailedCompaction(t *testing.T, dir string, repository *Repository) { + t.Helper() + for i := range minCompactionInputs { + batch := testBatch() + batch.ID = fmt.Sprintf("latched-%d", i) + batch.Spans[0].SpanID = fmt.Sprintf("span-%d", i) + if err := repository.Commit(context.Background(), batch); err != nil { + t.Fatal(err) + } + } + db := openTestDuckDB(t) + t.Cleanup(func() { db.Close() }) + compactor := &testParquetCompactor{db: db, publishErr: errors.New("publication unavailable")} + if _, err := repository.CompactParquet(context.Background(), compactor, 64); err == nil { + t.Fatal("compaction succeeded despite publication failure") + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("expected a live marker: %v", err) + } +} + +// TestRepositorySetsAsideUnrecoverableMarker pins the give-up path. A marker +// that cannot be recovered correctly gates retention, compaction, and retired +// cleanup — all three can destroy what a rollback needs — so without a bound on +// how long it may do so, one bad marker disables every form of maintenance for +// the process lifetime while storage grows behind a healthy-looking probe. +func TestRepositorySetsAsideUnrecoverableMarker(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + seedFailedCompaction(t, dir, repository) + + failing := testParquetPublisherFunc(func(context.Context, func(context.Context) error) error { + return errors.New("publication unavailable") + }) + for attempt := 1; attempt < maxCompactionRecoveryAttempts; attempt++ { + if err := repository.RecoverParquet(context.Background(), failing); err == nil { + t.Fatalf("attempt %d: recovery reported success", attempt) + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("attempt %d: marker set aside too early: %v", attempt, err) + } + } + if err := repository.RecoverParquet(context.Background(), failing); err == nil { + t.Fatal("final attempt reported success") + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { + t.Fatalf("marker still live after %d failures: %v", maxCompactionRecoveryAttempts, err) + } + if _, err := os.Stat(filepath.Join(dir, quarantinedMarkerName)); err != nil { + t.Fatalf("marker was not preserved for the operator: %v", err) + } + // Maintenance is unblocked: recovery is a no-op now, so a later pass runs. + if err := repository.RecoverParquet(context.Background(), failing); err != nil { + t.Fatalf("maintenance still gated after the marker was set aside: %v", err) + } +} + +// TestRepositoryCancelledRecoveryDoesNotCountTowardGivingUp keeps shutdown and +// publication contention from being mistaken for a bad marker. +func TestRepositoryCancelledRecoveryDoesNotCountTowardGivingUp(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + seedFailedCompaction(t, dir, repository) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + failing := testParquetPublisherFunc(func(ctx context.Context, _ func(context.Context) error) error { + return ctx.Err() + }) + for range maxCompactionRecoveryAttempts * 2 { + if err := repository.RecoverParquet(cancelled, failing); err == nil { + t.Fatal("cancelled recovery reported success") + } + } + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("cancelled attempts set the marker aside: %v", err) + } +} + +// TestRepositoryCleanupPreservesMarkerRollbackSet pins that cleanup never +// deletes the retired inputs a pending marker still needs. Deleting one is +// unrecoverable: the input is gone and its rows were never published under the +// replacement, so the rows exist in no queryable batch at all. +func TestRepositoryCleanupPreservesMarkerRollbackSet(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + + marker := compactionMarker{ + Output: telemetry.BatchMetadata{ID: "compact-live", Generation: 1}, + Inputs: []string{"input-a"}, + } + data, err := json.Marshal(marker) + if err != nil { + t.Fatal(err) + } + if err := writeDurableFile(filepath.Join(dir, "COMPACTION.json"), data); err != nil { + t.Fatal(err) + } + batches := repository.Parquet.BatchesDir() + needed := filepath.Join(batches, "input-a.retired-compact-live") + stale := filepath.Join(batches, "input-b.retired") + for _, path := range []string{needed, stale} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + } + + if err := repository.CleanupParquet(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(needed); err != nil { + t.Fatalf("cleanup deleted the rollback set of a live marker: %v", err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("cleanup left an unreferenced retired directory: %v", err) + } +} + +// TestRepositoryOpensDespiteUnrecoverableMarker pins that a bad marker cannot +// stop the process from booting. Failing Open left the operator with no way to +// run the cleanup that would clear it, so the only recovery was a manual +// rm -rf of live storage. +func TestRepositoryOpensDespiteUnrecoverableMarker(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + seedFailedCompaction(t, dir, repository) + if err := repository.Close(); err != nil { + t.Fatal(err) + } + // Corrupt the staged output so recovery cannot complete or roll back. + stage := filepath.Join(dir, "compaction") + entries, err := os.ReadDir(stage) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if err := os.WriteFile(filepath.Join(stage, entry.Name(), "metadata.json"), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + } + + reopened, err := Open(dir) + if err != nil { + t.Fatalf("Open refused to boot with an unrecoverable marker: %v", err) + } + defer reopened.Close() + if _, err := os.Stat(filepath.Join(dir, quarantinedMarkerName)); err != nil { + t.Fatalf("marker was not set aside at open: %v", err) + } + if _, err := os.Stat(stage); err != nil { + t.Fatalf("staged output was destroyed rather than preserved: %v", err) + } +} diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index c2a53e01..45045f87 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -23,7 +23,7 @@ const ( ) type batchCommitter interface { - Commit(Batch) error + Commit(context.Context, Batch) error } type Writer struct { @@ -230,7 +230,7 @@ func (w *Writer) commitJob(ctx context.Context, job commitJob) error { started := time.Now() var lastErr error for attempt := 0; attempt < commitRetryLimit; attempt++ { - if err := w.repository.Commit(batch); err == nil { + if err := w.repository.Commit(ctx, batch); err == nil { lastErr = nil break } else { diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index 0cb48f67..10eb5ad5 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -21,7 +21,7 @@ type recordingCommitter struct { batches []Batch } -func (c *recordingCommitter) Commit(batch Batch) error { +func (c *recordingCommitter) Commit(_ context.Context, batch Batch) error { c.mu.Lock() defer c.mu.Unlock() c.calls++ @@ -45,7 +45,7 @@ type blockingCommitter struct { once sync.Once } -func (c *blockingCommitter) Commit(Batch) error { +func (c *blockingCommitter) Commit(context.Context, Batch) error { c.once.Do(func() { close(c.entered) }) <-c.release return nil @@ -230,7 +230,7 @@ type parallelCommitter struct { once sync.Once } -func (c *parallelCommitter) Commit(Batch) error { +func (c *parallelCommitter) Commit(context.Context, Batch) error { c.mu.Lock() c.active++ c.maxActive = max(c.maxActive, c.active) From 426d4fe1d6290304c11354075c9881d369cf5ad8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 27 Aug 2026 20:25:25 -0700 Subject: [PATCH 21/31] fix(storage): stop set-aside markers from bricking startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting a failed compaction marker aside was supposed to end the failure, but two paths kept it going. protectedRetiredSuffixes returned an error when a marker would not parse. The set-aside marker stays on disk by design, so cleanupRetired failed on it during every subsequent Open and the process exited on each boot — precisely the outcome that setting it aside exists to prevent, and worse than the latch it replaced because it survives restarts. An unreadable marker names no rollback set, so it now retains every retired directory and logs, deleting nothing. cleanupCompactionArtifacts decided whether to keep the staged output from a boolean set on the boot that performed the quarantine. On the next boot the live marker was gone, the flag was false, and the staged output was removed — leaving an operator able to roll the compaction back but never to complete it, though the doc comment promised both. It now keys off the presence of the set-aside marker, so the stage survives as long as the marker does. PublishReplacement held p.mu across the output move, up to 64 input renames, and an fsync. CommitBatch's first action reads that same mutex, so this was the ingest-blocking pattern PruneBefore's comment already documents as forbidden; p.mu now covers only the in-memory map mutations. CommitBatch no longer leaks its staging directory when the publish rename loses to an existing final directory. selectCompactionBatches applies the same MaxIngestedNanos > 0 guard as the counting loop, so a batch with no ingest timestamp cannot be pulled into a group it was never counted in. Adds regression tests for repeated boots against an unreadable marker and for the staged output surviving every boot the marker survives. --- internal/telemetry/parquet.go | 33 +++++++--- internal/telemetry/store/compaction.go | 3 + internal/telemetry/store/repository.go | 46 +++++++++----- internal/telemetry/store/repository_test.go | 67 +++++++++++++++++++++ 4 files changed, 126 insertions(+), 23 deletions(-) diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index af8c0d35..052be804 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -277,10 +277,13 @@ func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, if err := os.Rename(stage, final); err != nil { if info, statErr := os.Stat(final); statErr == nil && info.IsDir() { complete = true + // The staged copy lost the race and is now unreferenced; drop it + // rather than leaving it to accumulate until the next open. + removeErr := os.RemoveAll(stage) if err := syncDirectory(p.batchesDir); err != nil { - return err + return errors.Join(err, removeErr) } - return p.registerBatch(final) + return errors.Join(p.registerBatch(final), removeErr) } return fmt.Errorf("publish Parquet batch: %w", err) } @@ -763,9 +766,14 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, return err } defer p.unlockPublish() - p.mu.Lock() - defer p.mu.Unlock() + // p.mu covers only the in-memory set, never the renames and fsync + // below. CommitBatch's first action reads this same mutex, so holding + // it across filesystem work would put ingest behind I/O that the + // publish gate cannot see or bound — the reason PruneBefore keeps its + // own critical section down to the map mutation. installReplacement := func() { + p.mu.Lock() + defer p.mu.Unlock() for _, id := range inputs { delete(p.batches, id) } @@ -776,14 +784,21 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, for i := len(retired) - 1; i >= 0; i-- { rollbackErr = errors.Join(rollbackErr, os.Rename(retired[i][1], retired[i][0])) } - delete(p.batches, metadata.ID) + restore := make(map[string]*storedBatch, len(inputBatches)) for id, batch := range inputBatches { if _, err := os.Stat(batch.dir); err == nil { - p.batches[id] = batch - } else { - delete(p.batches, id) + restore[id] = batch } } + p.mu.Lock() + delete(p.batches, metadata.ID) + for id := range inputBatches { + delete(p.batches, id) + } + for id, batch := range restore { + p.batches[id] = batch + } + p.mu.Unlock() return errors.Join(rollbackErr, syncDirectory(p.batchesDir)) } // Recovery may resume with the output already published. Move it back @@ -797,7 +812,9 @@ func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, return err } source = stage + p.mu.Lock() delete(p.batches, metadata.ID) + p.mu.Unlock() } for _, id := range inputs { active := p.BatchPath(id) diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 97e97ca8..c9bb6719 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -156,6 +156,9 @@ func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) } selected := make([]telemetry.BatchMetadata, 0, min(maxBatches, counts[chosen])) for _, batch := range batches { + if batch.MaxIngestedNanos <= 0 { + continue + } if (compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation}) == chosen { selected = append(selected, batch) if len(selected) == maxBatches { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 235ce8ca..b0d5efeb 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -60,7 +60,6 @@ func Open(root string) (*Repository, error) { // live storage. Set it aside instead and come up degraded. Nothing is // deleted — the staged output and the retired inputs both survive — so the // compaction can still be completed or rolled back by hand. - quarantined := false if err := r.recoverCompaction(context.Background(), func(ctx context.Context, publish func(context.Context) error) error { return publish(ctx) }); err != nil { slog.Error("Parquet compaction recovery failed at open; setting marker aside", "err", err, "marker", quarantinedMarkerName) @@ -68,9 +67,8 @@ func Open(root string) (*Repository, error) { _ = r.Close() return nil, errors.Join(fmt.Errorf("recover Parquet compaction: %w", err), quarantineErr) } - quarantined = true } - if err := r.cleanupCompactionArtifacts(quarantined); err != nil { + if err := r.cleanupCompactionArtifacts(); err != nil { _ = r.Close() return nil, fmt.Errorf("clean Parquet compaction staging: %w", err) } @@ -82,10 +80,18 @@ func Open(root string) (*Repository, error) { } // cleanupCompactionArtifacts drops staging left by an interrupted compaction. -// preserveStage keeps the staged output for a marker that was set aside, which -// is the only copy of that compaction's merged rows. -func (r *Repository) cleanupCompactionArtifacts(preserveStage bool) error { - if !preserveStage { +// +// A set-aside marker keeps its staged output: that directory holds the only +// copy of the compaction's merged rows, and it has to survive every boot the +// marker survives, not just the one that set it aside — otherwise the promise +// that an operator can still complete the compaction by hand lasts exactly one +// restart. +func (r *Repository) cleanupCompactionArtifacts() error { + setAside, err := pathExists(filepath.Join(r.root, quarantinedMarkerName)) + if err != nil { + return err + } + if !setAside { if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { return err } @@ -125,10 +131,13 @@ func (r *Repository) CleanupParquet() error { } func (r *Repository) cleanupRetired() error { - protected, err := r.protectedRetiredSuffixes() + protected, protectAll, err := r.protectedRetiredSuffixes() if err != nil { return err } + if protectAll { + return nil + } entries, err := os.ReadDir(r.Parquet.BatchesDir()) if err != nil { return err @@ -167,25 +176,32 @@ func protectedRetired(name string, protected map[string]bool) bool { // protectedRetiredSuffixes names the retired-input sets that a live or // set-aside compaction marker may still need in order to roll back. Deleting // one of those is unrecoverable: the input is gone and its rows were never -// published under the replacement. An unreadable marker therefore fails -// closed rather than protecting nothing. -func (r *Repository) protectedRetiredSuffixes() (map[string]bool, error) { - protected := make(map[string]bool) +// published under the replacement. +// +// A marker whose contents cannot be parsed names nothing, so protectAll tells +// the caller to delete no retired directory at all. Returning an error instead +// would fail Open on every boot — the marker is already set aside and stays on +// disk, so the failure repeats forever and cleanup can never run, which is the +// outcome setting it aside exists to avoid. +func (r *Repository) protectedRetiredSuffixes() (protected map[string]bool, protectAll bool, err error) { + protected = make(map[string]bool) for _, name := range []string{"COMPACTION.json", quarantinedMarkerName} { data, err := os.ReadFile(filepath.Join(r.root, name)) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { - return nil, err + return nil, false, err } var marker compactionMarker if err := json.Unmarshal(data, &marker); err != nil { - return nil, fmt.Errorf("read compaction marker %s: %w", name, err) + slog.Error("compaction marker is unreadable; retaining every retired batch", + "marker", name, "err", err) + return nil, true, nil } protected[".retired-"+marker.Output.ID] = true } - return protected, nil + return protected, false, nil } func (r *Repository) Commit(ctx context.Context, batch Batch) error { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 8ad199aa..b17dbad2 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -801,3 +801,70 @@ func TestRepositoryOpensDespiteUnrecoverableMarker(t *testing.T) { t.Fatalf("staged output was destroyed rather than preserved: %v", err) } } + +// TestRepositoryBootsRepeatedlyWithUnreadableMarker pins that setting a marker +// aside actually ends the failure. The set-aside marker stays on disk, so any +// boot path that errors on parsing it fails identically forever — turning the +// mechanism meant to keep one bad marker from bricking the instance into the +// thing that bricks it. +func TestRepositoryBootsRepeatedlyWithUnreadableMarker(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "COMPACTION.json"), []byte("not json"), 0o644); err != nil { + t.Fatal(err) + } + for boot := 1; boot <= 3; boot++ { + reopened, err := Open(dir) + if err != nil { + t.Fatalf("boot %d refused to start with an unreadable marker: %v", boot, err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } + } +} + +// TestRepositoryKeepsSetAsideCompactionStageAcrossBoots pins that the staged +// output outlives the boot that set the marker aside. It holds the only copy of +// that compaction's merged rows, so deleting it on the next restart leaves the +// operator able to roll back but never to complete. +func TestRepositoryKeepsSetAsideCompactionStageAcrossBoots(t *testing.T) { + dir := t.TempDir() + repository, err := Open(dir) + if err != nil { + t.Fatal(err) + } + seedFailedCompaction(t, dir, repository) + if err := repository.Close(); err != nil { + t.Fatal(err) + } + stage := filepath.Join(dir, "compaction") + entries, err := os.ReadDir(stage) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if err := os.WriteFile(filepath.Join(stage, entry.Name(), "metadata.json"), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + } + + for boot := 1; boot <= 3; boot++ { + reopened, err := Open(dir) + if err != nil { + t.Fatalf("boot %d: %v", boot, err) + } + if err := reopened.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage); err != nil { + t.Fatalf("boot %d deleted the set-aside compaction's staged output: %v", boot, err) + } + } +} From a70592e422bd60936e6552da5764930c56952ddb Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:09:38 -0700 Subject: [PATCH 22/31] fix(storage): harden recovery and publication Keep compaction recovery fail-closed instead of bypassing a live marker. Bound only publish admission and edge-rollup work, and read stats under a stable Parquet namespace. --- internal/query/duck.go | 63 +++++++- internal/query/duck_test.go | 27 ++++ internal/query/edge_backlog_test.go | 47 ++++++ internal/telemetry/parquet.go | 21 +-- internal/telemetry/store/compaction.go | 58 +++++--- internal/telemetry/store/repository.go | 155 +++++++------------- internal/telemetry/store/repository_test.go | 129 +++++----------- 7 files changed, 266 insertions(+), 234 deletions(-) diff --git a/internal/query/duck.go b/internal/query/duck.go index cfeefda6..28398059 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -377,7 +377,7 @@ func (d *Duck) RunRollups(ctx context.Context) { } else if rows > 0 { slog.Info("startup rollup complete", "rows", rows, "duration", time.Since(start)) } - d.updateParquetStats() + d.updateParquetStats(ctx) maintenanceDone := make(chan struct{}) go func() { defer close(maintenanceDone) @@ -396,7 +396,7 @@ func (d *Duck) RunRollups(ctx context.Context) { if err != nil { slog.Error("rollup failed", "component", "rollup", "rows", rows, "err", err) } - d.updateParquetStats() + d.updateParquetStats(ctx) case <-ctx.Done(): return } @@ -404,7 +404,12 @@ func (d *Duck) RunRollups(ctx context.Context) { } // updateParquetStats refreshes the per-signal file-count and byte-size gauges. -func (d *Duck) updateParquetStats() { +func (d *Duck) updateParquetStats(ctx context.Context) { + if err := d.lockParquetRead(ctx); err != nil { + slog.Warn("parquet stats skipped", "err", err) + return + } + defer d.parquetMu.RUnlock() stats, err := d.repository.Parquet.Stats() if err != nil { slog.Warn("parquet stats failed", "err", err) @@ -449,7 +454,7 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { if err := d.runRepositoryMaintenance(ctx); err != nil && ctx.Err() == nil { slog.Warn("telemetry maintenance failed", "err", err) } - d.updateParquetStats() + d.updateParquetStats(ctx) } run() ticker := time.NewTicker(every) @@ -906,7 +911,10 @@ WHERE ingested_unix_nano > ? nextCursor = subLo.UnixNano() break } - subHi := subLo.Add(time.Duration(edgeStartChunkNanos)) + subHi, err := edgeSubWindowEnd(ctx, tx, windowStart, windowEnd, subLo, maxT, maxEdgeSpansPerSubWindow) + if err != nil { + return 0, err + } if _, err := tx.ExecContext(ctx, edgeRollupDeleteSQL, windowStart, windowEnd, subLo, subHi); err != nil { return 0, err } @@ -983,6 +991,12 @@ const rollupChunkNanos = int64(time.Hour) // (catch-up/bulk-load) can't exhaust memory. The pass loops over sub-windows. const edgeStartChunkNanos = int64(30 * time.Minute) +// maxEdgeSpansPerSubWindow makes the start-time window adaptive under dense +// ingest. The one-minute rollup bucket is the indivisible lower bound: a hot +// bucket above this limit still runs as one correct aggregate instead of being +// split into partial results. +const maxEdgeSpansPerSubWindow int64 = 250_000 + // maxEdgeSubWindowsPerPass bounds how many start_time sub-windows one pass // processes, and with it how long that pass holds the Parquet read gate. // @@ -992,11 +1006,44 @@ const edgeStartChunkNanos = int64(30 * time.Minute) // the gate for as long as that takes. No timeout can bound that from outside — // cancelling mid-transaction only discards the work and retries it forever. So // the pass instead stops at a fixed number of sub-windows, persists where to -// resume, and leaves the ingested watermark where it was. Hold time is then a -// function of this constant rather than of the dataset's shape, and every pass -// makes durable forward progress. +// resume, and leaves the ingested watermark where it was. Combined with the +// adaptive row limit above, every pass makes bounded forward progress without +// assuming a uniform event rate. const maxEdgeSubWindowsPerPass = 8 +func edgeSubWindowEnd(ctx context.Context, tx *sql.Tx, windowStart, windowEnd int64, subLo, maxT time.Time, rowLimit int64) (time.Time, error) { + maxEnd := maxT.Add(time.Minute) + subHi := subLo.Add(time.Duration(edgeStartChunkNanos)) + if subHi.After(maxEnd) { + subHi = maxEnd + } + for rowLimit > 0 && subHi.Sub(subLo) > time.Minute { + var rows int64 + if err := tx.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM ( + SELECT 1 + FROM spans + WHERE ingested_unix_nano > ? + AND ingested_unix_nano <= ? + AND start_time >= ? + AND start_time < ? + LIMIT ? +)`, windowStart, windowEnd, subLo, subHi, rowLimit+1).Scan(&rows); err != nil { + return time.Time{}, err + } + if rows <= rowLimit { + break + } + minutes := int64(subHi.Sub(subLo) / time.Minute / 2) + if minutes < 1 { + minutes = 1 + } + subHi = subLo.Add(time.Duration(minutes) * time.Minute) + } + return subHi, nil +} + // rollupWindow bounds one pass's scan to (start, end]. start falls back to // just before the oldest ingested row when there's no stored watermark, so a // first pass doesn't open the window at the epoch. diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index c2a7b228..9795c4d7 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -447,6 +447,33 @@ func TestIndexedTraceReadHonorsParquetGateContext(t *testing.T) { } } +func TestParquetStatsWaitForStableNamespace(t *testing.T) { + repository, err := telemetrystore.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d := &Duck{repository: repository} + mustLock(&d.parquetMu) + done := make(chan struct{}) + go func() { + d.updateParquetStats(context.Background()) + close(done) + }() + select { + case <-done: + d.parquetMu.Unlock() + t.Fatal("Parquet stats read while the batch namespace was changing") + case <-time.After(25 * time.Millisecond): + } + d.parquetMu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Parquet stats did not resume after publication") + } +} + func TestRepositoryPublicationDoesNotWaitForDuckDBWrites(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { diff --git a/internal/query/edge_backlog_test.go b/internal/query/edge_backlog_test.go index 5e83ccf9..065747d4 100644 --- a/internal/query/edge_backlog_test.go +++ b/internal/query/edge_backlog_test.go @@ -376,3 +376,50 @@ SELECT count(DISTINCT date_trunc('minute', start_time)) FROM telemetry.spans`).S } t.Logf("converged in %d passes over %d sub-windows; buckets edge=%d spans=%d", passes, wantWindows, edgeBuckets, spanBuckets) } + +func TestEdgeSubWindowShrinksForDenseIngest(t *testing.T) { + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatal(err) + } + if err := CreateViews(db); err != nil { + t.Fatal(err) + } + ctx := context.Background() + base := time.Now().UTC().Truncate(time.Minute) + ingested := base.UnixNano() + if _, err := db.ExecContext(ctx, ` +WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) +INSERT INTO telemetry.spans ( + namespace, trace_id, span_id, service, start_time, start_unix_nano, + ingested_at, ingested_unix_nano +) +SELECT + 'default', printf('trace-%d', i), printf('span-%d', i), 'svc', + base_time + ((i % 30) * INTERVAL '1' MINUTE), + epoch_ns(base_time + ((i % 30) * INTERVAL '1' MINUTE)), base_time, ? +FROM range(100) t(i), input`, base, ingested); err != nil { + t.Fatal(err) + } + var affected int64 + if err := db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM spans +WHERE ingested_unix_nano > ? AND ingested_unix_nano <= ?`, ingested-1, ingested).Scan(&affected); err != nil { + t.Fatal(err) + } + if affected != 100 { + t.Fatalf("affected fixture rows = %d, want 100", affected) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + subHi, err := edgeSubWindowEnd(ctx, tx, ingested-1, ingested, base, base.Add(29*time.Minute), 10) + if err != nil { + t.Fatal(err) + } + if want := base.Add(time.Minute); !subHi.Equal(want) { + t.Fatalf("dense sub-window ended at %s, want %s", subHi, want) + } +} diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 052be804..61565d81 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -151,14 +151,11 @@ func (p *ParquetStore) RowCount() uint64 { return count } -// CommitBatch publishes one atomic batch directory. ctx bounds the wait for -// the publish gate: ingest is the only caller that used to wait on it without -// a deadline, so a publication that stalled took the ingest path down with it -// and OTLP clients lost rows to their own timeouts. A commit that cannot get -// the gate now fails, retries, and is counted, instead of hanging. +// CommitBatch publishes one atomic batch directory. Preparation is allowed to +// finish; only the final publish-gate wait is capped. A stalled publication +// therefore cannot hang ingest, while a large valid encode does not consume +// the gate budget before it starts waiting. func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error { - ctx, cancel := context.WithTimeout(ctx, commitPublishWait) - defer cancel() if err := validateBatchID(metadata.ID); err != nil { return err } @@ -177,7 +174,7 @@ func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, if err != nil { return err } - if err := p.lockPublish(ctx); err != nil { + if err := p.lockCommitPublish(ctx); err != nil { return err } defer p.unlockPublish() @@ -266,7 +263,7 @@ func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, prepared.traces.path = filepath.Join(final, "trace.fidx") } - if err := p.lockPublish(ctx); err != nil { + if err := p.lockCommitPublish(ctx); err != nil { return err } defer p.unlockPublish() @@ -306,6 +303,12 @@ func (p *ParquetStore) lockPublish(ctx context.Context) error { } } +func (p *ParquetStore) lockCommitPublish(ctx context.Context) error { + waitCtx, cancel := context.WithTimeout(ctx, commitPublishWait) + defer cancel() + return p.lockPublish(waitCtx) +} + func (p *ParquetStore) unlockPublish() { p.publishGate <- struct{}{} } // RestoreRetiredInputs rolls back a compaction whose durable output vanished. diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index c9bb6719..390aab65 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "log/slog" "math" "os" "path/filepath" @@ -189,31 +188,12 @@ func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCo type parquetPublishFunc func(context.Context, func(context.Context) error) error -// RecoverParquet resolves a pending compaction marker. A marker that keeps -// failing is set aside after maxCompactionRecoveryAttempts so it stops gating -// the rest of maintenance; see quarantineCompactionMarker for why that -// preserves every input and output it touched. +// RecoverParquet resolves a pending compaction marker before any cleanup, +// retention, or new compaction can mutate its rollback set. func (r *Repository) RecoverParquet(ctx context.Context, publisher ParquetPublisher) error { r.compactionMu.Lock() defer r.compactionMu.Unlock() - err := r.recoverCompaction(ctx, publisher.PublishParquet) - if err == nil { - r.recoveryFailures = 0 - return nil - } - // A cancelled pass says nothing about the marker: shutdown and publication - // contention must not count toward giving up on it. - if ctx.Err() != nil { - return err - } - r.recoveryFailures++ - if r.recoveryFailures < maxCompactionRecoveryAttempts { - return err - } - slog.Error("Parquet compaction recovery failed repeatedly; setting marker aside", - "attempts", r.recoveryFailures, "err", err, "marker", quarantinedMarkerName) - r.recoveryFailures = 0 - return errors.Join(err, r.quarantineCompactionMarker()) + return r.recoverCompaction(ctx, publisher.PublishParquet) } func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPublishFunc) error { @@ -228,6 +208,9 @@ func (r *Repository) recoverCompaction(ctx context.Context, publish parquetPubli if err := json.Unmarshal(data, &marker); err != nil { return err } + if err := validateCompactionMarker(marker); err != nil { + return err + } stageExists, err := pathExists(r.compactionStage(marker.Output.ID)) if err != nil { return err @@ -267,6 +250,35 @@ func (r *Repository) compactionStage(id string) string { return filepath.Join(r.root, "compaction", id) } +func validateCompactionMarker(marker compactionMarker) error { + if err := validateCompactionID(marker.Output.ID); err != nil { + return fmt.Errorf("invalid compaction output: %w", err) + } + if len(marker.Inputs) == 0 { + return errors.New("compaction marker has no inputs") + } + for _, id := range marker.Inputs { + if err := validateCompactionID(id); err != nil { + return fmt.Errorf("invalid compaction input: %w", err) + } + } + return nil +} + +// validateCompactionID rejects marker-controlled paths before they are joined +// to the storage root. It intentionally matches the Parquet batch-ID grammar. +func validateCompactionID(id string) error { + if id == "" || len(id) > 128 || id[0] == '.' { + return fmt.Errorf("invalid batch ID %q", id) + } + for _, r := range id { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') { + return fmt.Errorf("invalid batch ID %q", id) + } + } + return nil +} + func syncFile(path string) error { f, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index b0d5efeb..7db0f4b1 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -26,25 +26,11 @@ type Batch struct { // Repository publishes self-contained Parquet batch directories. The // directory rename is the transaction and the filesystem is the catalog. type Repository struct { - root string - Parquet *telemetry.ParquetStore - // compactionMu guards the compaction marker and the retired directories a - // pending marker may still need, along with recoveryFailures. - compactionMu sync.Mutex - recoveryFailures int + root string + Parquet *telemetry.ParquetStore + compactionMu sync.Mutex } -// maxCompactionRecoveryAttempts bounds how many passes a marker may fail -// recovery before it is set aside. A marker that cannot be recovered gates -// retention, compaction, and retired-directory cleanup — correctly, since all -// three could destroy what a rollback needs — so without a give-up path one -// bad marker latches every form of maintenance off for the process lifetime -// and storage grows unreclaimed behind a healthy-looking probe. -const maxCompactionRecoveryAttempts = 3 - -// quarantinedMarkerName is the marker set aside by quarantineCompactionMarker. -const quarantinedMarkerName = "COMPACTION.json.failed" - func Open(root string) (*Repository, error) { if err := os.MkdirAll(root, 0o755); err != nil { return nil, err @@ -54,19 +40,11 @@ func Open(root string) (*Repository, error) { return nil, err } r := &Repository{root: root, Parquet: parquetStore} - // A marker that cannot be recovered must not keep the process from - // booting: refusing to start leaves the operator with no way to run the - // cleanup that would clear it, so the only recovery was a manual rm -rf of - // live storage. Set it aside instead and come up degraded. Nothing is - // deleted — the staged output and the retired inputs both survive — so the - // compaction can still be completed or rolled back by hand. if err := r.recoverCompaction(context.Background(), func(ctx context.Context, publish func(context.Context) error) error { return publish(ctx) }); err != nil { - slog.Error("Parquet compaction recovery failed at open; setting marker aside", - "err", err, "marker", quarantinedMarkerName) - if quarantineErr := r.quarantineCompactionMarker(); quarantineErr != nil { - _ = r.Close() - return nil, errors.Join(fmt.Errorf("recover Parquet compaction: %w", err), quarantineErr) - } + r.logUnresolvedCompaction(err) + _ = r.Close() + return nil, fmt.Errorf("recover Parquet compaction (unresolved marker at %s): %w", + filepath.Join(root, "COMPACTION.json"), err) } if err := r.cleanupCompactionArtifacts(); err != nil { _ = r.Close() @@ -79,47 +57,45 @@ func Open(root string) (*Repository, error) { return r, nil } -// cleanupCompactionArtifacts drops staging left by an interrupted compaction. +// logUnresolvedCompaction spells out the operator's options for a marker that +// blocks startup. // -// A set-aside marker keeps its staged output: that directory holds the only -// copy of the compaction's merged rows, and it has to survive every boot the -// marker survives, not just the one that set it aside — otherwise the promise -// that an operator can still complete the compaction by hand lasts exactly one -// restart. +// Refusing to boot is only the safe half of the decision. A marker names the +// retired inputs a rollback still needs, and those directories hold the only +// copy of the rows the compaction was merging, so a startup that guessed at +// the rollback set could delete them — which is why recovery fails closed. The +// other half is saying where to look, because the reachable next step for an +// instance that will not start is rm -rf of the data directory, and that +// destroys exactly what failing closed preserved. +// +// The guidance is logged rather than wrapped into the error: the paths and the +// ordering constraint do not fit an error string that composes, and the +// rollback warning is the kind of thing an operator has to be able to read +// once, in full, at the moment the process refuses to come up. +func (r *Repository) logUnresolvedCompaction(err error) { + slog.Error("Parquet compaction is unresolved; Fanout will not start", + "err", err, + "marker", filepath.Join(r.root, "COMPACTION.json"), + "staged_replacement", filepath.Join(r.root, "compaction"), + "retired_inputs", r.Parquet.BatchesDir(), + "nothing_deleted", "the staged replacement and the retired inputs (named .retired-) are both intact", + "retry", "clear the underlying cause and start again; recovery re-runs on its own", + "rollback", "rename every .retired- directory to .batch, then delete the staged replacement and the marker", + "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") +} + +// cleanupCompactionArtifacts drops staging left after compaction recovery has +// completed and consumed its live marker. func (r *Repository) cleanupCompactionArtifacts() error { - setAside, err := pathExists(filepath.Join(r.root, quarantinedMarkerName)) - if err != nil { + if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { return err } - if !setAside { - if err := os.RemoveAll(filepath.Join(r.root, "compaction")); err != nil { - return err - } - } if err := os.Remove(filepath.Join(r.root, "COMPACTION.json.tmp")); err != nil && !errors.Is(err, os.ErrNotExist) { return err } return syncDirectory(r.root) } -// quarantineCompactionMarker sets aside a marker whose recovery keeps failing -// so it stops gating retention, compaction, and retired-directory cleanup. -// -// Nothing is deleted: the staged output stays, and the retired inputs are -// protected from cleanup by protectedRetiredSuffixes, so the operator can -// still complete or roll back the compaction by hand. This is not the -// batch-level quarantine that was rejected — no authoritative telemetry is -// discarded, and no batch becomes unreadable that was readable before. -func (r *Repository) quarantineCompactionMarker() error { - if err := os.Rename(filepath.Join(r.root, "COMPACTION.json"), filepath.Join(r.root, quarantinedMarkerName)); err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err - } - return syncDirectory(r.root) -} - func (r *Repository) Close() error { return r.Parquet.Close() } // CleanupParquet removes retired inputs only while no retention or compaction @@ -131,13 +107,10 @@ func (r *Repository) CleanupParquet() error { } func (r *Repository) cleanupRetired() error { - protected, protectAll, err := r.protectedRetiredSuffixes() + protected, err := r.protectedRetiredSuffix() if err != nil { return err } - if protectAll { - return nil - } entries, err := os.ReadDir(r.Parquet.BatchesDir()) if err != nil { return err @@ -149,7 +122,7 @@ func (r *Repository) cleanupRetired() error { if !entry.IsDir() || strings.HasSuffix(name, telemetry.BatchSuffix) || !strings.Contains(name, ".retired") { continue } - if protectedRetired(name, protected) { + if protected != "" && strings.HasSuffix(name, protected) { continue } if err := os.RemoveAll(filepath.Join(r.Parquet.BatchesDir(), name)); err != nil { @@ -164,44 +137,24 @@ func (r *Repository) cleanupRetired() error { return cleanupErr } -func protectedRetired(name string, protected map[string]bool) bool { - for suffix := range protected { - if strings.HasSuffix(name, suffix) { - return true - } +// protectedRetiredSuffix names the retired-input set a live compaction marker +// may still need. An unreadable marker fails cleanup closed. +func (r *Repository) protectedRetiredSuffix() (string, error) { + data, err := os.ReadFile(filepath.Join(r.root, "COMPACTION.json")) + if errors.Is(err, os.ErrNotExist) { + return "", nil } - return false -} - -// protectedRetiredSuffixes names the retired-input sets that a live or -// set-aside compaction marker may still need in order to roll back. Deleting -// one of those is unrecoverable: the input is gone and its rows were never -// published under the replacement. -// -// A marker whose contents cannot be parsed names nothing, so protectAll tells -// the caller to delete no retired directory at all. Returning an error instead -// would fail Open on every boot — the marker is already set aside and stays on -// disk, so the failure repeats forever and cleanup can never run, which is the -// outcome setting it aside exists to avoid. -func (r *Repository) protectedRetiredSuffixes() (protected map[string]bool, protectAll bool, err error) { - protected = make(map[string]bool) - for _, name := range []string{"COMPACTION.json", quarantinedMarkerName} { - data, err := os.ReadFile(filepath.Join(r.root, name)) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err != nil { - return nil, false, err - } - var marker compactionMarker - if err := json.Unmarshal(data, &marker); err != nil { - slog.Error("compaction marker is unreadable; retaining every retired batch", - "marker", name, "err", err) - return nil, true, nil - } - protected[".retired-"+marker.Output.ID] = true + if err != nil { + return "", err + } + var marker compactionMarker + if err := json.Unmarshal(data, &marker); err != nil { + return "", fmt.Errorf("read live compaction marker: %w", err) + } + if err := validateCompactionMarker(marker); err != nil { + return "", err } - return protected, false, nil + return ".retired-" + marker.Output.ID, nil } func (r *Repository) Commit(ctx context.Context, batch Batch) error { diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index b17dbad2..63483862 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -654,12 +654,10 @@ func seedFailedCompaction(t *testing.T, dir string, repository *Repository) { } } -// TestRepositorySetsAsideUnrecoverableMarker pins the give-up path. A marker -// that cannot be recovered correctly gates retention, compaction, and retired -// cleanup — all three can destroy what a rollback needs — so without a bound on -// how long it may do so, one bad marker disables every form of maintenance for -// the process lifetime while storage grows behind a healthy-looking probe. -func TestRepositorySetsAsideUnrecoverableMarker(t *testing.T) { +// TestRepositoryRecoveryFailureStaysLive pins fail-closed recovery. The marker +// and rollback set remain authoritative until recovery succeeds; silently +// bypassing them can make old inputs eligible for deletion. +func TestRepositoryRecoveryFailureStaysLive(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { @@ -671,52 +669,16 @@ func TestRepositorySetsAsideUnrecoverableMarker(t *testing.T) { failing := testParquetPublisherFunc(func(context.Context, func(context.Context) error) error { return errors.New("publication unavailable") }) - for attempt := 1; attempt < maxCompactionRecoveryAttempts; attempt++ { + for attempt := 1; attempt <= 5; attempt++ { if err := repository.RecoverParquet(context.Background(), failing); err == nil { t.Fatalf("attempt %d: recovery reported success", attempt) } if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("attempt %d: marker set aside too early: %v", attempt, err) + t.Fatalf("attempt %d: live marker was removed: %v", attempt, err) } } - if err := repository.RecoverParquet(context.Background(), failing); err == nil { - t.Fatal("final attempt reported success") - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); !os.IsNotExist(err) { - t.Fatalf("marker still live after %d failures: %v", maxCompactionRecoveryAttempts, err) - } - if _, err := os.Stat(filepath.Join(dir, quarantinedMarkerName)); err != nil { - t.Fatalf("marker was not preserved for the operator: %v", err) - } - // Maintenance is unblocked: recovery is a no-op now, so a later pass runs. - if err := repository.RecoverParquet(context.Background(), failing); err != nil { - t.Fatalf("maintenance still gated after the marker was set aside: %v", err) - } -} - -// TestRepositoryCancelledRecoveryDoesNotCountTowardGivingUp keeps shutdown and -// publication contention from being mistaken for a bad marker. -func TestRepositoryCancelledRecoveryDoesNotCountTowardGivingUp(t *testing.T) { - dir := t.TempDir() - repository, err := Open(dir) - if err != nil { - t.Fatal(err) - } - defer repository.Close() - seedFailedCompaction(t, dir, repository) - - cancelled, cancel := context.WithCancel(context.Background()) - cancel() - failing := testParquetPublisherFunc(func(ctx context.Context, _ func(context.Context) error) error { - return ctx.Err() - }) - for range maxCompactionRecoveryAttempts * 2 { - if err := repository.RecoverParquet(cancelled, failing); err == nil { - t.Fatal("cancelled recovery reported success") - } - } - if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { - t.Fatalf("cancelled attempts set the marker aside: %v", err) + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json.failed")); !os.IsNotExist(err) { + t.Fatalf("recovery created a fallback marker: %v", err) } } @@ -763,11 +725,7 @@ func TestRepositoryCleanupPreservesMarkerRollbackSet(t *testing.T) { } } -// TestRepositoryOpensDespiteUnrecoverableMarker pins that a bad marker cannot -// stop the process from booting. Failing Open left the operator with no way to -// run the cleanup that would clear it, so the only recovery was a manual -// rm -rf of live storage. -func TestRepositoryOpensDespiteUnrecoverableMarker(t *testing.T) { +func TestRepositoryOpenFailsClosedOnUnrecoverableMarker(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { @@ -789,25 +747,19 @@ func TestRepositoryOpensDespiteUnrecoverableMarker(t *testing.T) { } } - reopened, err := Open(dir) - if err != nil { - t.Fatalf("Open refused to boot with an unrecoverable marker: %v", err) + if reopened, err := Open(dir); err == nil { + _ = reopened.Close() + t.Fatal("Open succeeded with an unrecoverable marker") } - defer reopened.Close() - if _, err := os.Stat(filepath.Join(dir, quarantinedMarkerName)); err != nil { - t.Fatalf("marker was not set aside at open: %v", err) + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("Open removed the live marker: %v", err) } if _, err := os.Stat(stage); err != nil { - t.Fatalf("staged output was destroyed rather than preserved: %v", err) + t.Fatalf("Open destroyed the staged output: %v", err) } } -// TestRepositoryBootsRepeatedlyWithUnreadableMarker pins that setting a marker -// aside actually ends the failure. The set-aside marker stays on disk, so any -// boot path that errors on parsing it fails identically forever — turning the -// mechanism meant to keep one bad marker from bricking the instance into the -// thing that bricks it. -func TestRepositoryBootsRepeatedlyWithUnreadableMarker(t *testing.T) { +func TestRepositoryOpenFailsOnUnreadableMarker(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { @@ -820,51 +772,42 @@ func TestRepositoryBootsRepeatedlyWithUnreadableMarker(t *testing.T) { t.Fatal(err) } for boot := 1; boot <= 3; boot++ { - reopened, err := Open(dir) - if err != nil { - t.Fatalf("boot %d refused to start with an unreadable marker: %v", boot, err) + if reopened, err := Open(dir); err == nil { + _ = reopened.Close() + t.Fatalf("boot %d accepted an unreadable marker", boot) } - if err := reopened.Close(); err != nil { - t.Fatal(err) + if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { + t.Fatalf("boot %d removed the unreadable marker: %v", boot, err) } } } -// TestRepositoryKeepsSetAsideCompactionStageAcrossBoots pins that the staged -// output outlives the boot that set the marker aside. It holds the only copy of -// that compaction's merged rows, so deleting it on the next restart leaves the -// operator able to roll back but never to complete. -func TestRepositoryKeepsSetAsideCompactionStageAcrossBoots(t *testing.T) { +func TestRepositoryOpenRejectsUnsafeCompactionMarker(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) if err != nil { t.Fatal(err) } - seedFailedCompaction(t, dir, repository) if err := repository.Close(); err != nil { t.Fatal(err) } - stage := filepath.Join(dir, "compaction") - entries, err := os.ReadDir(stage) + marker := compactionMarker{ + Output: telemetry.BatchMetadata{ID: "../outside"}, + Inputs: []string{"input"}, + } + data, err := json.Marshal(marker) if err != nil { t.Fatal(err) } - for _, entry := range entries { - if err := os.WriteFile(filepath.Join(stage, entry.Name(), "metadata.json"), []byte("{"), 0o644); err != nil { - t.Fatal(err) - } + markerPath := filepath.Join(dir, "COMPACTION.json") + if err := os.WriteFile(markerPath, data, 0o600); err != nil { + t.Fatal(err) } - - for boot := 1; boot <= 3; boot++ { - reopened, err := Open(dir) - if err != nil { - t.Fatalf("boot %d: %v", boot, err) - } - if err := reopened.Close(); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(stage); err != nil { - t.Fatalf("boot %d deleted the set-aside compaction's staged output: %v", boot, err) - } + if reopened, err := Open(dir); err == nil { + _ = reopened.Close() + t.Fatal("Open accepted a marker-controlled path outside storage") + } + if _, err := os.Stat(markerPath); err != nil { + t.Fatalf("Open removed the invalid live marker: %v", err) } } From a78a712d8eded74db4e60cff2190098e686013d8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:15:52 -0700 Subject: [PATCH 23/31] fix(storage): bound the stats reader and unify the batch-ID grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of a70592e4, reading it directly rather than delegating. updateParquetStats now takes the Parquet read gate, which is the right call — it was the one reader that could observe a namespace mid-swap. But it runs on the rollup and maintenance loop goroutines under the process context, so unlike every other reader it had no deadline, and a stuck publication would park both loops indefinitely. The wait is now bounded; a skipped refresh costs one tick of stale gauges, which is strictly less than what waiting costs. validateCompactionID duplicated telemetry.validateBatchID's grammar in a second package, where the two had already begun to diverge in implementation. The rule decides which strings may become directory names under the storage root, so two copies means two gates that can disagree about what is safe. The telemetry version is now exported and the store calls it. cleanupRetired names the coupling it acts on: a live marker is the only thing that makes a retired directory unreclaimable, so deleting COMPACTION.json without first renaming its .retired- directories back to .batch is not a rollback — it makes those rows deletable, and this loop removes them on the next pass. The startup log warns about this; the code that does the deleting did not say so. Also documents the site-level runbook changes from the review: the troubleshooting section for a startup-blocking marker, and the data layout reference, which still described COMPACTION.json as a transient marker rather than a startup gate. Adds a regression test for the bounded stats refresh. --- internal/query/duck.go | 13 +++++- internal/query/duck_test.go | 24 +++++++++++ internal/telemetry/parquet.go | 19 +++++---- internal/telemetry/store/compaction.go | 18 +-------- internal/telemetry/store/repository.go | 6 +++ site/src/content/docs/guides/troubleshoot.mdx | 40 +++++++++++++++++++ .../content/docs/reference/data-layout.mdx | 2 +- 7 files changed, 97 insertions(+), 25 deletions(-) diff --git a/internal/query/duck.go b/internal/query/duck.go index 28398059..1bf43b40 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -84,6 +84,9 @@ const ( const ( parquetDrainBudget = 2 * defaultWriterGrace parquetSwapBudget = 30 * time.Second + // parquetStatsWait bounds the gauge refresh, which is the only Parquet + // reader that runs on a loop goroutine rather than behind a request. + parquetStatsWait = 5 * time.Second ) // rollupPublicationSafetyLag covers the maximum public SQL hold, publication @@ -404,8 +407,16 @@ func (d *Duck) RunRollups(ctx context.Context) { } // updateParquetStats refreshes the per-signal file-count and byte-size gauges. +// +// The wait for the snapshot is bounded. Every other reader carries a request +// deadline, but this one runs on the rollup and maintenance loops under the +// process context, so an unbounded wait here would park those loops for as +// long as a publication stayed stuck. Gauges are refreshed again next tick; +// skipping one refresh costs nothing that waiting would not cost more. func (d *Duck) updateParquetStats(ctx context.Context) { - if err := d.lockParquetRead(ctx); err != nil { + statsCtx, cancel := context.WithTimeout(ctx, parquetStatsWait) + defer cancel() + if err := d.lockParquetRead(statsCtx); err != nil { slog.Warn("parquet stats skipped", "err", err) return } diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 9795c4d7..27257652 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -810,3 +810,27 @@ func TestMaintenanceRunsOnEveryTick(t *testing.T) { t.Fatal(err) } } + +// TestUpdateParquetStatsDoesNotParkOnAStalledPublication pins that the gauge +// refresh gives up rather than waiting out a stuck publication. It is the only +// Parquet reader that runs on the rollup and maintenance loop goroutines under +// the process context, so an unbounded wait here stops those loops entirely — +// the same failure this storage layer has produced repeatedly, one level down. +func TestUpdateParquetStatsDoesNotParkOnAStalledPublication(t *testing.T) { + d := &Duck{} + mustLock(&d.parquetMu) + defer d.parquetMu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + d.updateParquetStats(ctx) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("stats refresh parked on a stalled publication instead of giving up") + } +} diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 61565d81..a21b987d 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -156,7 +156,7 @@ func (p *ParquetStore) RowCount() uint64 { // therefore cannot hang ingest, while a large valid encode does not consume // the gate budget before it starts waiting. func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, spans []Span, logs []Log, metrics []Metric) error { - if err := validateBatchID(metadata.ID); err != nil { + if err := ValidateBatchID(metadata.ID); err != nil { return err } metadata.Version = batchMetadataVersion @@ -314,11 +314,11 @@ func (p *ParquetStore) unlockPublish() { p.publishGate <- struct{}{} } // RestoreRetiredInputs rolls back a compaction whose durable output vanished. // The complete namespace change is hidden from readers by publish. func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func(context.Context) error) error) error { - if err := validateBatchID(replacementID); err != nil { + if err := ValidateBatchID(replacementID); err != nil { return err } for _, id := range inputs { - if err := validateBatchID(id); err != nil { + if err := ValidateBatchID(id); err != nil { return err } } @@ -711,11 +711,11 @@ func (p *ParquetStore) PrepareReplacement(dir string, metadata BatchMetadata) er // PublishReplacement validates a prepared compacted batch, atomically swaps it // for its inputs while readers are pinned, then deletes retired inputs. func (p *ParquetStore) PublishReplacement(stage string, metadata BatchMetadata, inputs []string, publish func(func(context.Context) error) error) error { - if err := validateBatchID(metadata.ID); err != nil { + if err := ValidateBatchID(metadata.ID); err != nil { return err } for _, id := range inputs { - if err := validateBatchID(id); err != nil { + if err := ValidateBatchID(id); err != nil { return err } } @@ -1033,7 +1033,7 @@ func readBatchMetadata(path string) (BatchMetadata, error) { if metadata.Version != batchMetadataVersion { return BatchMetadata{}, fmt.Errorf("unsupported batch metadata version %d", metadata.Version) } - if err := validateBatchID(metadata.ID); err != nil { + if err := ValidateBatchID(metadata.ID); err != nil { return BatchMetadata{}, err } if metadata.Spans < 0 || metadata.Logs < 0 || metadata.Metrics < 0 { @@ -1078,7 +1078,12 @@ func syncDirectory(dir string) error { return f.Sync() } -func validateBatchID(id string) error { +// ValidateBatchID is the one grammar for identifiers that become directory +// names under the storage root. It is exported because the store package +// validates marker-supplied IDs against the same rule before joining them to a +// path; a second copy there could drift and leave the two gates disagreeing +// about what is a safe name. +func ValidateBatchID(id string) error { if id == "" || len(id) > 128 || id[0] == '.' || strings.ContainsAny(id, `/\\`) { return fmt.Errorf("invalid telemetry batch ID %q", id) } diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 390aab65..5108dc6a 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -251,34 +251,20 @@ func (r *Repository) compactionStage(id string) string { } func validateCompactionMarker(marker compactionMarker) error { - if err := validateCompactionID(marker.Output.ID); err != nil { + if err := telemetry.ValidateBatchID(marker.Output.ID); err != nil { return fmt.Errorf("invalid compaction output: %w", err) } if len(marker.Inputs) == 0 { return errors.New("compaction marker has no inputs") } for _, id := range marker.Inputs { - if err := validateCompactionID(id); err != nil { + if err := telemetry.ValidateBatchID(id); err != nil { return fmt.Errorf("invalid compaction input: %w", err) } } return nil } -// validateCompactionID rejects marker-controlled paths before they are joined -// to the storage root. It intentionally matches the Parquet batch-ID grammar. -func validateCompactionID(id string) error { - if id == "" || len(id) > 128 || id[0] == '.' { - return fmt.Errorf("invalid batch ID %q", id) - } - for _, r := range id { - if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.') { - return fmt.Errorf("invalid batch ID %q", id) - } - } - return nil -} - func syncFile(path string) error { f, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 7db0f4b1..f9a4f093 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -122,6 +122,12 @@ func (r *Repository) cleanupRetired() error { if !entry.IsDir() || strings.HasSuffix(name, telemetry.BatchSuffix) || !strings.Contains(name, ".retired") { continue } + // The live marker is the only thing that makes a retired directory + // unreclaimable, and it protects exactly one output's inputs. So + // removing COMPACTION.json without first renaming its + // .retired- directories back to .batch does not + // roll the compaction back — it makes the rows here deletable, and + // this loop deletes them on the next pass. if protected != "" && strings.HasSuffix(name, protected) { continue } diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index 8d1fd0b2..8705a91b 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -19,6 +19,46 @@ The one that surprises people is an unrecognised variable. Fanout rejects any release — stops the process instead of being ignored. That is deliberate; see [configuration](/reference/configuration). +## An unresolved compaction blocks startup + +One startup refusal is not about configuration. If a compaction was interrupted +and Fanout can neither finish nor undo it, recovery fails closed and the error +names `telemetry/COMPACTION.json`. + +That is deliberate. The marker lists the batches the compaction retired, and +those directories hold the only copy of the rows it was merging. A startup that +guessed at the rollback set could delete them, so Fanout stops instead and +leaves everything where it is: the staged replacement under +`telemetry/compaction/` and the retired inputs under +`telemetry/parquet/batches/` (named `.retired-`) are both intact. + +Usually the interruption was transient — a full disk, or the process killed +mid-swap. **Clear the underlying cause and start Fanout again.** Recovery +re-runs on its own and needs no manual step. + +If recovery keeps failing, roll the compaction back by hand. With the process +stopped, and from `telemetry/parquet/batches/`: + +1. Read `telemetry/COMPACTION.json`. It is JSON, with an `output` object and an + `inputs` list of batch ids. +2. For each id in `inputs`, if `.retired-` exists, rename it back + to `.batch`. Ids already present as `.batch` were never retired and + need nothing. +3. Delete the staged replacement directory under `telemetry/compaction/`. +4. Delete `telemetry/COMPACTION.json`. +5. Start Fanout. + +Deleting the whole `telemetry/` directory also clears the refusal, and discards +exactly the telemetry the refusal was protecting. Neither step above needs it. + +:::caution[Restore the inputs before deleting the marker] +Step 2 comes before step 4 for a reason. The marker is what protects the +retired directories from cleanup — with it gone, the next successful startup +treats every `*.retired-*` directory as reclaimable and removes it. Deleting +the marker on its own is therefore not a rollback; it discards the rows the +compaction had retired. +::: + ## An exporter is rejected Work down this list in order: diff --git a/site/src/content/docs/reference/data-layout.mdx b/site/src/content/docs/reference/data-layout.mdx index 6f738ebd..6ec3011a 100644 --- a/site/src/content/docs/reference/data-layout.mdx +++ b/site/src/content/docs/reference/data-layout.mdx @@ -17,7 +17,7 @@ Everything Fanout persists lives under `FANOUT_DATA_DIR` (`./data` by default, | `telemetry/parquet/batches/_schema.batch/` | Empty schema anchors that keep every DuckDB view queryable | | `telemetry/parquet/staging/` | Unacknowledged writes being prepared for atomic publication | | `telemetry/compaction/` | Prepared replacement batches during compaction | -| `telemetry/COMPACTION.json` | Transient durable marker for an interrupted compaction swap | +| `telemetry/COMPACTION.json` | Durable marker for an interrupted compaction swap. Present only while one is unresolved, and blocks startup until it is — see [troubleshoot](/guides/troubleshoot) | | `query/catalog.duckdb` | Rebuildable DuckDB rollups and query state | | `query/tmp/` | Spill space for queries that exceed the memory cap | | `control/fanout.sqlite` | Users, sessions, dashboards, alert rules, agent history | From aacffe05e79d39c90bcaa60ed218300d5f8c9b6d Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:21:42 -0700 Subject: [PATCH 24/31] fix(storage): fix rollback and stats checks Remove both possible replacement locations during manual rollback so restored inputs cannot coexist with a published output. Exercise the stats reader's own wait limit without caller cancellation. --- internal/query/duck.go | 8 +++++--- internal/query/duck_test.go | 10 ++++++---- internal/telemetry/store/repository.go | 2 +- site/src/content/docs/guides/troubleshoot.mdx | 16 ++++++++++------ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/internal/query/duck.go b/internal/query/duck.go index 1bf43b40..052150b1 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -84,11 +84,13 @@ const ( const ( parquetDrainBudget = 2 * defaultWriterGrace parquetSwapBudget = 30 * time.Second - // parquetStatsWait bounds the gauge refresh, which is the only Parquet - // reader that runs on a loop goroutine rather than behind a request. - parquetStatsWait = 5 * time.Second ) +// parquetStatsWait bounds the gauge refresh, which is the only Parquet reader +// that runs on a loop goroutine rather than behind a request. It is a variable +// so the internal lease can be exercised without a five-second test. +var parquetStatsWait = 5 * time.Second + // rollupPublicationSafetyLag covers the maximum public SQL hold, publication // grace, bounded commit retries, and queued Parquet encoding with headroom for // a busy disk. Rows stamped at request receipt remain inside the recomputed diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 27257652..079f69ec 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -817,20 +817,22 @@ func TestMaintenanceRunsOnEveryTick(t *testing.T) { // the process context, so an unbounded wait here stops those loops entirely — // the same failure this storage layer has produced repeatedly, one level down. func TestUpdateParquetStatsDoesNotParkOnAStalledPublication(t *testing.T) { + originalWait := parquetStatsWait + parquetStatsWait = 25 * time.Millisecond + t.Cleanup(func() { parquetStatsWait = originalWait }) + d := &Duck{} mustLock(&d.parquetMu) defer d.parquetMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() done := make(chan struct{}) go func() { defer close(done) - d.updateParquetStats(ctx) + d.updateParquetStats(context.Background()) }() select { case <-done: - case <-time.After(10 * time.Second): + case <-time.After(time.Second): t.Fatal("stats refresh parked on a stalled publication instead of giving up") } } diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index f9a4f093..c5763bff 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -80,7 +80,7 @@ func (r *Repository) logUnresolvedCompaction(err error) { "retired_inputs", r.Parquet.BatchesDir(), "nothing_deleted", "the staged replacement and the retired inputs (named .retired-) are both intact", "retry", "clear the underlying cause and start again; recovery re-runs on its own", - "rollback", "rename every .retired- directory to .batch, then delete the staged replacement and the marker", + "rollback", "rename every .retired- directory to .batch, delete both possible replacement locations (compaction/ and parquet/batches/.batch), then delete the marker", "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") } diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index 8705a91b..debe3634 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -28,9 +28,10 @@ names `telemetry/COMPACTION.json`. That is deliberate. The marker lists the batches the compaction retired, and those directories hold the only copy of the rows it was merging. A startup that guessed at the rollback set could delete them, so Fanout stops instead and -leaves everything where it is: the staged replacement under -`telemetry/compaction/` and the retired inputs under -`telemetry/parquet/batches/` (named `.retired-`) are both intact. +leaves everything where it is: the replacement is either staged under +`telemetry/compaction/` or already published as +`telemetry/parquet/batches/.batch`, and the retired inputs under +`telemetry/parquet/batches/` (named `.retired-`) remain intact. Usually the interruption was transient — a full disk, or the process killed mid-swap. **Clear the underlying cause and start Fanout again.** Recovery @@ -44,9 +45,12 @@ stopped, and from `telemetry/parquet/batches/`: 2. For each id in `inputs`, if `.retired-` exists, rename it back to `.batch`. Ids already present as `.batch` were never retired and need nothing. -3. Delete the staged replacement directory under `telemetry/compaction/`. -4. Delete `telemetry/COMPACTION.json`. -5. Start Fanout. +3. Delete `telemetry/compaction/` if it exists. +4. Delete `telemetry/parquet/batches/.batch` if it exists. Leaving + a published replacement beside the restored inputs duplicates its rows; a + corrupt replacement continues to block startup. +5. Delete `telemetry/COMPACTION.json`. +6. Start Fanout. Deleting the whole `telemetry/` directory also clears the refusal, and discards exactly the telemetry the refusal was protecting. Neither step above needs it. From b8345fb29a126ae8a2724b9df8b0833e904564f6 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:30:00 -0700 Subject: [PATCH 25/31] fix(storage): align the startup log with the rollback procedure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aacffe05 corrected the rollback guidance to delete both possible replacement locations, since recovery checks for the output either staged under compaction/ or already published as .batch. The docs and the rollback field were updated; the two log fields that describe where things are were not. An operator reading the startup log was told the staged replacement was intact and pointed at compaction/. When the interruption happened after publication that directory is empty, and the reachable conclusion from "the log says it is here and it is not" is that the merged rows are gone — which is the panic that ends in rm -rf of the data directory, the one outcome failing closed exists to prevent. The field now names both locations, matching the rollback procedure and the troubleshooting guide, and staged_replacement is renamed to compaction_staging so the key stops asserting a location the code does not guarantee. --- internal/telemetry/store/repository.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index c5763bff..8ff57ccf 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -76,9 +76,9 @@ func (r *Repository) logUnresolvedCompaction(err error) { slog.Error("Parquet compaction is unresolved; Fanout will not start", "err", err, "marker", filepath.Join(r.root, "COMPACTION.json"), - "staged_replacement", filepath.Join(r.root, "compaction"), + "compaction_staging", filepath.Join(r.root, "compaction"), "retired_inputs", r.Parquet.BatchesDir(), - "nothing_deleted", "the staged replacement and the retired inputs (named .retired-) are both intact", + "nothing_deleted", "the replacement — staged as compaction/, or already published as parquet/batches/.batch — and the retired inputs (named .retired-) are all intact", "retry", "clear the underlying cause and start again; recovery re-runs on its own", "rollback", "rename every .retired- directory to .batch, delete both possible replacement locations (compaction/ and parquet/batches/.batch), then delete the marker", "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") From 238f2febc4b78a6693ee966eecd3dfa913ebc1b5 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:33:29 -0700 Subject: [PATCH 26/31] fix(storage): make rollback guidance safe A failed publish may have already removed some retired inputs. Require operators to verify the complete rollback set before deleting the replacement or marker. --- internal/telemetry/store/repository.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 8ff57ccf..2593b407 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -78,9 +78,10 @@ func (r *Repository) logUnresolvedCompaction(err error) { "marker", filepath.Join(r.root, "COMPACTION.json"), "compaction_staging", filepath.Join(r.root, "compaction"), "retired_inputs", r.Parquet.BatchesDir(), - "nothing_deleted", "the replacement — staged as compaction/, or already published as parquet/batches/.batch — and the retired inputs (named .retired-) are all intact", + "preserved_state", "the live marker is retained and startup cleanup will not run after this failure", "retry", "clear the underlying cause and start again; recovery re-runs on its own", - "rollback", "rename every .retired- directory to .batch, delete both possible replacement locations (compaction/ and parquet/batches/.batch), then delete the marker", + "rollback_precondition", "back up telemetry first and verify every marker input exists as either .batch or .retired-; otherwise do not delete the replacement or marker", + "rollback", "after verification, rename each retired input to .batch, delete both possible replacement locations (compaction/ and parquet/batches/.batch), then delete the marker", "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") } From 7e4ab8ca4103638441efc00295f5ccb107fc3818 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:36:01 -0700 Subject: [PATCH 27/31] docs(storage): carry the rollback precondition into the runbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 238f2feb dropped the claim that the retired inputs are all intact, because a publish that failed partway may already have removed some, and added a precondition to the startup log: verify every marker input exists as either .batch or .retired- before deleting anything. The troubleshooting guide was not updated with it and still made the stronger claim. That left the guide's procedure unsafe in exactly the case the log now warns about. Its step 2 covered ids present as .retired- and ids present as .batch, but said nothing about an id present as neither — the dangerous one, where the rows survive only inside the replacement. An operator with a partially removed rollback set would have followed the steps in order, deleted the replacement, and destroyed those rows permanently. The guide now states that the rollback set may be incomplete, verifies it as an explicit step before any deletion, and tells the operator to stop and either restore from backup or complete the compaction instead. The caution covers both ordering constraints rather than only the second. --- site/src/content/docs/guides/troubleshoot.mdx | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index debe3634..95e1a59c 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -28,10 +28,15 @@ names `telemetry/COMPACTION.json`. That is deliberate. The marker lists the batches the compaction retired, and those directories hold the only copy of the rows it was merging. A startup that guessed at the rollback set could delete them, so Fanout stops instead and -leaves everything where it is: the replacement is either staged under +leaves things where they are: the replacement is either staged under `telemetry/compaction/` or already published as -`telemetry/parquet/batches/.batch`, and the retired inputs under -`telemetry/parquet/batches/` (named `.retired-`) remain intact. +`telemetry/parquet/batches/.batch`, and the retired inputs sit under +`telemetry/parquet/batches/`, named `.retired-`. + +A publish that failed partway through may already have removed some of those +retired inputs, so the rollback set is not guaranteed to be complete. That is +what step 2 below checks, and why it has to be checked before anything is +deleted. Usually the interruption was transient — a full disk, or the process killed mid-swap. **Clear the underlying cause and start Fanout again.** Recovery @@ -42,25 +47,31 @@ stopped, and from `telemetry/parquet/batches/`: 1. Read `telemetry/COMPACTION.json`. It is JSON, with an `output` object and an `inputs` list of batch ids. -2. For each id in `inputs`, if `.retired-` exists, rename it back - to `.batch`. Ids already present as `.batch` were never retired and - need nothing. -3. Delete `telemetry/compaction/` if it exists. -4. Delete `telemetry/parquet/batches/.batch` if it exists. Leaving +2. Back up `telemetry/`, then check every id in `inputs`. Each must exist as + either `.batch` (never retired) or `.retired-` (retired, + restorable). **If any id is missing in both forms, stop here.** Its rows now + survive only inside the replacement, so deleting the replacement or the + marker would destroy them. Recover that batch from a backup, or complete the + compaction instead of rolling it back. +3. Rename each `.retired-` back to `.batch`. +4. Delete `telemetry/compaction/` if it exists. +5. Delete `telemetry/parquet/batches/.batch` if it exists. Leaving a published replacement beside the restored inputs duplicates its rows; a corrupt replacement continues to block startup. -5. Delete `telemetry/COMPACTION.json`. -6. Start Fanout. +6. Delete `telemetry/COMPACTION.json`. +7. Start Fanout. Deleting the whole `telemetry/` directory also clears the refusal, and discards exactly the telemetry the refusal was protecting. Neither step above needs it. -:::caution[Restore the inputs before deleting the marker] -Step 2 comes before step 4 for a reason. The marker is what protects the -retired directories from cleanup — with it gone, the next successful startup -treats every `*.retired-*` directory as reclaimable and removes it. Deleting -the marker on its own is therefore not a rollback; it discards the rows the -compaction had retired. +:::caution[Verify the set, then restore, then delete] +The order is load-bearing in both directions. Verifying first (step 2) is what +keeps you from deleting a replacement that holds the only surviving copy of an +input's rows. Restoring before deleting the marker (steps 3 and 6) matters +because the marker is what protects the retired directories from cleanup — with +it gone, the next successful startup treats every `*.retired-*` directory as +reclaimable and removes it. Deleting the marker on its own is not a rollback; +it discards the rows the compaction had retired. ::: ## An exporter is rejected From 33f3cb152427c3003bc7b26518036add3d166cad Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:38:35 -0700 Subject: [PATCH 28/31] docs(storage): tighten rollback checks An input present in both active and retired forms is ambiguous and cannot be safely renamed. Require exactly one copy before deleting the compacted replacement or marker. --- site/src/content/docs/guides/troubleshoot.mdx | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index 95e1a59c..c3b8b7cf 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -26,9 +26,10 @@ and Fanout can neither finish nor undo it, recovery fails closed and the error names `telemetry/COMPACTION.json`. That is deliberate. The marker lists the batches the compaction retired, and -those directories hold the only copy of the rows it was merging. A startup that -guessed at the rollback set could delete them, so Fanout stops instead and -leaves things where they are: the replacement is either staged under +recovery must not guess whether the original batches or their replacement are +authoritative. A wrong guess could delete the only surviving copy of some rows, +so Fanout stops instead and leaves things where they are: the replacement is +either staged under `telemetry/compaction/` or already published as `telemetry/parquet/batches/.batch`, and the retired inputs sit under `telemetry/parquet/batches/`, named `.retired-`. @@ -43,16 +44,17 @@ mid-swap. **Clear the underlying cause and start Fanout again.** Recovery re-runs on its own and needs no manual step. If recovery keeps failing, roll the compaction back by hand. With the process -stopped, and from `telemetry/parquet/batches/`: +stopped, use `telemetry/parquet/batches/` as the batch directory: 1. Read `telemetry/COMPACTION.json`. It is JSON, with an `output` object and an `inputs` list of batch ids. -2. Back up `telemetry/`, then check every id in `inputs`. Each must exist as - either `.batch` (never retired) or `.retired-` (retired, - restorable). **If any id is missing in both forms, stop here.** Its rows now - survive only inside the replacement, so deleting the replacement or the - marker would destroy them. Recover that batch from a backup, or complete the - compaction instead of rolling it back. +2. Back up `telemetry/`, then check every id in `inputs`. Each must exist in + exactly one form: `.batch` (never retired) or + `.retired-` (retired, restorable). **If an id exists in both + forms or neither form, stop here.** Both forms make the authoritative copy + ambiguous; neither means its rows may survive only inside the replacement. + Reconcile the batch from the backup, or complete the compaction instead of + rolling it back. 3. Rename each `.retired-` back to `.batch`. 4. Delete `telemetry/compaction/` if it exists. 5. Delete `telemetry/parquet/batches/.batch` if it exists. Leaving From a00476e0d07de1dbd94553a1cf5046faa2c2c56f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 06:43:53 -0700 Subject: [PATCH 29/31] fix(storage): keep the rollback procedure in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manual rollback lived in both logUnresolvedCompaction and the troubleshooting guide, and every revision since it was written corrected one copy and left the other stating something the code no longer guarantees: b8345fb2 guide named both replacement locations; the log did not 238f2feb log dropped the "all intact" claim and added a verification precondition; the guide kept the claim and the old procedure 7e4ab8ca guide caught up 33f3cb15 guide required each input in exactly one form; the log still said either form, which permits the ambiguous case it was added to reject Each copy was correct when written. Operator guidance that is wrong in one of the two places an operator might read is worse than guidance in one place they have to open, so the log now links the runbook and states only what it alone knows: the absolute paths on this machine, that nothing has been cleaned up, that a plain retry is the first move, and the one invariant that must not be got wrong offline — deleting the marker by itself is not a rollback. The link is checked by a test that resolves its anchor against the guide's headings, so the pointer cannot rot the way the copy drifted. Also rewraps the paragraph 33f3cb15 left broken mid-sentence. --- internal/telemetry/store/repository.go | 12 +++++-- internal/telemetry/store/repository_test.go | 32 +++++++++++++++++++ site/src/content/docs/guides/troubleshoot.mdx | 3 +- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/internal/telemetry/store/repository.go b/internal/telemetry/store/repository.go index 2593b407..c78c1410 100644 --- a/internal/telemetry/store/repository.go +++ b/internal/telemetry/store/repository.go @@ -57,6 +57,15 @@ func Open(root string) (*Repository, error) { return r, nil } +// compactionRunbookURL is the manual rollback procedure. The startup log links +// to it rather than restating it, because the procedure has preconditions that +// are refined as the failure modes are understood, and a copy here drifts from +// the guide silently — each revision so far has corrected one location and left +// the other saying something the code no longer guarantees. Guidance an +// operator can act on wrongly is worse than guidance in one place they have to +// open. TestUnresolvedCompactionRunbookExists keeps the link honest. +const compactionRunbookURL = "https://fanout.run/guides/troubleshoot#an-unresolved-compaction-blocks-startup" + // logUnresolvedCompaction spells out the operator's options for a marker that // blocks startup. // @@ -80,8 +89,7 @@ func (r *Repository) logUnresolvedCompaction(err error) { "retired_inputs", r.Parquet.BatchesDir(), "preserved_state", "the live marker is retained and startup cleanup will not run after this failure", "retry", "clear the underlying cause and start again; recovery re-runs on its own", - "rollback_precondition", "back up telemetry first and verify every marker input exists as either .batch or .retired-; otherwise do not delete the replacement or marker", - "rollback", "after verification, rename each retired input to .batch, delete both possible replacement locations (compaction/ and parquet/batches/.batch), then delete the marker", + "runbook", compactionRunbookURL, "warning", "deleting the marker on its own is not a rollback; cleanup then treats the retired inputs as reclaimable and removes them") } diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index 63483862..bb334bbf 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -811,3 +811,35 @@ func TestRepositoryOpenRejectsUnsafeCompactionMarker(t *testing.T) { t.Fatalf("Open removed the invalid live marker: %v", err) } } + +// TestUnresolvedCompactionRunbookExists keeps the startup log's runbook link +// pointing at a section that exists. The log stopped restating the rollback +// procedure because maintaining it in two places produced a string of +// revisions that corrected one copy and left the other unsafe; a link only +// removes that risk while it resolves, so the anchor is checked here rather +// than trusted. +func TestUnresolvedCompactionRunbookExists(t *testing.T) { + const guide = "../../../site/src/content/docs/guides/troubleshoot.mdx" + data, err := os.ReadFile(guide) + if err != nil { + t.Fatalf("read the runbook the startup log links to: %v", err) + } + _, anchor, found := strings.Cut(compactionRunbookURL, "#") + if !found { + t.Fatalf("runbook URL %q has no anchor", compactionRunbookURL) + } + var headings []string + for _, line := range strings.Split(string(data), "\n") { + title, isHeading := strings.CutPrefix(line, "## ") + if !isHeading { + continue + } + slug := strings.ToLower(strings.TrimSpace(title)) + slug = strings.Join(strings.Fields(slug), "-") + headings = append(headings, slug) + if slug == anchor { + return + } + } + t.Fatalf("runbook anchor %q is not a section in %s; sections are %v", anchor, guide, headings) +} diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index c3b8b7cf..9294bbf3 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -29,8 +29,7 @@ That is deliberate. The marker lists the batches the compaction retired, and recovery must not guess whether the original batches or their replacement are authoritative. A wrong guess could delete the only surviving copy of some rows, so Fanout stops instead and leaves things where they are: the replacement is -either staged under -`telemetry/compaction/` or already published as +either staged under `telemetry/compaction/` or already published as `telemetry/parquet/batches/.batch`, and the retired inputs sit under `telemetry/parquet/batches/`, named `.retired-`. From 87cb03cd61d459c85811eb9a45e607b791ef07fb Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 10:21:20 -0700 Subject: [PATCH 30/31] perf(storage): harden Parquet data path Parallelize group commit and native compaction while preserving atomic publication and bounded query memory. Add deep offline verification and explicit quarantine for unreadable authoritative batches. --- cmd/fanout/main.go | 85 ++++- cmd/fanout/main_test.go | 56 ++- go.mod | 2 +- internal/query/duck.go | 60 ++-- internal/query/duck_test.go | 18 +- internal/telemetry/parquet.go | 321 ++++++++++++++++-- internal/telemetry/parquet_test.go | 34 ++ internal/telemetry/store/compaction.go | 149 ++++++-- internal/telemetry/store/repair.go | 106 ++++++ internal/telemetry/store/repair_test.go | 142 ++++++++ internal/telemetry/store/repository_test.go | 105 ++++-- internal/telemetry/store/writer.go | 38 ++- internal/telemetry/store/writer_test.go | 25 +- site/src/content/docs/guides/troubleshoot.mdx | 33 ++ 14 files changed, 1021 insertions(+), 153 deletions(-) create mode 100644 internal/telemetry/store/repair.go create mode 100644 internal/telemetry/store/repair_test.go diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 9f0fcab8..dfcb1aeb 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -50,8 +50,13 @@ var tokenRedactRe = regexp.MustCompile(`token=[^&]+`) // version is set at build time via -ldflags "-X main.version=...". var version = "dev" +type repairCommand struct { + action string + batch string +} + func main() { - configPath, showVersion, loginEmail, healthURL, err := parseCommandLine(os.Args[1:], os.Stderr) + configPath, showVersion, loginEmail, healthURL, repair, err := parseCommandLine(os.Args[1:], os.Stderr) if errors.Is(err, flag.ErrHelp) { return } @@ -77,6 +82,13 @@ func main() { slog.Error("invalid configuration", "err", err) os.Exit(1) } + if repair != nil { + if err := runRepair(cfg.TelemetryDir(), *repair, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "fanout repair:", err) + os.Exit(1) + } + return + } if loginEmail != "" { if err := createLoginLink(cfg, loginEmail, os.Stderr); err != nil { slog.Error("create login link failed", "err", err) @@ -456,15 +468,15 @@ func main() { httpCancel() // triggers graceful HTTP shutdown (5s timeout) } -func parseCommandLine(args []string, output io.Writer) (configPath string, showVersion bool, loginEmail, healthURL string, err error) { +func parseCommandLine(args []string, output io.Writer) (configPath string, showVersion bool, loginEmail, healthURL string, repair *repairCommand, err error) { if len(args) == 1 && args[0] == "version" { - return "", true, "", "", nil + return "", true, "", "", nil, nil } if len(args) >= 1 && len(args) <= 2 && args[0] == "healthcheck" { if len(args) == 2 { - return "", false, "", args[1], nil + return "", false, "", args[1], nil, nil } - return "", false, "", "http://127.0.0.1:7520/healthz", nil + return "", false, "", "http://127.0.0.1:7520/healthz", nil, nil } flags := flag.NewFlagSet("fanout", flag.ContinueOnError) @@ -473,6 +485,8 @@ func parseCommandLine(args []string, output io.Writer) (configPath string, showV fmt.Fprintln(output, "Usage of fanout:") fmt.Fprintln(output, " fanout [flags]") fmt.Fprintln(output, " fanout [--config path] login-link ") + fmt.Fprintln(output, " fanout [--config path] repair verify") + fmt.Fprintln(output, " fanout [--config path] repair quarantine --batch ") fmt.Fprintln(output, " fanout healthcheck [url]") fmt.Fprintln(output, " fanout version") fmt.Fprintln(output, "Flags:") @@ -482,18 +496,71 @@ func parseCommandLine(args []string, output io.Writer) (configPath string, showV flags.BoolVar(&showVersion, "version", false, "print the Fanout version") flags.BoolVar(&showVersion, "v", false, "print the Fanout version") if err := flags.Parse(args); err != nil { - return "", false, "", "", err + return "", false, "", "", nil, err } if flags.NArg() == 2 && flags.Arg(0) == "login-link" { - return configPath, false, flags.Arg(1), "", nil + return configPath, false, flags.Arg(1), "", nil, nil + } + if flags.NArg() > 0 && flags.Arg(0) == "repair" { + repair, err := parseRepairCommand(flags.Args()[1:], output) + return configPath, false, "", "", repair, err } if flags.NArg() != 0 { err := fmt.Errorf("unexpected arguments: %s", strings.Join(flags.Args(), " ")) fmt.Fprintln(output, err) flags.Usage() - return "", false, "", "", err + return "", false, "", "", nil, err + } + return configPath, showVersion, "", "", nil, nil +} + +func parseRepairCommand(args []string, output io.Writer) (*repairCommand, error) { + if len(args) == 1 && args[0] == "verify" { + return &repairCommand{action: "verify"}, nil + } + if len(args) == 0 || args[0] != "quarantine" { + return nil, errors.New("repair requires 'verify' or 'quarantine --batch '") + } + flags := flag.NewFlagSet("fanout repair quarantine", flag.ContinueOnError) + flags.SetOutput(output) + batch := flags.String("batch", "", "unreadable authoritative batch ID to set aside") + if err := flags.Parse(args[1:]); err != nil { + return nil, err + } + if strings.TrimSpace(*batch) == "" || flags.NArg() != 0 { + return nil, errors.New("repair quarantine requires exactly --batch ") + } + return &repairCommand{action: "quarantine", batch: *batch}, nil +} + +func runRepair(root string, command repairCommand, output io.Writer) error { + switch command.action { + case "verify": + issues, err := telemetrystore.VerifyBatches(root) + if err != nil { + return err + } + if len(issues) == 0 { + fmt.Fprintln(output, "all authoritative telemetry batches passed validation") + return nil + } + for _, issue := range issues { + fmt.Fprintf(output, "%s: %v\n", issue.ID, issue.Err) + } + return fmt.Errorf("%d unreadable authoritative telemetry batch(es)", len(issues)) + case "quarantine": + destination, err := telemetrystore.QuarantineBatch(root, command.batch) + if destination != "" { + fmt.Fprintf(output, "batch %s set aside at %s\n", command.batch, destination) + } + if err != nil { + return err + } + fmt.Fprintln(output, "the quarantined telemetry is no longer queryable; preserve it for recovery from backup") + return nil + default: + return fmt.Errorf("unknown repair action %q", command.action) } - return configPath, showVersion, "", "", nil } func checkHealth(healthURL string) error { diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index 70e64387..5e0482e6 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "errors" "flag" "io" @@ -9,12 +10,15 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "strings" "testing" "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" "github.com/labstack/fanout/internal/store" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) func TestParseCommandLine(t *testing.T) { @@ -25,6 +29,7 @@ func TestParseCommandLine(t *testing.T) { wantVersion bool wantEmail string wantHealth string + wantRepair *repairCommand wantErr bool }{ {name: "server defaults", args: nil}, @@ -36,25 +41,32 @@ func TestParseCommandLine(t *testing.T) { {name: "login link with config", args: []string{"--config", "/etc/fanout.yaml", "login-link", "admin@example.com"}, wantPath: "/etc/fanout.yaml", wantEmail: "admin@example.com"}, {name: "default healthcheck", args: []string{"healthcheck"}, wantHealth: "http://127.0.0.1:7520/healthz"}, {name: "custom healthcheck", args: []string{"healthcheck", "http://fanout:8080/healthz"}, wantHealth: "http://fanout:8080/healthz"}, + {name: "repair verify", args: []string{"repair", "verify"}, wantRepair: &repairCommand{action: "verify"}}, + {name: "repair quarantine with config", args: []string{"--config", "/etc/fanout.yaml", "repair", "quarantine", "--batch", "broken"}, wantPath: "/etc/fanout.yaml", wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, + {name: "repair quarantine missing batch", args: []string{"repair", "quarantine"}, wantErr: true}, + {name: "unknown repair action", args: []string{"repair", "delete"}, wantErr: true}, {name: "login link missing email", args: []string{"login-link"}, wantErr: true}, {name: "unexpected argument", args: []string{"serve"}, wantErr: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - path, showVersion, email, healthURL, err := parseCommandLine(test.args, io.Discard) + path, showVersion, email, healthURL, repair, err := parseCommandLine(test.args, io.Discard) if (err != nil) != test.wantErr { t.Fatalf("error = %v, wantErr %v", err, test.wantErr) } if path != test.wantPath || showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth { t.Fatalf("result = (%q, %v, %q, %q), want (%q, %v, %q, %q)", path, showVersion, email, healthURL, test.wantPath, test.wantVersion, test.wantEmail, test.wantHealth) } + if repair == nil != (test.wantRepair == nil) || repair != nil && *repair != *test.wantRepair { + t.Fatalf("repair = %#v, want %#v", repair, test.wantRepair) + } }) } } func TestParseCommandLineHelp(t *testing.T) { - _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) + _, _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) if !errors.Is(err, flag.ErrHelp) { t.Fatalf("error = %v, want flag.ErrHelp", err) } @@ -62,7 +74,7 @@ func TestParseCommandLineHelp(t *testing.T) { func TestParseCommandLinePrintsOneErrorAndUsage(t *testing.T) { var output bytes.Buffer - _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) + _, _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) if err == nil { t.Fatal("expected unexpected-argument error") } @@ -78,6 +90,9 @@ func TestParseCommandLinePrintsOneErrorAndUsage(t *testing.T) { if !strings.Contains(output.String(), "healthcheck [url]") { t.Fatalf("output does not include healthcheck command: %q", output.String()) } + if !strings.Contains(output.String(), "repair quarantine --batch ") { + t.Fatalf("output does not include repair command: %q", output.String()) + } } func TestCheckHealth(t *testing.T) { @@ -102,6 +117,41 @@ func TestCheckHealth(t *testing.T) { } } +func TestRunRepairVerifiesAndQuarantinesUnreadableBatch(t *testing.T) { + root := t.TempDir() + repository, err := telemetrystore.Open(root) + if err != nil { + t.Fatal(err) + } + if err := repository.Commit(context.Background(), telemetrystore.Batch{ + ID: "broken", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}, + }); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "parquet", "batches", "broken.batch", "trace.fidx"), []byte("bad"), 0o644); err != nil { + t.Fatal(err) + } + + var output bytes.Buffer + if err := runRepair(root, repairCommand{action: "verify"}, &output); err == nil || !strings.Contains(output.String(), "broken:") { + t.Fatalf("verify = %v, output %q", err, output.String()) + } + output.Reset() + if err := runRepair(root, repairCommand{action: "quarantine", batch: "broken"}, &output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "no longer queryable") { + t.Fatalf("quarantine output = %q", output.String()) + } + output.Reset() + if err := runRepair(root, repairCommand{action: "verify"}, &output); err != nil { + t.Fatal(err) + } +} + func TestCreateLoginLink(t *testing.T) { cfg := config.Config{ AuthMode: "local", diff --git a/go.mod b/go.mod index f43c144e..d81ba590 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/zeebo/xxh3 v1.1.0 go.opentelemetry.io/proto/otlp v1.11.0 golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 golang.org/x/time v0.15.0 google.golang.org/grpc v1.83.2 @@ -89,7 +90,6 @@ require ( golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect golang.org/x/mod v0.40.0 // indirect golang.org/x/net v0.58.0 // indirect - golang.org/x/sync v0.22.0 // indirect golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/tools v0.49.0 // indirect diff --git a/internal/query/duck.go b/internal/query/duck.go index 052150b1..02c6bdff 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -72,7 +72,9 @@ const ( EndpointReadyStateKey = "endpoint_rollup_v1_ready" EndpointDisabledStateKey = "endpoint_rollup_v1_disabled" defaultDuckDBPoolSize = 1 - parquetMaintenanceBatchLimit = 64 + parquetCompactionCycle = 10 * time.Second + parquetCompactionCycleBudget = 8 * time.Second + parquetMaintenanceBatchLimit = 128 parquetMaintenancePhaseBudget = 10 * time.Minute ) @@ -470,18 +472,49 @@ func (d *Duck) runMaintenanceLoop(ctx context.Context) { d.updateParquetStats(ctx) } run() - ticker := time.NewTicker(every) - defer ticker.Stop() + maintenanceTicker := time.NewTicker(every) + defer maintenanceTicker.Stop() + var compactionTicker *time.Ticker + var compactionTick <-chan time.Time + if every > parquetCompactionCycle { + compactionTicker = time.NewTicker(parquetCompactionCycle) + compactionTick = compactionTicker.C + defer compactionTicker.Stop() + } for { select { - case <-ticker.C: + case <-maintenanceTicker.C: run() + case <-compactionTick: + if err := d.runParquetCompactionCycle(ctx); err != nil && ctx.Err() == nil { + slog.Warn("telemetry compaction cycle failed", "err", err) + } case <-ctx.Done(): return } } } +func (d *Duck) runParquetCompactionCycle(ctx context.Context) error { + if d.repository == nil { + return nil + } + started := time.Now() + if err := d.repository.RecoverParquet(ctx, d); err != nil { + metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, metrics.TelemetryError, time.Since(started).Seconds()) + return err + } + compacted, err := d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetCompactionCycleBudget) + result := metrics.TelemetryNoop + if err != nil { + result = metrics.TelemetryError + } else if compacted > 0 { + result = metrics.TelemetrySuccess + } + metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, result, time.Since(started).Seconds()) + return err +} + func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { start := time.Now() var pruneErr error @@ -1591,23 +1624,6 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet return d.repository.Parquet.Trace(ctx, query) } -// MergeParquet executes the query-engine-specific half of compaction. It reads -// immutable inputs and writes an unpublished staging file, so it does not -// contend with DuckDB rollup-cache writes. -func (d *Duck) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { - quoted := make([]string, len(inputs)) - for i, input := range inputs { - quoted[i] = quoteDuckString(input) - } - query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(quoted, ",")) - if signal == "spans" { - query += " ORDER BY _trace_hash, start_unix_nano, span_id" - } - stmt := fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 122880)", query, quoteDuckString(output)) - _, err := d.DB.ExecContext(ctx, stmt) - return err -} - // PublishParquet limits reader exclusion to the atomic directory swap. // // The drain and the swap get separate budgets. Sharing one deadline meant a @@ -1634,8 +1650,6 @@ func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) return publish(swapCtx) } -func quoteDuckString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } - func (d *Duck) lockParquetRead(ctx context.Context) error { if err := d.parquetMu.RLockContext(ctx); err != nil { return errors.Join(ErrParquetReadWait, err) diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 079f69ec..67a1521b 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -310,21 +310,12 @@ func TestPublishParquetHonorsContext(t *testing.T) { } } -func TestParquetWorkDoesNotWaitForDuckDBWriteGate(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } - defer db.Close() - mock.ExpectExec("COPY").WillReturnResult(sqlmock.NewResult(0, 1)) - d := &Duck{DB: db} +func TestParquetPublicationDoesNotWaitForDuckDBWriteGate(t *testing.T) { + d := &Duck{} release := d.writeGate.Lock(writegate.WriteRollupService) defer release() ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - if err := d.MergeParquet(ctx, "logs", []string{"/tmp/input.parquet"}, "/tmp/output.parquet"); err != nil { - t.Fatalf("merge waited for unrelated DuckDB write gate: %v", err) - } called := false if err := d.PublishParquet(ctx, func(context.Context) error { called = true; return nil }); err != nil { t.Fatalf("publication waited for unrelated DuckDB write gate: %v", err) @@ -332,9 +323,6 @@ func TestParquetWorkDoesNotWaitForDuckDBWriteGate(t *testing.T) { if !called { t.Fatal("publication callback did not run") } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatal(err) - } } func TestWaitingMaintenanceDoesNotBlockNewReaders(t *testing.T) { @@ -607,7 +595,7 @@ func TestMaintenanceRecoversCompactionBeforeRetention(t *testing.T) { t.Fatal(err) } } - if _, err := repository.CompactParquet(context.Background(), failingPublishCompactor{d}, 64); err == nil { + if _, err := repository.CompactParquet(context.Background(), failingPublishCompactor{d}, 8); err == nil { t.Fatal("compaction unexpectedly published") } if err := d.runRepositoryMaintenance(context.Background()); err != nil { diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index a21b987d..e945e0c8 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -18,6 +18,7 @@ import ( "github.com/parquet-go/parquet-go" "github.com/parquet-go/parquet-go/compress/zstd" "github.com/zeebo/xxh3" + "golang.org/x/sync/errgroup" ) const ( @@ -122,6 +123,22 @@ func (p *ParquetStore) StagingPath(id string) string { return filepath.Join(p.stagingDir, id) } +// MergeParquet rewrites immutable files into one native Parquet output. Span +// rows retain the index order required by trace.fidx; the other signals keep +// their input order. Re-encoding also consolidates tiny ingest row groups. +func (p *ParquetStore) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { + switch signal { + case "spans": + return mergeTypedParquet[spanParquetRow](ctx, inputs, output, true) + case "logs": + return mergeTypedParquet[logParquetRow](ctx, inputs, output, false) + case "metrics": + return mergeTypedParquet[metricParquetRow](ctx, inputs, output, false) + default: + return fmt.Errorf("unsupported Parquet signal %q", signal) + } +} + func (p *ParquetStore) BatchMetadata() []BatchMetadata { p.mu.RLock() defer p.mu.RUnlock() @@ -201,51 +218,72 @@ func (p *ParquetStore) CommitBatch(ctx context.Context, metadata BatchMetadata, } }() + var spanRows []spanParquetRow if len(spans) > 0 { - rows := make([]spanParquetRow, len(spans)) + spanRows = make([]spanParquetRow, len(spans)) for i := range spans { - rows[i] = makeSpanParquetRow(spans[i]) - rows[i].TraceHash = xxh3.HashString(rows[i].TraceID) - if rows[i].StartUnixNano > 0 && (metadata.MinSpanStartNanos == 0 || rows[i].StartUnixNano < metadata.MinSpanStartNanos) { - metadata.MinSpanStartNanos = rows[i].StartUnixNano + spanRows[i] = makeSpanParquetRow(spans[i]) + spanRows[i].TraceHash = xxh3.HashString(spanRows[i].TraceID) + if spanRows[i].StartUnixNano > 0 && (metadata.MinSpanStartNanos == 0 || spanRows[i].StartUnixNano < metadata.MinSpanStartNanos) { + metadata.MinSpanStartNanos = spanRows[i].StartUnixNano } - metadata.MaxSpanStartNanos = max(metadata.MaxSpanStartNanos, rows[i].StartUnixNano) + metadata.MaxSpanStartNanos = max(metadata.MaxSpanStartNanos, spanRows[i].StartUnixNano) } - sort.Slice(rows, func(i, j int) bool { - if rows[i].TraceHash != rows[j].TraceHash { - return rows[i].TraceHash < rows[j].TraceHash + sort.Slice(spanRows, func(i, j int) bool { + if spanRows[i].TraceHash != spanRows[j].TraceHash { + return spanRows[i].TraceHash < spanRows[j].TraceHash } - if rows[i].StartUnixNano != rows[j].StartUnixNano { - return rows[i].StartUnixNano < rows[j].StartUnixNano + if spanRows[i].StartUnixNano != spanRows[j].StartUnixNano { + return spanRows[i].StartUnixNano < spanRows[j].StartUnixNano } - return rows[i].SpanID < rows[j].SpanID + return spanRows[i].SpanID < spanRows[j].SpanID }) - if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), rows, parquetPageSize); err != nil { - return fmt.Errorf("write span Parquet: %w", err) - } - if err := writeTraceIndex(filepath.Join(stage, "trace.fidx"), rows); err != nil { - return fmt.Errorf("write trace index: %w", err) - } } + logRows := make([]logParquetRow, len(logs)) if len(logs) > 0 { - rows := make([]logParquetRow, len(logs)) for i := range logs { - rows[i] = makeLogParquetRow(logs[i]) - } - if err := writeTypedParquet(filepath.Join(stage, "logs.parquet"), rows, parquetPageSize); err != nil { - return fmt.Errorf("write log Parquet: %w", err) + logRows[i] = makeLogParquetRow(logs[i]) } } + metricRows := make([]metricParquetRow, len(metrics)) if len(metrics) > 0 { - rows := make([]metricParquetRow, len(metrics)) for i := range metrics { - rows[i] = makeMetricParquetRow(metrics[i]) - } - if err := writeTypedParquet(filepath.Join(stage, "metrics.parquet"), rows, parquetPageSize); err != nil { - return fmt.Errorf("write metric Parquet: %w", err) + metricRows[i] = makeMetricParquetRow(metrics[i]) } } - if err := writeJSONFile(filepath.Join(stage, "metadata.json"), metadata); err != nil { + var writes errgroup.Group + if len(spanRows) > 0 { + writes.Go(func() error { + if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), spanRows, parquetPageSize, parquet.SortingWriterConfig(spanSortingColumns())); err != nil { + return fmt.Errorf("write span Parquet: %w", err) + } + return nil + }) + writes.Go(func() error { + if err := writeTraceIndex(filepath.Join(stage, "trace.fidx"), spanRows); err != nil { + return fmt.Errorf("write trace index: %w", err) + } + return nil + }) + } + if len(logRows) > 0 { + writes.Go(func() error { + if err := writeTypedParquet(filepath.Join(stage, "logs.parquet"), logRows, parquetPageSize); err != nil { + return fmt.Errorf("write log Parquet: %w", err) + } + return nil + }) + } + if len(metricRows) > 0 { + writes.Go(func() error { + if err := writeTypedParquet(filepath.Join(stage, "metrics.parquet"), metricRows, parquetPageSize); err != nil { + return fmt.Errorf("write metric Parquet: %w", err) + } + return nil + }) + } + writes.Go(func() error { return writeJSONFile(filepath.Join(stage, "metadata.json"), metadata) }) + if err := writes.Wait(); err != nil { return err } if err := syncDirectory(stage); err != nil { @@ -871,7 +909,7 @@ func (p *ParquetStore) ensureSchemaBatch() error { if err := os.Mkdir(stage, 0o755); err != nil { return err } - if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), []spanParquetRow{}, parquetPageSize); err != nil { + if err := writeTypedParquet(filepath.Join(stage, "spans.parquet"), []spanParquetRow{}, parquetPageSize, parquet.SortingWriterConfig(spanSortingColumns())); err != nil { return err } if err := writeTypedParquet(filepath.Join(stage, "logs.parquet"), []logParquetRow{}, parquetPageSize); err != nil { @@ -927,6 +965,128 @@ func loadRegisteredBatch(dir string) (*storedBatch, error) { return batch, nil } +// ValidatePublishedBatch performs an offline deep validation: startup's +// structural checks plus full Parquet decoding and an exact span/index walk. +// Repair uses it to prove a batch is unreadable before it can be set aside. +func ValidatePublishedBatch(dir string) error { + batch, err := loadRegisteredBatch(dir) + if err != nil { + return err + } + if batch.metadata.Spans > 0 { + if err := verifySpanParquet(filepath.Join(dir, "spans.parquet"), batch.metadata.Spans, batch.traces); err != nil { + return fmt.Errorf("verify spans Parquet: %w", err) + } + } + if batch.metadata.Logs > 0 { + if err := verifyTypedParquet[logParquetRow](filepath.Join(dir, "logs.parquet"), batch.metadata.Logs, nil); err != nil { + return fmt.Errorf("verify logs Parquet: %w", err) + } + } + if batch.metadata.Metrics > 0 { + if err := verifyTypedParquet[metricParquetRow](filepath.Join(dir, "metrics.parquet"), batch.metadata.Metrics, nil); err != nil { + return fmt.Errorf("verify metrics Parquet: %w", err) + } + } + return nil +} + +func verifySpanParquet(path string, expected int, index traceIndex) (err error) { + indexFile, err := os.Open(index.path) + if err != nil { + return err + } + defer func() { err = errors.Join(err, indexFile.Close()) }() + var current traceRange + var have bool + var entry uint64 + flush := func() error { + if !have { + return nil + } + indexed, err := readTraceRangeAt(indexFile, entry) + if err != nil { + return err + } + if indexed != current { + return fmt.Errorf("trace index entry %d does not match span rows", entry) + } + entry++ + have = false + return nil + } + err = verifyTypedParquet[spanParquetRow](path, expected, func(row spanParquetRow, position uint64) error { + if row.TraceHash != xxh3.HashString(row.TraceID) { + return fmt.Errorf("span row %d trace hash does not match trace ID", position) + } + if !have { + current = traceRange{hash: row.TraceHash, row: position, count: 1} + have = true + return nil + } + if current.hash == row.TraceHash { + current.count++ + return nil + } + if current.hash > row.TraceHash { + return fmt.Errorf("span row %d is not sorted by trace hash", position) + } + if err := flush(); err != nil { + return err + } + current = traceRange{hash: row.TraceHash, row: position, count: 1} + have = true + return nil + }) + if err != nil { + return err + } + if err := flush(); err != nil { + return err + } + if entry != index.entries { + return fmt.Errorf("trace index has %d entries; span rows produced %d", index.entries, entry) + } + return nil +} + +func verifyTypedParquet[T any](path string, expected int, visit func(T, uint64) error) (err error) { + file, err := os.Open(path) + if err != nil { + return err + } + defer func() { err = errors.Join(err, file.Close()) }() + reader := parquet.NewGenericReader[T](file) + defer func() { err = errors.Join(err, reader.Close()) }() + buffer := make([]T, 256) + var rows uint64 + for { + n, readErr := reader.Read(buffer) + for i := 0; i < n; i++ { + if visit != nil { + if err := visit(buffer[i], rows); err != nil { + return err + } + } + rows++ + } + clear(buffer[:n]) + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return readErr + } + if n == 0 { + return io.ErrNoProgress + } + } + if rows != uint64(expected) { + return fmt.Errorf("decoded %d rows; metadata declares %d", rows, expected) + } + return nil +} + // installBatch publishes a validated batch into the queryable set. func (p *ParquetStore) installBatch(batch *storedBatch) { p.mu.Lock() @@ -989,7 +1149,102 @@ func (p *ParquetStore) hasBatch(id string) bool { return ok } -func writeTypedParquet[T any](path string, rows []T, pageSize int) error { +func spanSortingColumns() parquet.SortingOption { + return parquet.SortingColumns( + parquet.Ascending("_trace_hash"), + parquet.Ascending("start_unix_nano"), + parquet.Ascending("span_id"), + ) +} + +func parquetWriterOptions(pageSize int, extra ...parquet.WriterOption) []parquet.WriterOption { + options := []parquet.WriterOption{ + parquet.Compression(&zstd.Codec{Level: zstd.SpeedFastest, Concurrency: 1}), + parquet.MaxRowsPerRowGroup(parquetRowGroupRows), + parquet.PageBufferSize(pageSize), + } + return append(options, extra...) +} + +type contextParquetRows struct { + context.Context + parquet.Rows +} + +func (r contextParquetRows) ReadRows(rows []parquet.Row) (int, error) { + if err := r.Context.Err(); err != nil { + return 0, err + } + return r.Rows.ReadRows(rows) +} + +func mergeTypedParquet[T any](ctx context.Context, inputs []string, output string, sorted bool) (err error) { + if err := ctx.Err(); err != nil { + return err + } + if len(inputs) == 0 { + return errors.New("merge Parquet requires at least one input") + } + files := make([]*os.File, 0, len(inputs)) + defer func() { + for _, file := range files { + err = errors.Join(err, file.Close()) + } + }() + groups := make([]parquet.RowGroup, 0, len(inputs)) + for _, input := range inputs { + file, openErr := os.Open(input) + if openErr != nil { + return openErr + } + files = append(files, file) + info, statErr := file.Stat() + if statErr != nil { + return statErr + } + parquetFile, parquetErr := parquet.OpenFile(file, info.Size()) + if parquetErr != nil { + return parquetErr + } + groups = append(groups, parquetFile.RowGroups()...) + } + var source parquet.RowGroup + var options []parquet.WriterOption + if sorted { + source, err = parquet.MergeRowGroups(groups, parquet.SortingRowGroupConfig(spanSortingColumns())) + options = append(options, parquet.SortingWriterConfig(spanSortingColumns())) + } else { + source = parquet.MultiRowGroup(groups...) + } + if err != nil { + return err + } + rows := source.Rows() + defer func() { err = errors.Join(err, rows.Close()) }() + out, err := os.OpenFile(output, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err != nil { + return err + } + ok := false + defer func() { + _ = out.Close() + if !ok { + _ = os.Remove(output) + } + }() + writer := parquet.NewGenericWriter[T](out, parquetWriterOptions(parquetPageSize, options...)...) + _, copyErr := parquet.CopyRows(writer, contextParquetRows{Context: ctx, Rows: rows}) + if closeErr := writer.Close(); copyErr != nil || closeErr != nil { + return errors.Join(copyErr, closeErr) + } + if err := out.Close(); err != nil { + return err + } + ok = true + return nil +} + +func writeTypedParquet[T any](path string, rows []T, pageSize int, options ...parquet.WriterOption) error { f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) if err != nil { return err @@ -1001,9 +1256,7 @@ func writeTypedParquet[T any](path string, rows []T, pageSize int) error { _ = os.Remove(path) } }() - writer := parquet.NewGenericWriter[T](f, - parquet.Compression(&zstd.Codec{Level: zstd.SpeedFastest, Concurrency: 1}), - parquet.MaxRowsPerRowGroup(parquetRowGroupRows), parquet.PageBufferSize(pageSize)) + writer := parquet.NewGenericWriter[T](f, parquetWriterOptions(pageSize, options...)...) if _, err := writer.Write(rows); err != nil { _ = writer.Close() return err diff --git a/internal/telemetry/parquet_test.go b/internal/telemetry/parquet_test.go index 61cf4bc6..17e60df1 100644 --- a/internal/telemetry/parquet_test.go +++ b/internal/telemetry/parquet_test.go @@ -2,6 +2,7 @@ package telemetry import ( "context" + "encoding/binary" "encoding/json" "errors" "fmt" @@ -109,6 +110,39 @@ func TestParquetStoreTraceUsesExactIDAndEventOrder(t *testing.T) { } } +func TestValidatePublishedBatchDetectsTraceIndexDataDivergence(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.CommitBatch(context.Background(), BatchMetadata{ID: "diverged"}, []Span{{ + TraceID: "trace", SpanID: "span", StartUnixNanos: 1, + }}, nil, nil); err != nil { + t.Fatal(err) + } + batchDir := store.BatchPath("diverged") + indexPath := filepath.Join(batchDir, "trace.fidx") + file, err := os.OpenFile(indexPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + var encoded [8]byte + binary.LittleEndian.PutUint64(encoded[:], 1) + if _, err := file.WriteAt(encoded[:], traceIndexHeaderSize); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := loadRegisteredBatch(batchDir); err != nil { + t.Fatalf("startup structural validation unexpectedly caught divergence: %v", err) + } + if err := ValidatePublishedBatch(batchDir); err == nil { + t.Fatal("deep validation accepted a trace index that disagrees with Parquet") + } +} + func TestParquetStoreTraceFiltersAndBoundsResultsDuringRead(t *testing.T) { store, err := OpenParquetStore(t.TempDir()) if err != nil { diff --git a/internal/telemetry/store/compaction.go b/internal/telemetry/store/compaction.go index 5108dc6a..f5f0ec6e 100644 --- a/internal/telemetry/store/compaction.go +++ b/internal/telemetry/store/compaction.go @@ -8,12 +8,17 @@ import ( "math" "os" "path/filepath" + "sort" "time" "github.com/labstack/fanout/internal/telemetry" + "golang.org/x/sync/errgroup" ) -const minCompactionInputs = 8 +const ( + maxCompactionRows = 25_000_000 + minCompactionInputs = 8 +) var parquetSignals = [...]string{"spans", "logs", "metrics"} @@ -27,21 +32,16 @@ type compactionKey struct { generation uint32 } -// ParquetCompactor keeps DuckDB execution and publication locking in the query -// layer while storage owns batch selection and crash-safe replacement state. +// ParquetPublisher lets the query layer exclude readers only for the atomic +// namespace swap; storage owns native merges and crash-safe replacement state. type ParquetPublisher interface { PublishParquet(context.Context, func(context.Context) error) error } -type ParquetCompactor interface { - ParquetPublisher - MergeParquet(context.Context, string, []string, string) error -} - // CompactParquet combines one same-day, same-generation group. The output is // prepared outside the query gate and swapped as one batch directory. -func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompactor, maxBatches int) (int, error) { - if compactor == nil || maxBatches < minCompactionInputs { +func (r *Repository) CompactParquet(ctx context.Context, publisher ParquetPublisher, maxBatches int) (int, error) { + if publisher == nil || maxBatches < minCompactionInputs { return 0, nil } r.compactionMu.Lock() @@ -50,12 +50,12 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac if exists, err := pathExists(markerPath); err != nil { return 0, err } else if exists { - if err := r.recoverCompaction(ctx, compactor.PublishParquet); err != nil { + if err := r.recoverCompaction(ctx, publisher.PublishParquet); err != nil { return 0, fmt.Errorf("recover pending Parquet compaction: %w", err) } } selected := selectCompactionBatches(r.Parquet.BatchMetadata(), maxBatches) - if len(selected) < minCompactionInputs { + if len(selected) < 2 { return 0, nil } output := telemetry.BatchMetadata{ @@ -92,6 +92,12 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac _ = os.RemoveAll(stage) } }() + type mergePlan struct { + signal string + inputs []string + output string + } + plans := make([]mergePlan, 0, len(parquetSignals)) for _, signal := range parquetSignals { inputs := make([]string, 0, len(selected)) for _, batch := range selected { @@ -105,13 +111,25 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac if len(inputs) == 0 { continue } - outputPath := filepath.Join(stage, signal+".parquet") - if err := compactor.MergeParquet(ctx, signal, inputs, outputPath); err != nil { - return 0, fmt.Errorf("compact %s Parquet: %w", signal, err) - } - if err := syncFile(outputPath); err != nil { - return 0, err - } + plans = append(plans, mergePlan{signal: signal, inputs: inputs, output: filepath.Join(stage, signal+".parquet")}) + } + group, mergeCtx := errgroup.WithContext(ctx) + for _, plan := range plans { + group.Go(func() error { + if err := r.Parquet.MergeParquet(mergeCtx, plan.signal, plan.inputs, plan.output); err != nil { + return fmt.Errorf("compact %s Parquet: %w", plan.signal, err) + } + if err := mergeCtx.Err(); err != nil { + return err + } + if err := syncFile(plan.output); err != nil { + return fmt.Errorf("sync compacted %s Parquet: %w", plan.signal, err) + } + return nil + }) + } + if err := group.Wait(); err != nil { + return 0, err } if err := r.Parquet.PrepareReplacement(stage, marker.Output); err != nil { return 0, err @@ -127,7 +145,7 @@ func (r *Repository) CompactParquet(ctx context.Context, compactor ParquetCompac return 0, err } prepared = true - if err := r.completeCompaction(ctx, marker, compactor.PublishParquet); err != nil { + if err := r.completeCompaction(ctx, marker, publisher.PublishParquet); err != nil { return 0, err } return len(selected), nil @@ -137,48 +155,105 @@ func selectCompactionBatches(batches []telemetry.BatchMetadata, maxBatches int) if maxBatches < minCompactionInputs { return nil } - counts := make(map[compactionKey]int) + groups := make(map[compactionKey][]telemetry.BatchMetadata) for _, batch := range batches { - if batch.MaxIngestedNanos > 0 { - counts[compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation}]++ + if batch.MaxIngestedNanos <= 0 { + continue } + key := compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation} + groups[key] = append(groups[key], batch) } var chosen compactionKey + var selected []telemetry.BatchMetadata found := false - for key, count := range counts { - if count >= minCompactionInputs && (!found || key.day < chosen.day || key.day == chosen.day && key.generation < chosen.generation) { - chosen, found = key, true + for key, group := range groups { + candidate := selectBoundedCompactionGroup(group, maxBatches) + if len(candidate) < 2 { + continue + } + if !found || key.day < chosen.day || key.day == chosen.day && key.generation < chosen.generation { + chosen, selected, found = key, candidate, true } } if !found { return nil } - selected := make([]telemetry.BatchMetadata, 0, min(maxBatches, counts[chosen])) - for _, batch := range batches { - if batch.MaxIngestedNanos <= 0 { + return selected +} + +// selectBoundedCompactionGroup keeps the high-reclaim full-group behavior for +// small files, but admits a smaller group when the row ceiling fills first. +// Without the latter, one successful generation can make every later group too +// large for the ceiling and permanently strand those files. +func selectBoundedCompactionGroup(group []telemetry.BatchMetadata, maxBatches int) []telemetry.BatchMetadata { + ordered := append([]telemetry.BatchMetadata(nil), group...) + sort.Slice(ordered, func(i, j int) bool { + left, right := compactionBatchRows(ordered[i]), compactionBatchRows(ordered[j]) + if left != right { + return left < right + } + if ordered[i].MinIngestedNanos != ordered[j].MinIngestedNanos { + return ordered[i].MinIngestedNanos < ordered[j].MinIngestedNanos + } + return ordered[i].ID < ordered[j].ID + }) + candidate := make([]telemetry.BatchMetadata, 0, min(maxBatches, len(ordered))) + var rows int64 + saturated := false + var smallest int64 + for _, batch := range ordered { + batchRows := compactionBatchRows(batch) + if batchRows <= 0 || batchRows > maxCompactionRows { continue } - if (compactionKey{day: batch.MaxIngestedNanos / int64(24*time.Hour), generation: batch.Generation}) == chosen { - selected = append(selected, batch) - if len(selected) == maxBatches { - break - } + if smallest == 0 { + smallest = batchRows + } + if len(candidate) == maxBatches { + saturated = true + break + } + if rows > maxCompactionRows-batchRows { + saturated = true + break } + candidate = append(candidate, batch) + rows += batchRows } - return selected + if rows == maxCompactionRows || smallest > 0 && smallest > maxCompactionRows-rows { + saturated = true + } + if len(candidate) == maxBatches || saturated && len(candidate) >= 2 { + return candidate + } + return nil +} + +func compactionBatchRows(batch telemetry.BatchMetadata) int64 { + if batch.Spans < 0 { + return math.MaxInt64 + } + rows := int64(batch.Spans) + for _, count := range [...]int{batch.Logs, batch.Metrics} { + if count < 0 || rows > math.MaxInt64-int64(count) { + return math.MaxInt64 + } + rows += int64(count) + } + return rows } // CompactParquetPass starts complete compactions until the phase budget // expires. An in-flight merge keeps the caller context so slow, valid work // commits instead of restarting the same input group on every pass. -func (r *Repository) CompactParquetPass(ctx context.Context, compactor ParquetCompactor, maxBatches int, budget time.Duration) (int, error) { +func (r *Repository) CompactParquetPass(ctx context.Context, publisher ParquetPublisher, maxBatches int, budget time.Duration) (int, error) { if maxBatches <= 0 || budget <= 0 { return 0, nil } deadline := time.Now().Add(budget) total := 0 for { - count, err := r.CompactParquet(ctx, compactor, maxBatches) + count, err := r.CompactParquet(ctx, publisher, maxBatches) total += count if err != nil || count == 0 || !time.Now().Before(deadline) { return total, err diff --git a/internal/telemetry/store/repair.go b/internal/telemetry/store/repair.go new file mode 100644 index 00000000..0ec6897a --- /dev/null +++ b/internal/telemetry/store/repair.go @@ -0,0 +1,106 @@ +package store + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/labstack/fanout/internal/telemetry" +) + +// BatchIssue is one authoritative batch that startup cannot validate. +type BatchIssue struct { + ID string + Path string + Err error +} + +// VerifyBatches validates every authoritative batch without opening a live +// repository or changing the filesystem. +func VerifyBatches(root string) ([]BatchIssue, error) { + batchesDir := filepath.Join(root, "parquet", "batches") + entries, err := os.ReadDir(batchesDir) + if err != nil { + return nil, fmt.Errorf("read telemetry batches: %w", err) + } + var issues []BatchIssue + for _, entry := range entries { + if !entry.IsDir() || entry.Name() == telemetry.SchemaBatch || !strings.HasSuffix(entry.Name(), telemetry.BatchSuffix) { + continue + } + path := filepath.Join(batchesDir, entry.Name()) + if err := telemetry.ValidatePublishedBatch(path); err != nil { + issues = append(issues, BatchIssue{ + ID: strings.TrimSuffix(entry.Name(), telemetry.BatchSuffix), Path: path, Err: err, + }) + } + } + return issues, nil +} + +// QuarantineBatch atomically sets aside one specifically named unreadable +// batch. Valid data and any batch protected by a live compaction transaction +// are refused. The returned directory remains beside the authoritative set so +// an operator can recover it from a backup or rename it back after repair. +func QuarantineBatch(root, id string) (string, error) { + if err := telemetry.ValidateBatchID(id); err != nil { + return "", err + } + if id+telemetry.BatchSuffix == telemetry.SchemaBatch { + return "", errors.New("the Parquet schema batch cannot be quarantined") + } + if err := ensureBatchOutsideLiveCompaction(root, id); err != nil { + return "", err + } + batchesDir := filepath.Join(root, "parquet", "batches") + source := filepath.Join(batchesDir, id+telemetry.BatchSuffix) + info, err := os.Lstat(source) + if err != nil { + return "", fmt.Errorf("inspect telemetry batch %s: %w", id, err) + } + if !info.IsDir() { + return "", fmt.Errorf("telemetry batch %s is not a directory", id) + } + if err := telemetry.ValidatePublishedBatch(source); err == nil { + return "", fmt.Errorf("telemetry batch %s is valid; refusing to quarantine authoritative data", id) + } + destination := filepath.Join(batchesDir, fmt.Sprintf("%s.quarantined-%d", id, time.Now().UTC().UnixNano())) + if err := os.Rename(source, destination); err != nil { + return "", fmt.Errorf("quarantine telemetry batch %s: %w", id, err) + } + if err := syncDirectory(batchesDir); err != nil { + return destination, fmt.Errorf("quarantined telemetry batch %s at %s but could not sync the directory: %w", id, destination, err) + } + return destination, nil +} + +func ensureBatchOutsideLiveCompaction(root, id string) error { + path := filepath.Join(root, "COMPACTION.json") + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect live compaction marker: %w", err) + } + var marker compactionMarker + if err := json.Unmarshal(data, &marker); err != nil { + return fmt.Errorf("read live compaction marker: %w", err) + } + if err := validateCompactionMarker(marker); err != nil { + return fmt.Errorf("read live compaction marker: %w", err) + } + if marker.Output.ID == id { + return fmt.Errorf("telemetry batch %s belongs to a live compaction; resolve that transaction before repair", id) + } + for _, input := range marker.Inputs { + if input == id { + return fmt.Errorf("telemetry batch %s belongs to a live compaction; resolve that transaction before repair", id) + } + } + return nil +} diff --git a/internal/telemetry/store/repair_test.go b/internal/telemetry/store/repair_test.go new file mode 100644 index 00000000..855f647f --- /dev/null +++ b/internal/telemetry/store/repair_test.go @@ -0,0 +1,142 @@ +package store + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/labstack/fanout/internal/telemetry" +) + +func TestVerifyBatchesReportsOnlyUnreadableBatches(t *testing.T) { + root := t.TempDir() + repository, err := Open(root) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"good", "broken"} { + if err := repository.Commit(context.Background(), Batch{ + ID: id, Spans: []telemetry.Span{{TraceID: id, SpanID: "span"}}, + }); err != nil { + t.Fatal(err) + } + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + corruptBatchFile(t, root, "broken", "spans.parquet") + + issues, err := VerifyBatches(root) + if err != nil { + t.Fatal(err) + } + if len(issues) != 1 || issues[0].ID != "broken" || issues[0].Err == nil { + t.Fatalf("issues = %#v", issues) + } +} + +func TestQuarantineBatchRefusesValidData(t *testing.T) { + root := t.TempDir() + repository, err := Open(root) + if err != nil { + t.Fatal(err) + } + if err := repository.Commit(context.Background(), Batch{ + ID: "valid", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}, + }); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + if _, err := QuarantineBatch(root, "valid"); err == nil || !strings.Contains(err.Error(), "valid") { + t.Fatalf("QuarantineBatch valid error = %v", err) + } +} + +func TestQuarantineBatchSetsAsideUnreadableDataAndRestoresStartup(t *testing.T) { + root := t.TempDir() + repository, err := Open(root) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"good", "broken"} { + if err := repository.Commit(context.Background(), Batch{ + ID: id, Spans: []telemetry.Span{{TraceID: id, SpanID: "span"}}, + }); err != nil { + t.Fatal(err) + } + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + corruptBatchFile(t, root, "broken", "trace.fidx") + + destination, err := QuarantineBatch(root, "broken") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(filepath.Base(destination), "broken.quarantined-") { + t.Fatalf("destination = %q", destination) + } + if _, err := os.Stat(filepath.Join(root, "parquet", "batches", "broken.batch")); !os.IsNotExist(err) { + t.Fatalf("authoritative batch still exists: %v", err) + } + if info, err := os.Stat(destination); err != nil || !info.IsDir() { + t.Fatalf("quarantine destination = %v, %v", info, err) + } + + reopened, err := Open(root) + if err != nil { + t.Fatalf("startup after quarantine: %v", err) + } + defer reopened.Close() + if got := reopened.RowCount(); got != 1 { + t.Fatalf("row count after quarantine = %d, want 1", got) + } +} + +func TestQuarantineBatchRefusesLiveCompactionMembers(t *testing.T) { + root := t.TempDir() + repository, err := Open(root) + if err != nil { + t.Fatal(err) + } + if err := repository.Commit(context.Background(), Batch{ + ID: "input", Spans: []telemetry.Span{{TraceID: "trace", SpanID: "span"}}, + }); err != nil { + t.Fatal(err) + } + if err := repository.Close(); err != nil { + t.Fatal(err) + } + corruptBatchFile(t, root, "input", "spans.parquet") + marker := compactionMarker{Output: telemetry.BatchMetadata{ID: "replacement"}, Inputs: []string{"input"}} + data, err := json.Marshal(marker) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "COMPACTION.json"), data, 0o644); err != nil { + t.Fatal(err) + } + if _, err := QuarantineBatch(root, "input"); err == nil || !strings.Contains(err.Error(), "live compaction") { + t.Fatalf("QuarantineBatch live input error = %v", err) + } +} + +func TestQuarantineBatchRejectsUnsafeID(t *testing.T) { + if _, err := QuarantineBatch(t.TempDir(), "../outside"); err == nil { + t.Fatal("unsafe batch ID was accepted") + } +} + +func corruptBatchFile(t *testing.T, root, id, name string) { + t.Helper() + path := filepath.Join(root, "parquet", "batches", id+telemetry.BatchSuffix, name) + if err := os.WriteFile(path, []byte("corrupt"), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/internal/telemetry/store/repository_test.go b/internal/telemetry/store/repository_test.go index bb334bbf..4a5bf334 100644 --- a/internal/telemetry/store/repository_test.go +++ b/internal/telemetry/store/repository_test.go @@ -17,7 +17,6 @@ import ( ) type testParquetCompactor struct { - db *sql.DB publishErr error afterSwap func() error } @@ -28,19 +27,6 @@ func (f testParquetPublisherFunc) PublishParquet(ctx context.Context, publish fu return f(ctx, publish) } -func (c *testParquetCompactor) MergeParquet(ctx context.Context, signal string, inputs []string, output string) error { - quoted := make([]string, len(inputs)) - for i, input := range inputs { - quoted[i] = sqlQuote(input) - } - query := fmt.Sprintf("SELECT * FROM read_parquet([%s], union_by_name=true)", strings.Join(quoted, ",")) - if signal == "spans" { - query += " ORDER BY _trace_hash, start_unix_nano, span_id" - } - _, err := c.db.ExecContext(ctx, fmt.Sprintf("COPY (%s) TO %s (FORMAT PARQUET, COMPRESSION ZSTD)", query, sqlQuote(output))) - return err -} - func (c *testParquetCompactor) PublishParquet(ctx context.Context, publish func(context.Context) error) error { if c.publishErr != nil { return c.publishErr @@ -460,7 +446,7 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } db := openTestDuckDB(t) defer db.Close() - compactor := &testParquetCompactor{db: db, afterSwap: func() error { + compactor := &testParquetCompactor{afterSwap: func() error { retired, err := filepath.Glob(filepath.Join(repository.Parquet.BatchesDir(), "*.retired-*")) if err != nil { return err @@ -470,7 +456,9 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } return nil }} - compacted, err := repository.CompactParquet(context.Background(), compactor, 64) + compactCtx, cancelCompact := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelCompact() + compacted, err := repository.CompactParquet(compactCtx, compactor, minCompactionInputs) if err != nil { t.Fatal(err) } @@ -502,6 +490,77 @@ func TestRepositoryCompactsParquetWithoutChangingRows(t *testing.T) { } } +func TestSelectCompactionBatchesRequiresFullSmallFileGroup(t *testing.T) { + const groupSize = 16 + batches := make([]telemetry.BatchMetadata, groupSize) + for i := range batches { + batches[i] = telemetry.BatchMetadata{ID: fmt.Sprintf("batch-%d", i), MaxIngestedNanos: 1, Spans: 1} + } + if selected := selectCompactionBatches(batches[:groupSize-1], groupSize); len(selected) != 0 { + t.Fatalf("selected partial group of %d batches", len(selected)) + } + if selected := selectCompactionBatches(batches, groupSize); len(selected) != groupSize { + t.Fatalf("selected full group = %d, want %d", len(selected), groupSize) + } +} + +func TestSelectCompactionBatchesBuildsRowBoundedGroup(t *testing.T) { + const maxBatches = 16 + batches := make([]telemetry.BatchMetadata, maxBatches) + for i := range batches { + batches[i] = telemetry.BatchMetadata{ + ID: fmt.Sprintf("batch-%d", i), MaxIngestedNanos: 1, Generation: 2, + Spans: 2_000_000, + } + } + selected := selectCompactionBatches(batches, maxBatches) + if len(selected) != 12 { + t.Fatalf("selected %d batches, want 12 below row limit", len(selected)) + } + var rows int + for _, batch := range selected { + rows += batch.Spans + batch.Logs + batch.Metrics + } + if rows > maxCompactionRows { + t.Fatalf("selected %d rows above limit %d", rows, maxCompactionRows) + } +} + +func TestSelectCompactionBatchesCombinesSaturatedLargeFiles(t *testing.T) { + const maxBatches = 16 + batches := make([]telemetry.BatchMetadata, 4) + for i := range batches { + batches[i] = telemetry.BatchMetadata{ + ID: fmt.Sprintf("large-%d", i), MaxIngestedNanos: 1, Generation: 3, + Spans: 6_000_000, + } + } + selected := selectCompactionBatches(batches, maxBatches) + if len(selected) != len(batches) { + t.Fatalf("selected %d saturated large batches, want %d", len(selected), len(batches)) + } +} + +func TestSelectCompactionBatchesSkipsUncompactableOlderGroup(t *testing.T) { + batches := make([]telemetry.BatchMetadata, 2*minCompactionInputs) + for i := range minCompactionInputs { + batches[i] = telemetry.BatchMetadata{ + ID: fmt.Sprintf("oversized-%d", i), MaxIngestedNanos: 1, + Spans: maxCompactionRows + 1, + } + } + for i := minCompactionInputs; i < len(batches); i++ { + batches[i] = telemetry.BatchMetadata{ + ID: fmt.Sprintf("newer-%d", i), MaxIngestedNanos: int64(24*time.Hour) + 1, + Spans: 1, + } + } + selected := selectCompactionBatches(batches, minCompactionInputs) + if len(selected) != minCompactionInputs || !strings.HasPrefix(selected[0].ID, "newer-") { + t.Fatalf("selection did not skip uncompactable older group: %#v", selected) + } +} + func TestRepositoryRecoversPendingCompactionWithoutRestart(t *testing.T) { dir := t.TempDir() repository, err := Open(dir) @@ -517,17 +576,15 @@ func TestRepositoryRecoversPendingCompactionWithoutRestart(t *testing.T) { t.Fatal(err) } } - db := openTestDuckDB(t) - defer db.Close() - compactor := &testParquetCompactor{db: db, publishErr: errors.New("publication unavailable")} - if _, err := repository.CompactParquet(context.Background(), compactor, 64); err == nil { + compactor := &testParquetCompactor{publishErr: errors.New("publication unavailable")} + if _, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err == nil { t.Fatal("compaction succeeded despite publication failure") } if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { t.Fatalf("pending marker: %v", err) } compactor.publishErr = nil - if compacted, err := repository.CompactParquet(context.Background(), compactor, 64); err != nil || compacted != 0 { + if compacted, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err != nil || compacted != 0 { t.Fatalf("resume compaction = %d, %v", compacted, err) } if got := repository.Parquet.BatchMetadata(); len(got) != 1 || got[0].Generation != 1 { @@ -643,10 +700,8 @@ func seedFailedCompaction(t *testing.T, dir string, repository *Repository) { t.Fatal(err) } } - db := openTestDuckDB(t) - t.Cleanup(func() { db.Close() }) - compactor := &testParquetCompactor{db: db, publishErr: errors.New("publication unavailable")} - if _, err := repository.CompactParquet(context.Background(), compactor, 64); err == nil { + compactor := &testParquetCompactor{publishErr: errors.New("publication unavailable")} + if _, err := repository.CompactParquet(context.Background(), compactor, minCompactionInputs); err == nil { t.Fatal("compaction succeeded despite publication failure") } if _, err := os.Stat(filepath.Join(dir, "COMPACTION.json")); err != nil { diff --git a/internal/telemetry/store/writer.go b/internal/telemetry/store/writer.go index 45045f87..6c84252d 100644 --- a/internal/telemetry/store/writer.go +++ b/internal/telemetry/store/writer.go @@ -14,8 +14,10 @@ import ( ) const ( - commitQueueDepth = 256 + commitQueueDepth = maxCommitWorkers commitRetryLimit = 5 + groupAdmissionWindow = 20 * time.Millisecond + maxAdmissionRequests = 512 maxGroupBatchRows = 50_000 maxCommitWorkers = 4 submissionQueueDepth = 256 @@ -30,6 +32,7 @@ type Writer struct { repository batchCommitter batchSize int retryDelay func(int) time.Duration + groupWindow time.Duration shutdownGrace time.Duration done chan struct{} submissions chan submission @@ -46,7 +49,10 @@ type commitJob struct { } func NewWriter(repository *Repository, batchSize int) *Writer { - return &Writer{repository: repository, batchSize: batchSize, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth)} + return &Writer{ + repository: repository, batchSize: batchSize, groupWindow: groupAdmissionWindow, + done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth), + } } func (w *Writer) Wait() { <-w.done } @@ -129,8 +135,31 @@ func (w *Writer) Run(ctx context.Context) error { } func (w *Writer) enqueueSubmissions(ctx context.Context, request submission, out chan<- commitJob) error { - requests := []submission{request} - for len(requests) < submissionQueueDepth { + requests := make([]submission, 1, maxAdmissionRequests) + requests[0] = request + rows := batchRows(request.batch) + limit := w.batchLimit() + if rows < limit { + window := w.groupWindow + if window <= 0 { + window = groupAdmissionWindow + } + timer := time.NewTimer(window) + defer timer.Stop() + admit: + for len(requests) < maxAdmissionRequests && rows < limit { + select { + case next := <-w.submissions: + requests = append(requests, next) + rows += batchRows(next.batch) + case <-timer.C: + break admit + case <-ctx.Done(): + return ctx.Err() + } + } + } + for len(requests) < maxAdmissionRequests { select { case next := <-w.submissions: requests = append(requests, next) @@ -141,7 +170,6 @@ func (w *Writer) enqueueSubmissions(ctx context.Context, request submission, out drained: metrics.UpdateQueueDepth("batch", len(w.submissions)) - limit := w.batchLimit() for len(requests) > 0 { firstRows := batchRows(requests[0].batch) // A full batch, a lone request, or a request that cannot share the diff --git a/internal/telemetry/store/writer_test.go b/internal/telemetry/store/writer_test.go index 10eb5ad5..54bede99 100644 --- a/internal/telemetry/store/writer_test.go +++ b/internal/telemetry/store/writer_test.go @@ -34,7 +34,7 @@ func (c *recordingCommitter) Commit(_ context.Context, batch Batch) error { func testWriter(committer batchCommitter, batchSize int) *Writer { return &Writer{ - repository: committer, batchSize: batchSize, done: make(chan struct{}), + repository: committer, batchSize: batchSize, groupWindow: groupAdmissionWindow, done: make(chan struct{}), submissions: make(chan submission, submissionQueueDepth), } } @@ -110,6 +110,29 @@ func TestWriterGroupsQueuedSubmissions(t *testing.T) { } } +func TestWriterAdmitsSubmissionArrivingDuringBoundedWindow(t *testing.T) { + committer := &recordingCommitter{} + w := testWriter(committer, maxGroupBatchRows) + w.groupWindow = 100 * time.Millisecond + first := submission{batch: Batch{Spans: []telemetry.Span{{SpanID: "first"}}}, ack: make(chan error, 1)} + second := submission{batch: Batch{Spans: []telemetry.Span{{SpanID: "second"}}}, ack: make(chan error, 1)} + out := make(chan commitJob, 1) + started := make(chan struct{}) + go func() { + close(started) + time.Sleep(time.Millisecond) + w.submissions <- second + }() + <-started + if err := w.enqueueSubmissions(context.Background(), first, out); err != nil { + t.Fatal(err) + } + job := <-out + if len(job.batches) != 1 || batchRows(job.batches[0]) != 2 || len(job.acks) != 2 { + t.Fatalf("admitted job = %#v", job) + } +} + func TestWriterCommitsOversizedSubmissionAtomically(t *testing.T) { committer := &recordingCommitter{} w := testWriter(committer, maxGroupBatchRows) diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index 9294bbf3..8fd02a01 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -75,6 +75,39 @@ reclaimable and removes it. Deleting the marker on its own is not a rollback; it discards the rows the compaction had retired. ::: +## An unreadable telemetry batch blocks startup + +Fanout also refuses to start when an authoritative `.batch` directory has +invalid metadata, a corrupt Parquet file, a missing signal file, or a damaged +trace index. First stop Fanout and back up `telemetry/`, then verify the full +authoritative set: + +```sh +fanout --config /etc/fanout/fanout.yaml repair verify +``` + +The command is read-only and names every unreadable batch. Restore each named +batch from backup when possible. If one cannot be recovered and discarding +only its telemetry is preferable to leaving the instance offline, set it aside +explicitly: + +```sh +fanout --config /etc/fanout/fanout.yaml repair quarantine --batch +``` + +Quarantine is an atomic rename beside the authoritative batch set, not a +deletion. It refuses a batch that validates successfully and refuses any input +or output protected by a live compaction marker. Preserve the reported +quarantine directory for later recovery, run `repair verify` again, and only +then restart Fanout. + +:::caution[Quarantine makes those rows unavailable] +The command is intentionally batch-specific and never runs during startup. +Every row in the quarantined directory disappears from queries until the batch +is restored, so do not use it as a substitute for fixing a transient disk or +permission error. +::: + ## An exporter is rejected Work down this list in order: From 16d0ef52944421bb59598db11af8978cf8e417ec Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 10:49:09 -0700 Subject: [PATCH 31/31] docs(storage): document Parquet architecture --- README.md | 30 ++- docs/diagrams/architecture.d2 | 4 +- docs/diagrams/architecture.svg | 197 ++++++++--------- docs/diagrams/persistence.d2 | 68 +++--- docs/diagrams/persistence.svg | 200 ++++++++---------- justfile | 32 ++- site/public/diagrams/architecture.svg | 121 +++++++++++ site/public/diagrams/persistence.svg | 138 ++++++++++++ .../content/docs/explanation/performance.mdx | 9 +- .../docs/explanation/storage-model.mdx | 37 +++- .../docs/explanation/why-one-binary.mdx | 11 +- .../content/docs/guides/tune-retention.mdx | 24 ++- .../content/docs/reference/data-layout.mdx | 11 +- .../src/content/docs/start/what-fanout-is.mdx | 16 +- site/src/styles/fanout.css | 44 ++++ 15 files changed, 652 insertions(+), 290 deletions(-) create mode 100644 site/public/diagrams/architecture.svg create mode 100644 site/public/diagrams/persistence.svg diff --git a/README.md b/README.md index 529e970c..de570e59 100644 --- a/README.md +++ b/README.md @@ -22,28 +22,36 @@ executable, including the React client. ![Fanout architecture](docs/diagrams/architecture.svg) -Telemetry lands over OTLP/gRPC or OTLP/HTTP, is durably committed as atomic -Parquet batches with persistent trace indexes, and is read back through a -DuckDB query kernel that also maintains service, -endpoint, and edge rollups. The browser client, an in-process agent, and any +Telemetry lands over OTLP/gRPC or OTLP/HTTP. Concurrent small requests may +share a group-commit batch, while up to four workers independently encode and +durably publish atomic Parquet directories with persistent trace indexes. +Targeted trace reads go through those indexes; DuckDB scans the same Parquet +for SQL and maintains rebuildable service, endpoint, and edge rollups. The +browser client, an in-process agent, and any external MCP host all reach the same typed observability contract rather than issuing raw SQL. -Independent ingest batches encode in parallel. Rollup-cache writes are -serialized inside DuckDB, while retention and compaction atomically swap -immutable Parquet directories behind active readers: +Parquet is the telemetry source of truth, DuckDB query state is disposable, +and SQLite is reserved for transactional product state. Native compaction +prepares replacements while reads continue and briefly gates readers only for +the crash-safe namespace swap: ![Fanout persistence](docs/diagrams/persistence.svg) Application state (users, sessions, dashboards, alert rules, agent threads) -lives in a separate SQLite database and never sits on the telemetry write path. +lives in that separate SQLite database and never sits on the telemetry write +path. There is no Iceberg, DuckLake, external catalog, or telemetry server +database. ## Performance The bundled [`cmd/bench`](cmd/bench) driver measures authenticated ingest and -optional dashboard read load against your hardware. Fanout does not publish a -throughput headline until the raw reports and exact driver revision can ship -with it; see [the benchmark publication standard](docs/benchmarking.md). +optional dashboard read load against your hardware. Ingest, DuckDB queries, +and native Parquet maintenance have separate coordination paths but still +compete for the same CPU, memory bandwidth, filesystem cache, and disk. Fanout +does not publish a throughput headline until the raw reports and exact driver +revision can ship with it; see [the benchmark publication +standard](docs/benchmarking.md). ## How it compares diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 index 52383e83..d2b34aaa 100644 --- a/docs/diagrams/architecture.d2 +++ b/docs/diagrams/architecture.d2 @@ -28,6 +28,7 @@ fanout: "fanout — one Go process" { obs: "Typed observability contract" commit: "Telemetry commit workers\natomic Parquet batches" query: "Query kernel\nDuckDB + rollups" + maint: "Storage maintenance\nretention + native compaction" alert: "Alert engine\nrule evaluation + webhooks" ingest -> commit @@ -39,7 +40,6 @@ fanout: "fanout — one Go process" { http -> obs: typed HTTP API obs -> query alert -> query: evaluates rollups - query -> commit: "merge and maintenance" } store: Storage { @@ -56,7 +56,9 @@ clients.browser -> fanout.http: HTTPS clients.ext -> fanout.http: "/mcp — OAuth" fanout.commit -> store.telemetry +fanout.query -> store.telemetry: "indexed traces + SQL scans" fanout.query -> store.qstate +fanout.maint -> store.telemetry: "atomic replacement" fanout.http -> store.control: "users, sessions, settings, dashboards, threads" fanout.alert -> store.control: "rules and fired alerts" fanout.agent -> model: HTTPS diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 0ed5235f..6f054670 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsmerge and maintenanceOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - - - - - - - - - - - - - + .d2-634443295 .fill-N1{fill:#0A0F25;} + .d2-634443295 .fill-N2{fill:#676C7E;} + .d2-634443295 .fill-N3{fill:#9499AB;} + .d2-634443295 .fill-N4{fill:#CFD2DD;} + .d2-634443295 .fill-N5{fill:#DEE1EB;} + .d2-634443295 .fill-N6{fill:#EEF1F8;} + .d2-634443295 .fill-N7{fill:#FFFFFF;} + .d2-634443295 .fill-B1{fill:#000536;} + .d2-634443295 .fill-B2{fill:#0F66B7;} + .d2-634443295 .fill-B3{fill:#4393DD;} + .d2-634443295 .fill-B4{fill:#87BFF3;} + .d2-634443295 .fill-B5{fill:#BCDDFB;} + .d2-634443295 .fill-B6{fill:#E5F3FF;} + .d2-634443295 .fill-AA2{fill:#7639C5;} + .d2-634443295 .fill-AA4{fill:#C1A2F3;} + .d2-634443295 .fill-AA5{fill:#DACEFB;} + .d2-634443295 .fill-AB4{fill:#EA99C6;} + .d2-634443295 .fill-AB5{fill:#FFDEF1;} + .d2-634443295 .stroke-N1{stroke:#0A0F25;} + .d2-634443295 .stroke-N2{stroke:#676C7E;} + .d2-634443295 .stroke-N3{stroke:#9499AB;} + .d2-634443295 .stroke-N4{stroke:#CFD2DD;} + .d2-634443295 .stroke-N5{stroke:#DEE1EB;} + .d2-634443295 .stroke-N6{stroke:#EEF1F8;} + .d2-634443295 .stroke-N7{stroke:#FFFFFF;} + .d2-634443295 .stroke-B1{stroke:#000536;} + .d2-634443295 .stroke-B2{stroke:#0F66B7;} + .d2-634443295 .stroke-B3{stroke:#4393DD;} + .d2-634443295 .stroke-B4{stroke:#87BFF3;} + .d2-634443295 .stroke-B5{stroke:#BCDDFB;} + .d2-634443295 .stroke-B6{stroke:#E5F3FF;} + .d2-634443295 .stroke-AA2{stroke:#7639C5;} + .d2-634443295 .stroke-AA4{stroke:#C1A2F3;} + .d2-634443295 .stroke-AA5{stroke:#DACEFB;} + .d2-634443295 .stroke-AB4{stroke:#EA99C6;} + .d2-634443295 .stroke-AB5{stroke:#FFDEF1;} + .d2-634443295 .background-color-N1{background-color:#0A0F25;} + .d2-634443295 .background-color-N2{background-color:#676C7E;} + .d2-634443295 .background-color-N3{background-color:#9499AB;} + .d2-634443295 .background-color-N4{background-color:#CFD2DD;} + .d2-634443295 .background-color-N5{background-color:#DEE1EB;} + .d2-634443295 .background-color-N6{background-color:#EEF1F8;} + .d2-634443295 .background-color-N7{background-color:#FFFFFF;} + .d2-634443295 .background-color-B1{background-color:#000536;} + .d2-634443295 .background-color-B2{background-color:#0F66B7;} + .d2-634443295 .background-color-B3{background-color:#4393DD;} + .d2-634443295 .background-color-B4{background-color:#87BFF3;} + .d2-634443295 .background-color-B5{background-color:#BCDDFB;} + .d2-634443295 .background-color-B6{background-color:#E5F3FF;} + .d2-634443295 .background-color-AA2{background-color:#7639C5;} + .d2-634443295 .background-color-AA4{background-color:#C1A2F3;} + .d2-634443295 .background-color-AA5{background-color:#DACEFB;} + .d2-634443295 .background-color-AB4{background-color:#EA99C6;} + .d2-634443295 .background-color-AB5{background-color:#FFDEF1;} + .d2-634443295 .color-N1{color:#0A0F25;} + .d2-634443295 .color-N2{color:#676C7E;} + .d2-634443295 .color-N3{color:#9499AB;} + .d2-634443295 .color-N4{color:#CFD2DD;} + .d2-634443295 .color-N5{color:#DEE1EB;} + .d2-634443295 .color-N6{color:#EEF1F8;} + .d2-634443295 .color-N7{color:#FFFFFF;} + .d2-634443295 .color-B1{color:#000536;} + .d2-634443295 .color-B2{color:#0F66B7;} + .d2-634443295 .color-B3{color:#4393DD;} + .d2-634443295 .color-B4{color:#87BFF3;} + .d2-634443295 .color-B5{color:#BCDDFB;} + .d2-634443295 .color-B6{color:#E5F3FF;} + .d2-634443295 .color-AA2{color:#7639C5;} + .d2-634443295 .color-AA4{color:#C1A2F3;} + .d2-634443295 .color-AA5{color:#DACEFB;} + .d2-634443295 .color-AB4{color:#EA99C6;} + .d2-634443295 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-634443295);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-634443295);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-634443295);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-634443295);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-634443295);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-634443295);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-634443295);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-634443295);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-634443295);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-634443295);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-634443295);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-634443295);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + + + + + + + + + + + + + + diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 index 850947f3..33472831 100644 --- a/docs/diagrams/persistence.d2 +++ b/docs/diagrams/persistence.d2 @@ -1,4 +1,4 @@ -# Where Fanout keeps state, and what serializes writes to it. +# How telemetry becomes durable, queryable, and compacted. # Render with `just diagrams`; commit the generated SVG beside this file. direction: down @@ -10,44 +10,54 @@ vars: { } } -writers: Writers { - style.fill: transparent - ingest: "OTLP request\nspans, logs, metrics" {shape: rectangle} - rollup: "Rollups\nservice, endpoint, edge" {shape: rectangle} - maint: "Merge and maintenance" {shape: rectangle} +ingest: "Authenticated OTLP requests\nspans, logs, metrics" { + shape: rectangle + style.stroke-dash: 3 } -commit: "Parallel atomic Parquet commits\nacknowledged after directory publication" { - shape: rectangle - style: { - stroke-width: 2 - bold: true - } +writers: "Durable write path" { + style.fill: transparent + admission: "Bounded group admission\nconfigured row target + 20 ms window" {shape: rectangle} + workers: "Up to four commit workers\nindependent batches encode in parallel" {shape: rectangle} + stage: "Staging batch directory\nmetadata + signal Parquet + trace.fidx for spans" {shape: rectangle} + publish: "Durable publication\nfsync → atomic rename → fsync parent" {shape: rectangle} + + admission -> workers + workers -> stage + stage -> publish } -telemetry: "Parquet batches + trace sidecars\nstorage.data_dir/telemetry" { +authority: "Authoritative telemetry\nimmutable *.batch directories" { shape: cylinder - tooltip: Authoritative telemetry and its crash-recovery state. + tooltip: Ordinary Parquet files are the source of truth; the filesystem is the catalog. } -querystate: "DuckDB query state\nstorage.data_dir/query" { - shape: cylinder - tooltip: Rebuildable rollups and temp spill. Not the telemetry itself. +readers: Readers { + style.fill: transparent + trace: "Indexed trace read\nbinary-search .fidx → Parquet rows" {shape: rectangle} + sql: "DuckDB SQL\nfilters, ordering, broad scans" {shape: rectangle} + rollups: "DuckDB rollups\nrebuildable analytical cache" {shape: rectangle} } -control: "Control SQLite\nstorage.data_dir/control/fanout.sqlite" { - shape: cylinder - tooltip: Application state. Never on the telemetry write path. +maintenance: "Bounded maintenance" { + style.fill: transparent + choose: "Retention + compaction admission\nbounded batch and row groups" {shape: rectangle} + merge: "Native Parquet merge\nprepared outside the reader gate" {shape: rectangle} + swap: "Crash-safe namespace swap\nmarker + bounded renames + fsync" {shape: rectangle} + + choose -> merge + merge -> swap } -control_tables: "users, user_identities, verifications, sessions, auth_audit_events\noauth_clients, oauth_tokens, oauth_authorization_codes\ndashboards, dashboard_widgets, dashboard_state\nagui_threads, agui_runs, alert_rules, alerts, settings" { - shape: text - style.font-size: 13 +querystate: "Rebuildable query state\nstorage.data_dir/query" { + shape: cylinder + tooltip: DuckDB rollups and temporary spill. Not authoritative telemetry. } -writers.ingest -> commit -commit -> telemetry: "atomic directory rename" -writers.rollup -> querystate -writers.maint -> telemetry -telemetry <- querystate: "DuckDB scans Parquet" -control -> control_tables: {style.stroke-dash: 3} +ingest -> writers.admission +writers.publish -> authority: "acknowledge only after publication" +authority -> readers.trace +authority -> readers.sql +readers.sql -> readers.rollups +readers.rollups -> querystate +authority -> maintenance.choose diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 17f7976a..e1c636dc 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,27 +1,27 @@ -WritersParallel atomic Parquet commitsacknowledged after directory publicationParquet batches + trace sidecarsstorage.data_dir/telemetryAuthoritative telemetry and its crash-recovery state.DuckDB query statestorage.data_dir/queryRebuildable rollups and temp spill. Not the telemetry itself.Control SQLitestorage.data_dir/control/fanout.sqliteApplication state. Never on the telemetry write path.users, user_identities, verifications, sessions, auth_audit_eventsoauth_clients, oauth_tokens, oauth_authorization_codesdashboards, dashboard_widgets, dashboard_stateagui_threads, agui_runs, alert_rules, alerts, settingsOTLP requestspans, logs, metricsRollupsservice, endpoint, edgeMerge and maintenance atomic directory rename DuckDB scans Parquet Authoritative telemetry and its crash-recovery state. - + .d2-2487203756 .fill-N1{fill:#0A0F25;} + .d2-2487203756 .fill-N2{fill:#676C7E;} + .d2-2487203756 .fill-N3{fill:#9499AB;} + .d2-2487203756 .fill-N4{fill:#CFD2DD;} + .d2-2487203756 .fill-N5{fill:#DEE1EB;} + .d2-2487203756 .fill-N6{fill:#EEF1F8;} + .d2-2487203756 .fill-N7{fill:#FFFFFF;} + .d2-2487203756 .fill-B1{fill:#000536;} + .d2-2487203756 .fill-B2{fill:#0F66B7;} + .d2-2487203756 .fill-B3{fill:#4393DD;} + .d2-2487203756 .fill-B4{fill:#87BFF3;} + .d2-2487203756 .fill-B5{fill:#BCDDFB;} + .d2-2487203756 .fill-B6{fill:#E5F3FF;} + .d2-2487203756 .fill-AA2{fill:#7639C5;} + .d2-2487203756 .fill-AA4{fill:#C1A2F3;} + .d2-2487203756 .fill-AA5{fill:#DACEFB;} + .d2-2487203756 .fill-AB4{fill:#EA99C6;} + .d2-2487203756 .fill-AB5{fill:#FFDEF1;} + .d2-2487203756 .stroke-N1{stroke:#0A0F25;} + .d2-2487203756 .stroke-N2{stroke:#676C7E;} + .d2-2487203756 .stroke-N3{stroke:#9499AB;} + .d2-2487203756 .stroke-N4{stroke:#CFD2DD;} + .d2-2487203756 .stroke-N5{stroke:#DEE1EB;} + .d2-2487203756 .stroke-N6{stroke:#EEF1F8;} + .d2-2487203756 .stroke-N7{stroke:#FFFFFF;} + .d2-2487203756 .stroke-B1{stroke:#000536;} + .d2-2487203756 .stroke-B2{stroke:#0F66B7;} + .d2-2487203756 .stroke-B3{stroke:#4393DD;} + .d2-2487203756 .stroke-B4{stroke:#87BFF3;} + .d2-2487203756 .stroke-B5{stroke:#BCDDFB;} + .d2-2487203756 .stroke-B6{stroke:#E5F3FF;} + .d2-2487203756 .stroke-AA2{stroke:#7639C5;} + .d2-2487203756 .stroke-AA4{stroke:#C1A2F3;} + .d2-2487203756 .stroke-AA5{stroke:#DACEFB;} + .d2-2487203756 .stroke-AB4{stroke:#EA99C6;} + .d2-2487203756 .stroke-AB5{stroke:#FFDEF1;} + .d2-2487203756 .background-color-N1{background-color:#0A0F25;} + .d2-2487203756 .background-color-N2{background-color:#676C7E;} + .d2-2487203756 .background-color-N3{background-color:#9499AB;} + .d2-2487203756 .background-color-N4{background-color:#CFD2DD;} + .d2-2487203756 .background-color-N5{background-color:#DEE1EB;} + .d2-2487203756 .background-color-N6{background-color:#EEF1F8;} + .d2-2487203756 .background-color-N7{background-color:#FFFFFF;} + .d2-2487203756 .background-color-B1{background-color:#000536;} + .d2-2487203756 .background-color-B2{background-color:#0F66B7;} + .d2-2487203756 .background-color-B3{background-color:#4393DD;} + .d2-2487203756 .background-color-B4{background-color:#87BFF3;} + .d2-2487203756 .background-color-B5{background-color:#BCDDFB;} + .d2-2487203756 .background-color-B6{background-color:#E5F3FF;} + .d2-2487203756 .background-color-AA2{background-color:#7639C5;} + .d2-2487203756 .background-color-AA4{background-color:#C1A2F3;} + .d2-2487203756 .background-color-AA5{background-color:#DACEFB;} + .d2-2487203756 .background-color-AB4{background-color:#EA99C6;} + .d2-2487203756 .background-color-AB5{background-color:#FFDEF1;} + .d2-2487203756 .color-N1{color:#0A0F25;} + .d2-2487203756 .color-N2{color:#676C7E;} + .d2-2487203756 .color-N3{color:#9499AB;} + .d2-2487203756 .color-N4{color:#CFD2DD;} + .d2-2487203756 .color-N5{color:#DEE1EB;} + .d2-2487203756 .color-N6{color:#EEF1F8;} + .d2-2487203756 .color-N7{color:#FFFFFF;} + .d2-2487203756 .color-B1{color:#000536;} + .d2-2487203756 .color-B2{color:#0F66B7;} + .d2-2487203756 .color-B3{color:#4393DD;} + .d2-2487203756 .color-B4{color:#87BFF3;} + .d2-2487203756 .color-B5{color:#BCDDFB;} + .d2-2487203756 .color-B6{color:#E5F3FF;} + .d2-2487203756 .color-AA2{color:#7639C5;} + .d2-2487203756 .color-AA4{color:#C1A2F3;} + .d2-2487203756 .color-AA5{color:#DACEFB;} + .d2-2487203756 .color-AB4{color:#EA99C6;} + .d2-2487203756 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2487203756);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2487203756);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2487203756);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2487203756);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2487203756);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2487203756);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2487203756);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2487203756);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2487203756);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2487203756);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2487203756);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2487203756);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are the source of truthReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentthe filesystem is the catalogIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are the source of truth + - + -Rebuildable rollups and temp spill. Not the telemetry itself. - +DuckDB rollups and temporary spill. Not authoritative telemetry. + - + -Application state. Never on the telemetry write path. - - - - - - - - - - - - - - - - + + + diff --git a/justfile b/justfile index 90d5e12c..c7734b4a 100644 --- a/justfile +++ b/justfile @@ -184,21 +184,31 @@ ui-check: # and `check`: the SVG is committed, so only someone editing a diagram needs d2 # installed at all. # -# The committed SVG was produced by d2 0.7.1. d2's output changes between minor -# versions, so a different version re-renders every file and produces a large -# diff that looks like a change but is not — check `d2 --version` before -# committing one. +# D2 output changes between minor versions, so the recipe pins the generator +# used for committed SVGs instead of depending on a contributor's installed +# binary. -# Render every d2 diagram to SVG. +# Render every d2 diagram to SVG and publish the same assets on the docs site. diagrams: #!/usr/bin/env bash set -euo pipefail - if ! command -v d2 >/dev/null; then - echo "d2 not found — install with: brew install d2" >&2 - exit 1 - fi for src in docs/diagrams/*.d2; do - d2 "$src" "${src%.d2}.svg" + go run oss.terrastruct.com/d2@v0.7.1 "$src" "${src%.d2}.svg" + done + mkdir -p site/public/diagrams + cp docs/diagrams/*.svg site/public/diagrams/ + +# Fail when the site serves a diagram other than the committed documentation +# copy. Regeneration stays a deliberate writing step. +diagrams-check: + #!/usr/bin/env bash + set -euo pipefail + for src in docs/diagrams/*.svg; do + dst="site/public/diagrams/$(basename "$src")" + if ! cmp -s "$src" "$dst"; then + echo "$dst does not match $src; run 'just diagrams'" >&2 + exit 1 + fi done # ── Documentation site ─────────────────────────────────────────────────────── @@ -233,7 +243,7 @@ site: docs-generate # a gate that rewrites the tree it is inspecting cannot tell you whether the # tree was already right. Use `just site` for the writing path during # development. -site-build: docs-generate-check site-deps +site-build: docs-generate-check diagrams-check site-deps cd site && npm run build # Renders the social preview card from docs/media/social-card.typ into diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg new file mode 100644 index 00000000..6f054670 --- /dev/null +++ b/site/public/diagrams/architecture.svg @@ -0,0 +1,121 @@ +Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC :4317 + HTTP :4318Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryQuery catalogstorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + + + + + + + + + + + + + + + diff --git a/site/public/diagrams/persistence.svg b/site/public/diagrams/persistence.svg new file mode 100644 index 00000000..e1c636dc --- /dev/null +++ b/site/public/diagrams/persistence.svg @@ -0,0 +1,138 @@ +Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are the source of truthReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentthe filesystem is the catalogIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are the source of truth + + + + + + + + + + + + +DuckDB rollups and temporary spill. Not authoritative telemetry. + + + + + + + + + + + + + + + + diff --git a/site/src/content/docs/explanation/performance.mdx b/site/src/content/docs/explanation/performance.mdx index e378124e..a78da39f 100644 --- a/site/src/content/docs/explanation/performance.mdx +++ b/site/src/content/docs/explanation/performance.mdx @@ -40,9 +40,12 @@ Before any figure goes back on this page, it has to ship with: `cmd/bench` is the supported load generator. It sends traces, metrics and logs, ramps to find the ingest boundary, and confirms the result at the rate it found. -It can add authenticated dashboard reads on top, which is the part that matters: -ingest and query contend for the same write gate, so an ingest-only number -overstates what an instance does while anyone is looking at it. +It can add authenticated dashboard reads on top, which is the part that matters. +Parquet ingest and DuckDB reads no longer serialize through one write gate, but +they still compete for the same CPU, memory bandwidth, filesystem cache, and +disk. Maintenance adds native Parquet merge work and short reader-exclusive +namespace changes. An ingest-only number therefore still overstates what an +instance does while anyone is looking at it. Build it, create an ingest token through first-admin setup, then: diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index c59989bd..3d71ac76 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -22,18 +22,32 @@ state—users, sessions, dashboards, alert rules, and agent history—lives in S There is no Iceberg, DuckLake, external catalog, or server database in the telemetry path. +
+ +
Fanout persistence flow: authenticated OTLP requests enter bounded group admission and parallel commit workers, publish immutable Parquet batches atomically, then serve indexed trace reads and DuckDB SQL while native maintenance prepares and swaps compacted replacements. + +
Parquet is authoritative; indexes and DuckDB rollups are derived state.
+ + ## Writes are durable immediately -Concurrent small OTLP requests may share one bounded Parquet batch. Up to four +Concurrent small OTLP requests may share one group-commit batch. Up to four commit workers encode independent batches in parallel. Each worker writes all -present signals, metadata, and the trace index into a staging directory, fsyncs -them, and publishes the complete directory with one atomic rename. Fanout +present signals and metadata into a staging directory, adds `trace.fidx` when +spans are present, fsyncs them, and publishes the complete directory with one +atomic rename. Fanout returns success only after every batch belonging to the request is published. A crash before the rename leaves only an unacknowledged staging directory, which startup removes. A crash after the rename leaves a complete batch that startup discovers directly from the filesystem. There is no separate WAL, -manifest, catalog, timer-based flush window, or acknowledged memory-only state. +manifest, catalog, post-acknowledgement flush timer, or acknowledged memory-only +state. ## One authoritative copy for logs and metrics @@ -49,11 +63,16 @@ stores only transactional product state. ## Small files are bounded by compaction Atomic ingestion creates immutable batch directories. Maintenance drains every -eligible compaction group, combines files within bounded day/generation levels, -builds a replacement trace index, and atomically swaps the replacement for its -inputs. A durable compaction marker makes an interrupted swap resumable. Active -DuckDB readers pin immutable files while retention or compaction removes them, -so reads never lose an open input file. +eligible compaction group, choosing inputs by both count and a 25-million-row +ceiling. It combines same-day, same-generation files, builds a replacement trace +index, and atomically swaps the replacement for its inputs. A durable compaction +marker makes an interrupted swap resumable. Active DuckDB readers pin immutable +files while retention or compaction removes them, so reads never lose an open +input file. + +The expensive merge runs before readers are excluded. Only the bounded directory +swap takes the exclusive reader gate; new ingest batches continue encoding while +that work is prepared. Retention is based on ingestion time rather than event time. A service with a bad clock therefore cannot pin a batch on disk by emitting a far-future event. diff --git a/site/src/content/docs/explanation/why-one-binary.mdx b/site/src/content/docs/explanation/why-one-binary.mdx index d18d6097..c638d9a5 100644 --- a/site/src/content/docs/explanation/why-one-binary.mdx +++ b/site/src/content/docs/explanation/why-one-binary.mdx @@ -38,10 +38,13 @@ loader binds. a machine. There is no scaling the query layer without also scaling ingest, because they are the same thing. -**They contend.** Maintenance and rollups serialise against ingest through one -write gate. On a busy instance, compacting harder is not free — it is traded -against ingest headroom. [Tuning retention](/guides/tune-retention) is mostly -about managing that trade. +**They contend for one machine.** Parquet ingest, DuckDB reads, rollups, and +native compaction have separate coordination paths, but they consume the same +CPU, memory bandwidth, filesystem cache, and disk. Maintenance excludes readers +only for bounded namespace changes—compaction swaps and retention removals; the +merge itself runs outside that gate. +Compacting harder is still traded against ingest and query headroom. [Tuning +retention](/guides/tune-retention) is mostly about managing that trade. **One process is one failure domain.** Nothing degrades independently. A process that dies takes ingest, query, alerting and the UI with it. diff --git a/site/src/content/docs/guides/tune-retention.mdx b/site/src/content/docs/guides/tune-retention.mdx index 722cbb0e..4c4b1494 100644 --- a/site/src/content/docs/guides/tune-retention.mdx +++ b/site/src/content/docs/guides/tune-retention.mdx @@ -30,26 +30,28 @@ FANOUT_MAINTENANCE_INTERVAL=1h This is the expensive pass: retention deletes plus full compaction. Lower it to reclaim space sooner, at the cost of running the heavy work more often. -## Publication batching +## Group-commit target -The maximum number of telemetry rows in one atomic Parquet batch: +The target number of telemetry rows that concurrent small requests may share in +one atomic Parquet batch: ```sh FANOUT_INGEST_BATCH_SIZE=50000 ``` -A larger batch can improve sustained write throughput and create fewer files -under concurrent ingest. Every request is still acknowledged only after all of -its Parquet batches are durably published; this setting does not create a timer -window or put accepted telemetry at risk. Requests larger than the limit are -split into bounded atomic batches. +A larger target can improve sustained write throughput and create fewer files +under concurrent ingest. Every request is still acknowledged only after its +Parquet batch is durably published. An individual request is never split merely +to satisfy this target: an oversized request remains one atomic batch. The +setting controls admission grouping, not the durability boundary. ## What the knobs interact with -Maintenance briefly gates readers while it swaps or removes immutable files. -New batches continue encoding independently and need only the short publication -lock. Running maintenance much more often still creates extra disk and CPU -work, so change one setting at a time and watch `/-/metrics`. +Maintenance prepares merges while reads continue, then briefly gates readers +while it swaps or removes immutable files. New batches keep encoding and need +only the short publication lock. Ingest, queries, and maintenance still share +CPU, memory bandwidth, and disk, so running maintenance much more often is not +free. Change one setting at a time and watch `/-/metrics`. The full list, with defaults and types, is in the [storage settings](/reference/settings/storage) reference. diff --git a/site/src/content/docs/reference/data-layout.mdx b/site/src/content/docs/reference/data-layout.mdx index 6ec3011a..2b547e8a 100644 --- a/site/src/content/docs/reference/data-layout.mdx +++ b/site/src/content/docs/reference/data-layout.mdx @@ -13,8 +13,10 @@ Everything Fanout persists lives under `FANOUT_DATA_DIR` (`./data` by default, | Path | Holds | |---|---| -| `telemetry/parquet/batches/*.batch/` | Atomic batches containing Parquet signals, metadata, and a trace index | +| `telemetry/parquet/batches/*.batch/` | Authoritative atomic batches containing `metadata.json`, present signal Parquet files, and `trace.fidx` when spans are present | | `telemetry/parquet/batches/_schema.batch/` | Empty schema anchors that keep every DuckDB view queryable | +| `telemetry/parquet/batches/*.retired-*/` | Compaction inputs retained temporarily for crash rollback; a live marker protects its own set | +| `telemetry/parquet/batches/*.quarantined-*/` | Unreadable batches explicitly set aside by `fanout repair quarantine`; preserved but not queryable | | `telemetry/parquet/staging/` | Unacknowledged writes being prepared for atomic publication | | `telemetry/compaction/` | Prepared replacement batches during compaction | | `telemetry/COMPACTION.json` | Durable marker for an interrupted compaction swap. Present only while one is unresolved, and blocks startup until it is — see [troubleshoot](/guides/troubleshoot) | @@ -25,9 +27,10 @@ Everything Fanout persists lives under `FANOUT_DATA_DIR` (`./data` by default, ## Why it is one unit Each published Parquet batch directory is a self-contained crash-recovery unit. -Its trace sidecar is an index over the batch's span file, while DuckDB rollups -can be rebuilt. Keeping the complete data directory is the supported and fastest -restore path. +Its metadata records row counts and time bounds; its fixed-width trace sidecar +maps hashes to sorted span row ranges. The sidecar is verified against the +Parquet rows by offline repair, while DuckDB rollups can be rebuilt. Keeping the +complete data directory is the supported and fastest restore path. `control/fanout.sqlite` is independent of both in format but not in meaning: it holds the dashboards and alert rules that refer to the telemetry, and the diff --git a/site/src/content/docs/start/what-fanout-is.mdx b/site/src/content/docs/start/what-fanout-is.mdx index bcc27be0..e572f721 100644 --- a/site/src/content/docs/start/what-fanout-is.mdx +++ b/site/src/content/docs/start/what-fanout-is.mdx @@ -13,6 +13,18 @@ OTLP over gRPC and HTTP, writes telemetry to disk as Parquet, answers queries with DuckDB, evaluates alert rules, serves a chat investigator and MCP tools, and hosts the browser client. +
+ + Fanout architecture: collectors, browsers, and MCP hosts connect to one process containing ingest, query, maintenance, alerts, agent, and UI components backed by Parquet, DuckDB query state, and SQLite control state. + +
One process owns the complete path from OTLP ingest to investigation.
+
+ The point of that list is that it is one list. A conventional deployment of the same capability is a collector, a storage backend, a query layer, a dashboard service and an alertmanager, each with its own configuration, failure modes and @@ -24,8 +36,8 @@ restart. | Part | What it does | |---|---| | OTLP ingest | Accepts traces, logs and metrics on gRPC and HTTP | -| Storage | Writes Parquet to a local data directory; no external database | -| Query | DuckDB in-process, over the Parquet it wrote | +| Storage | Atomically publishes indexed Parquet batches to a local data directory | +| Query | Direct indexed trace reads plus in-process DuckDB SQL and rollups | | Alerts | Evaluates persisted rules on a fixed interval against rollups | | Investigator | A chat agent working from the same typed query tools | | MCP server | Exposes those tools to an external agent | diff --git a/site/src/styles/fanout.css b/site/src/styles/fanout.css index 25ccaf78..47890e3c 100644 --- a/site/src/styles/fanout.css +++ b/site/src/styles/fanout.css @@ -186,6 +186,50 @@ h1#_top { outline-offset: 2px; } +/* Architecture diagrams are generated for a light technical canvas. Give them + one explicit blueprint-like media surface in both themes rather than letting + dark mode erase their ink. The border carries hierarchy; no decorative + shadow or gradient competes with the diagram itself. */ +.sl-markdown-content .docs-diagram { + margin: 1.75rem 0 2rem; + padding: clamp(0.75rem, 2vw, 1.25rem); + border: 1px solid var(--sl-color-gray-5); + border-radius: 0.5rem; + background: #fafafa; +} + +.sl-markdown-content .docs-diagram__asset { + display: block; + border-radius: 0.25rem; +} + +.sl-markdown-content .docs-diagram__asset:focus-visible { + outline: 2px solid var(--sl-color-accent); + outline-offset: 4px; +} + +.sl-markdown-content .docs-diagram img { + display: block; + width: 100%; + height: auto; + margin: 0 auto; +} + +.sl-markdown-content .docs-diagram figcaption { + margin-top: 0.75rem; + color: #565b69; + font-family: var(--fo-font-display); + font-size: 0.75rem; + line-height: 1.5; + text-align: center; +} + +@media (max-width: 50rem) { + .sl-markdown-content .docs-diagram { + margin-inline: 0; + } +} + /* Without a minimum, a four-column reference table compresses to fit a phone and every column becomes a ribbon of wrapped text. The minimum makes it overflow instead, which is what the scroll container is there to absorb.