From 054bb5ea2adec1a8b78e9ddf1d5ad870bba940d0 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Mon, 19 Oct 2026 15:35:42 +0000 Subject: [PATCH 01/13] Revert "Remove telemetry (#6431)" This reverts commit a2dd31c1d2e5caefa5ab628c065d03733d798f2c. Conflict resolutions against current main: - Cargo.toml files: keep quickwit-transport, quickwit-telemetry-exporters and procfs; add quickwit-telemetry back. The workspace no longer has a default-members list, so only members and workspace.dependencies change. - quickwit-cli/src/main.rs: keep parse_cli_command() and have it call the restored about_text(). - quickwit-cli/src/service.rs: keep the SIGHUP TLS reload. The service override now happens in load_node_config(), so only the telemetry loop and RunCommand/EndCommand events are restored. - Drop the datafusion-ci workflow, datafusion sandbox and metrics_distributed_tests changes: those files were removed in #6809. - Set QW_DISABLE_TELEMETRY in full-tests.yml (added in #6718), since make test-all runs RunCliCommand::execute from quickwit-cli tests. - Cargo.lock: only adds the telemetry crate and its dependencies, with no other re-resolution. --- .github/workflows/ci.yml | 1 + .github/workflows/coverage.yml | 1 + .github/workflows/full-tests.yml | 2 + LICENSE-3rdparty.csv | 3 + docs/reference/cli.md | 8 + docs/telemetry.md | 44 ++ quickwit/Cargo.lock | 66 ++- quickwit/Cargo.toml | 2 + quickwit/quickwit-cli/Cargo.toml | 1 + quickwit/quickwit-cli/src/main.rs | 14 +- quickwit/quickwit-cli/src/service.rs | 45 +- quickwit/quickwit-serve/Cargo.toml | 1 + quickwit/quickwit-serve/src/ui_handler.rs | 4 + quickwit/quickwit-telemetry/Cargo.toml | 35 ++ quickwit/quickwit-telemetry/src/lib.rs | 65 +++ quickwit/quickwit-telemetry/src/payload.rs | 164 +++++++ quickwit/quickwit-telemetry/src/sender.rs | 469 +++++++++++++++++++++ quickwit/quickwit-telemetry/src/sink.rs | 75 ++++ 18 files changed, 988 insertions(+), 12 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 quickwit/quickwit-telemetry/Cargo.toml create mode 100644 quickwit/quickwit-telemetry/src/lib.rs create mode 100644 quickwit/quickwit-telemetry/src/payload.rs create mode 100644 quickwit/quickwit-telemetry/src/sender.rs create mode 100644 quickwit/quickwit-telemetry/src/sink.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d02227842cc..21a8705d322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ permissions: env: CARGO_INCREMENTAL: 0 + QW_DISABLE_TELEMETRY: 1 QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev RUST_BACKTRACE: 1 RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index dd45a377897..703f9a67040 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,6 +16,7 @@ env: AWS_SECRET_ACCESS_KEY: "placeholder" CARGO_INCREMENTAL: 0 PUBSUB_EMULATOR_HOST: "localhost:8681" + QW_DISABLE_TELEMETRY: 1 QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container. QW_S3_FORCE_PATH_STYLE_ACCESS: 1 QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev diff --git a/.github/workflows/full-tests.yml b/.github/workflows/full-tests.yml index 101d6277fff..6d8a0e63f01 100644 --- a/.github/workflows/full-tests.yml +++ b/.github/workflows/full-tests.yml @@ -55,6 +55,8 @@ jobs: timeout-minutes: 60 permissions: contents: read + env: + QW_DISABLE_TELEMETRY: 1 steps: - name: Free disk space run: | diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index bc99a4da800..698238bb139 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2,6 +2,7 @@ Component,Origin,License,Copyright RustyXML,https://github.com/Florob/RustyXML,MIT OR Apache-2.0,Florian Zeitz addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line Authors adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " +advapi32-sys,https://github.com/retep998/winapi-rs,MIT,Peter Atashian aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers aes,https://github.com/RustCrypto/block-ciphers,MIT OR Apache-2.0,RustCrypto Developers aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Developers @@ -828,6 +829,7 @@ unwrap-infallible,https://github.com/mzabaluev/unwrap-infallible,MIT OR Apache-2 ureq-proto,https://github.com/algesten/ureq-proto,MIT OR Apache-2.0,Martin Algesten url,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers urlencoding,https://github.com/kornelski/rust_urlencoding,MIT,"Kornel , Bertram Truong " +username,https://pijul.org/darcs/user,MIT OR Apache-2.0,Pierre-Étienne Meunier utf8-ranges,https://github.com/BurntSushi/utf8-ranges,Unlicense OR MIT,Andrew Gallant utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8-zero,https://github.com/algesten/utf8-zero,MIT OR Apache-2.0,"Simon Sapin , Martin Algesten " @@ -864,6 +866,7 @@ web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Aut webpki-root-certs,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-root-certs Authors webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors +winapi,https://github.com/retep998/winapi-rs,MIT,Peter Atashian winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant diff --git a/docs/reference/cli.md b/docs/reference/cli.md index e4cbaa5e1e3..ad14b3a5fbc 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -855,6 +855,14 @@ Specifies the path to the [quickwit config](../configuration/node-config.md). Co `export QW_CONFIG=config/quickwit.yaml` +### QW_DISABLE_TELEMETRY + +Disables [telemetry](../telemetry.md) when set to any non-empty value. + +*Example* + +`QW_DISABLE_TELEMETRY=1 quickwit help` + ### QW_POSTGRES_SKIP_MIGRATIONS Don't run database migrations (but verify that migrations were run successfully before, and no that unknown migration was run). diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 00000000000..21ec1aef219 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,44 @@ +--- +title: Telemetry +sidebar_position: 12 +--- + +Quickwit, Inc. collects anonymous data regarding general usage to help us drive our development. Privacy and transparency are at the heart of Quickwit values and we only collect the minimal useful data and don't use any third party tool for the collection. + +## Disabling data collection + +Data collection are opt-out. To disable them, just set the environment variable `QW_DISABLE_TELEMETRY` to whatever value. + +```bash +export QW_DISABLE_TELEMETRY=1 +``` + +Look at `--help` command output to check whether telemetry is enabled or not: +```bash +quickwit --help +Quickwit 0.7 +Sub-second search & analytics engine on cloud storage. + Find more information at https://quickwit.io/docs + +Telemetry enabled +``` + +The line `Telemetry enabled` disappears when you disable it. + +## Which data are collected? + +We collect the minimum amount of information to respect privacy. Here are the data collected: +- type of events among create, index, delete and serve events +- client information: + - session uuid: uuid generated on the fly + - quickwit version + - os (linux, macos, freebsd, android...) + - architecture of the CPU + - md5 hash of host and username + - a boolean to know if `KUBERNETES_SERVICE_HOST` is set. + +All data are sent to `telemetry.quickwit.io`. + +## No third party + +We did not want to add any untrusted third party tool in the collection so we decided to implement and host our own metric collection server. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 601594bdcda..cdb5db4cc91 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -23,6 +23,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "advapi32-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "aead" version = "0.5.2" @@ -3170,7 +3180,7 @@ dependencies = [ "cc", "lazy_static", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3317,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6031,7 +6041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6442,7 +6452,7 @@ dependencies = [ "libc", "pnet_base", "pnet_sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6485,7 +6495,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d4643d3d4db6b08741050c2f3afa9a892c4244c085a72fcda93c9c2c9a00f4b" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -7041,7 +7051,7 @@ dependencies = [ "raw-cpuid", "wasi 0.11.1+wasi-snapshot-preview1", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -7175,6 +7185,7 @@ dependencies = [ "quickwit-search", "quickwit-serve", "quickwit-storage", + "quickwit-telemetry", "quickwit-telemetry-exporters", "quickwit-transport", "reqwest 0.12.28", @@ -8037,6 +8048,7 @@ dependencies = [ "quickwit-query", "quickwit-search", "quickwit-storage", + "quickwit-telemetry", "quickwit-telemetry-exporters", "quickwit-transport", "regex", @@ -8125,6 +8137,24 @@ dependencies = [ "ulid", ] +[[package]] +name = "quickwit-telemetry" +version = "0.9.1" +dependencies = [ + "async-trait", + "encoding_rs", + "hostname", + "md5", + "quickwit-common", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "tracing", + "username", + "uuid", +] + [[package]] name = "quickwit-telemetry-exporters" version = "0.9.1" @@ -10357,7 +10387,7 @@ dependencies = [ "typetag", "unwrap-infallible", "uuid", - "winapi", + "winapi 0.3.9", "zstd 0.14.0", ] @@ -11383,6 +11413,16 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "username" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e425df6527f7bc1adc7eb3b829ecaec746fbbc0b05e42133ff84afef3b1a09" +dependencies = [ + "advapi32-sys", + "winapi 0.2.8", +] + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -11874,6 +11914,12 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -11884,6 +11930,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index a58d5f3797f..9427e024fa1 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -37,6 +37,7 @@ members = [ "quickwit-search", "quickwit-serve", "quickwit-storage", + "quickwit-telemetry", "quickwit-telemetry-exporters", "quickwit-transport", ] @@ -375,6 +376,7 @@ quickwit-rest-client = { path = "quickwit-rest-client" } quickwit-search = { path = "quickwit-search" } quickwit-serve = { path = "quickwit-serve" } quickwit-storage = { path = "quickwit-storage" } +quickwit-telemetry = { path = "quickwit-telemetry" } quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" } quickwit-transport = { path = "quickwit-transport" } diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml index 5257d6f5c85..41733db7b6b 100644 --- a/quickwit/quickwit-cli/Cargo.toml +++ b/quickwit/quickwit-cli/Cargo.toml @@ -65,6 +65,7 @@ quickwit-rest-client = { workspace = true } quickwit-search = { workspace = true } quickwit-serve = { workspace = true } quickwit-storage = { workspace = true } +quickwit-telemetry = { workspace = true } quickwit-transport = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index 0af72990277..181ae89da15 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -64,8 +64,7 @@ fn main() -> anyhow::Result<()> { } fn parse_cli_command() -> (CliCommand, bool) { - let about_text = "Sub-second search & analytics engine on cloud storage.\n Find more \ - information at https://quickwit.io/docs\n\n"; + let about_text = about_text(); let version_text = BuildInfo::get_version_text(); let app = build_cli().about(about_text).version(version_text); @@ -151,6 +150,17 @@ async fn main_impl() -> anyhow::Result<()> { std::process::exit(return_code) } +/// Return the about text with telemetry info. +fn about_text() -> String { + let mut about_text = String::from( + "Sub-second search & analytics engine on cloud storage.\n Find more information at https://quickwit.io/docs\n\n", + ); + if !quickwit_telemetry::is_telemetry_disabled() { + about_text += "Telemetry: enabled"; + } + about_text +} + #[cfg(test)] mod tests { use std::str::FromStr; diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index 04b5762db54..e1c6e1ddab0 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -21,10 +21,12 @@ use colored::Colorize; use futures::future::select; use itertools::Itertools; use quickwit_common::runtimes::RuntimesConfig; -use quickwit_common::uri::Uri; +use quickwit_common::uri::{Protocol, Uri}; +use quickwit_config::NodeConfig; use quickwit_config::service::QuickwitService; use quickwit_serve::tcp_listener::DefaultTcpListenerResolver; use quickwit_serve::{BuildInfo, EnvFilterReloadFn, reload_tls_cert, serve_quickwit}; +use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent}; use tokio::signal; use tracing::{debug, info}; @@ -124,6 +126,9 @@ impl RunCliCommand { let (storage_resolver, metastore_resolver) = get_resolvers(&node_config.storage_configs, &node_config.metastore_configs); crate::busy_detector::set_enabled(true); + let telemetry_handle_opt = + quickwit_telemetry::start_telemetry_loop(quickwit_telemetry_info(&node_config)); + quickwit_telemetry::send_telemetry_event(TelemetryEvent::RunCommand).await; // TODO move in serve quickwit? let runtimes_config = RuntimesConfig::default(); start_actor_runtimes(runtimes_config, &node_config.enabled_services)?; @@ -132,7 +137,7 @@ impl RunCliCommand { }); // Reload TLS certificates on SIGHUP for the lifetime of the process. tokio::spawn(listen_sighup()); - serve_quickwit( + let serve_result = serve_quickwit( node_config, runtimes_config, metastore_resolver, @@ -141,12 +146,46 @@ impl RunCliCommand { shutdown_signal, env_filter_reload_fn, ) - .await?; + .await; + let return_code = match serve_result { + Ok(_) => 0, + Err(_) => 1, + }; + quickwit_telemetry::send_telemetry_event(TelemetryEvent::EndCommand { return_code }).await; + if let Some(telemetry_handle) = telemetry_handle_opt { + telemetry_handle.terminate_telemetry().await; + } + serve_result?; info!("quickwit successfully terminated"); Ok(()) } } +fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { + let mut features = HashSet::new(); + if config.indexer_config.enable_otlp_endpoint { + features.insert(QuickwitFeature::Otlp); + } + if config.jaeger_config.enable_endpoint { + features.insert(QuickwitFeature::Jaeger); + } + // The metastore URI is only relevant if the metastore is enabled. + if config.is_service_enabled(QuickwitService::Metastore) { + let feature = if config.metastore_uri.protocol() == Protocol::PostgreSQL { + QuickwitFeature::PostgresqMetastore + } else { + QuickwitFeature::FileBackedMetastore + }; + features.insert(feature); + } + let services = config + .enabled_services + .iter() + .map(|service| service.to_string()) + .collect(); + QuickwitTelemetryInfo::new(services, features) +} + #[cfg(test)] mod tests { diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml index 0d554ffc2ee..a09676ac03b 100644 --- a/quickwit/quickwit-serve/Cargo.toml +++ b/quickwit/quickwit-serve/Cargo.toml @@ -76,6 +76,7 @@ quickwit-proto = { workspace = true } quickwit-query = { workspace = true } quickwit-search = { workspace = true } quickwit-storage = { workspace = true } +quickwit-telemetry = { workspace = true } quickwit-telemetry-exporters = { workspace = true } quickwit-transport = { workspace = true } diff --git a/quickwit/quickwit-serve/src/ui_handler.rs b/quickwit/quickwit-serve/src/ui_handler.rs index f00e95238c2..2cdecec50a6 100644 --- a/quickwit/quickwit-serve/src/ui_handler.rs +++ b/quickwit/quickwit-serve/src/ui_handler.rs @@ -14,6 +14,7 @@ use std::sync::LazyLock; +use quickwit_telemetry::payload::TelemetryEvent; use regex::Regex; use rust_embed::RustEmbed; use warp::hyper::header::HeaderValue; @@ -52,6 +53,9 @@ async fn serve_impl(path: &str) -> Result, Rejection> { } else { // Quickwit UI is a single page application. // Any path request that is not an asset should serve the `index.html` file. + // The client (browser) usually request `index.html` once unless the user refreshes the + // page. + quickwit_telemetry::send_telemetry_event(TelemetryEvent::UiIndexPageLoad).await; UI_INDEX_FILE_NAME }; let asset = Asset::get(path_to_file).ok_or_else(warp::reject::not_found)?; diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml new file mode 100644 index 00000000000..3027fddf8a6 --- /dev/null +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "quickwit-telemetry" +description = "Open Telemetry services" + +version.workspace = true +edition.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +hostname = { workspace = true } +md5 = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +username = { workspace = true } +uuid = { workspace = true } + +# This is actually not used directly the goal is to fix the version +# used by reqwest. 0.8.30 has an unclear license. +encoding_rs = { workspace = true } + +quickwit-common = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } + +[package.metadata.cargo-machete] +# see above +ignored = ["encoding_rs"] diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs new file mode 100644 index 00000000000..72f821005e4 --- /dev/null +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -0,0 +1,65 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(clippy::bool_assert_comparison)] +#![deny(clippy::disallowed_methods)] + +pub mod payload; +/// This crate contains the code responsible for sending usage data to Quickwit inc's server. +mod sender; +pub(crate) mod sink; + +use std::sync::OnceLock; + +use payload::QuickwitTelemetryInfo; +use tracing::info; + +use crate::payload::TelemetryEvent; +pub use crate::sender::is_telemetry_disabled; +use crate::sender::{TelemetryLoopHandle, TelemetrySender}; + +static TELEMETRY_SENDER: OnceLock = OnceLock::new(); + +/// Returns a `TelemetryLoopHandle` if the telemetry loop is not yet started. +pub fn start_telemetry_loop(quickwit_info: QuickwitTelemetryInfo) -> Option { + let telemetry_sender = + TELEMETRY_SENDER.get_or_init(|| TelemetrySender::from_quickwit_info(quickwit_info)); + // This should not happen... unless telemetry is enabled and you are running tests in parallel + // in the same process. + if telemetry_sender.loop_started() { + info!("telemetry loop already started. please disable telemetry during tests"); + return None; + } + Some(telemetry_sender.start_loop()) +} + +/// Sends a telemetry event to Quickwit's server via HTTP. +/// +/// Telemetry guarantees to send at most 1 request per minute. +/// Each requests can ship at most 10 messages. +/// +/// If this methods is called too often, some events will be dropped. +/// +/// If the http requests fail, the error will be silent. +/// +/// We voluntarily use an enum here to make it easier for reader +/// to audit the type of information that is send home. +pub async fn send_telemetry_event(event: TelemetryEvent) { + if let Some(telemetry_sender) = TELEMETRY_SENDER.get() { + telemetry_sender.send(event).await; + } +} + +/// This environment variable can be set to disable sending telemetry events. +pub const DISABLE_TELEMETRY_ENV_KEY: &str = "QW_DISABLE_TELEMETRY"; diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs new file mode 100644 index 00000000000..2b7f484b283 --- /dev/null +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -0,0 +1,164 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; +use std::env; +use std::time::UNIX_EPOCH; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Represents the payload of the request sent with telemetry requests. +#[derive(Debug, Serialize, Deserialize)] +pub struct TelemetryPayload { + /// Client information. See details in `[ClientInformation]`. + pub client_info: ClientInfo, + /// Quickwit information. See details in `[QuickwitInfo]`. + pub quickwit_info: QuickwitTelemetryInfo, + pub events: Vec, + /// Represents the number of events that where drops due to the + /// combination of the `TELEMETRY_PUSH_COOLDOWN` and `MAX_EVENT_IN_QUEUE`. + pub num_dropped_events: usize, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct EventWithTimestamp { + /// Unix time in seconds. + pub unixtime: u64, + /// Telemetry event. + #[serde(flatten)] + pub event: TelemetryEvent, +} + +/// Returns the number of seconds elapsed since UNIX_EPOCH. +/// +/// If the system clock is set before 1970, returns 0. +fn unixtime() -> u64 { + match UNIX_EPOCH.elapsed() { + Ok(duration) => duration.as_secs(), + Err(_) => 0u64, + } +} + +impl From for EventWithTimestamp { + fn from(event: TelemetryEvent) -> Self { + EventWithTimestamp { + unixtime: unixtime(), + event, + } + } +} + +/// Represents a Telemetry Event send to Quickwit's telemetry server for usage information. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum TelemetryEvent { + RunCommand, + /// EndCommand (with the return code). + EndCommand { + return_code: i32, + }, + /// Event sent every 12h to signal the server is running. + Running, + /// UI index.html was requested. + UiIndexPageLoad, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ClientInfo { + session_uuid: uuid::Uuid, + os: String, + arch: String, + hashed_host_username: String, + kubernetes: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct QuickwitTelemetryInfo { + pub version: String, + pub services: HashSet, + pub features: HashSet, +} + +impl QuickwitTelemetryInfo { + pub fn new(services: HashSet, features: HashSet) -> Self { + Self { + features, + version: env!("CARGO_PKG_VERSION").to_string(), + services, + } + } +} + +impl Default for QuickwitTelemetryInfo { + fn default() -> Self { + Self { + features: HashSet::new(), + version: env!("CARGO_PKG_VERSION").to_string(), + services: HashSet::new(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum QuickwitFeature { + FileBackedMetastore, + Jaeger, + Otlp, + PostgresqMetastore, +} + +fn hashed_host_username() -> String { + let hostname = hostname::get() + .map(|hostname| hostname.to_string_lossy().to_string()) + .unwrap_or_default(); + let username = username::get_user_name().unwrap_or_default(); + let hashed_value = format!("{hostname}:{username}"); + let digest = md5::compute(hashed_value.as_bytes()); + format!("{digest:x}") +} + +impl Default for ClientInfo { + fn default() -> ClientInfo { + ClientInfo { + session_uuid: Uuid::new_v4(), + os: env::consts::OS.to_string(), + arch: env::consts::ARCH.to_string(), + hashed_host_username: hashed_host_username(), + kubernetes: std::env::var_os("KUBERNETES_SERVICE_HOST").is_some(), + } + } +} + +#[cfg(test)] +mod tests { + use serde_json; + + use super::{EventWithTimestamp, TelemetryEvent}; + + #[test] + fn test_serialize_payload_as_expected() { + let event = EventWithTimestamp { + unixtime: 0, + event: TelemetryEvent::EndCommand { return_code: 0 }, + }; + let json = serde_json::to_string(&event).unwrap(); + assert_eq!( + json, + r#"{"unixtime":0,"type":"end_command","return_code":0}"# + ); + } +} diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs new file mode 100644 index 00000000000..68c9b5deed4 --- /dev/null +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -0,0 +1,469 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::mem; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::{Mutex, RwLock, oneshot}; +use tokio::task::JoinHandle; +use tokio::time::Interval; +use tracing::info; + +use crate::payload::{ + ClientInfo, EventWithTimestamp, QuickwitTelemetryInfo, TelemetryEvent, TelemetryPayload, +}; +use crate::sink::{HttpClient, Sink}; + +/// At most 1 Request per minutes. +const TELEMETRY_PUSH_COOLDOWN: Duration = Duration::from_secs(60); + +/// Interval at which to send telemetry `Running` event. +const TELEMETRY_RUNNING_EVENT_INTERVAL: Duration = + Duration::from_secs(if cfg!(test) { 3 } else { 60 * 60 * 12 }); // 12h + +/// Upon termination of the program, we send one last telemetry request with pending events. +/// This duration is the amount of time we wait for at most to send that last telemetry request. +const LAST_REQUEST_TIMEOUT: Duration = Duration::from_secs(1); + +const MAX_NUM_EVENTS_IN_QUEUE: usize = 10; + +#[cfg(test)] +struct ClockButton(Sender<()>); + +#[cfg(test)] +impl ClockButton { + async fn tick(&self) { + let _ = self.0.send(()).await; + } +} + +enum Clock { + Periodical(Mutex), + #[cfg(test)] + Manual(Mutex>), +} + +impl Clock { + pub fn periodical(period: Duration) -> Clock { + let interval = tokio::time::interval(period); + Clock::Periodical(Mutex::new(interval)) + } + + #[cfg(test)] + pub async fn manual() -> (ClockButton, Clock) { + let (tx, rx) = tokio::sync::mpsc::channel(1); + let _ = tx.send(()).await; + let button = ClockButton(tx); + (button, Clock::Manual(Mutex::new(rx))) + } + + async fn tick(&self) { + match self { + Clock::Periodical(interval) => { + interval.lock().await.tick().await; + } + #[cfg(test)] + Clock::Manual(channel) => { + channel.lock().await.recv().await; + } + } + } +} + +#[derive(Default)] +struct EventsState { + events: Vec, + num_dropped_events: usize, +} + +impl EventsState { + fn drain_events(&mut self) -> EventsState { + mem::replace( + self, + EventsState { + events: Vec::new(), + num_dropped_events: 0, + }, + ) + } + + /// Adds an event. + /// If the queue is already saturated, (ie. it has reached the len `MAX_NUM_EVENTS_IN_QUEUE`) + // Returns true iff it was the first event in the queue. + fn push_event(&mut self, event: TelemetryEvent) -> bool { + if self.events.len() >= MAX_NUM_EVENTS_IN_QUEUE { + self.num_dropped_events += 1; + return false; + } + let events_was_empty = self.events.is_empty(); + self.events.push(EventWithTimestamp::from(event)); + events_was_empty + } +} + +struct Events { + state: RwLock, + items_available_tx: Sender<()>, + items_available_rx: RwLock>, +} + +impl Default for Events { + fn default() -> Self { + let (items_available_tx, items_available_rx) = tokio::sync::mpsc::channel(1); + Events { + state: RwLock::new(EventsState::default()), + items_available_tx, + items_available_rx: RwLock::new(items_available_rx), + } + } +} + +impl Events { + /// Wait for events to be available (if there are pending events, then do not wait) + /// and then send them to the ingest API server. + async fn drain_events(&self) -> EventsState { + self.items_available_rx.write().await.recv().await; + self.state.write().await.drain_events() + } + + async fn push_event(&self, event: TelemetryEvent) { + let is_first_event = self.state.write().await.push_event(event); + if is_first_event { + let _ = self.items_available_tx.send(()).await; + } + } +} + +pub(crate) struct Inner { + sink: Option>, + client_info: ClientInfo, + quickwit_info: QuickwitTelemetryInfo, + /// This channel is just used to signal there are new items available. + events: Events, + clock: Clock, + is_started: AtomicBool, +} + +impl Inner { + pub fn is_disabled(&self) -> bool { + self.sink.is_none() + } + + async fn create_telemetry_payload(&self) -> TelemetryPayload { + let events_state = self.events.drain_events().await; + TelemetryPayload { + client_info: self.client_info.clone(), + quickwit_info: self.quickwit_info.clone(), + events: events_state.events, + num_dropped_events: events_state.num_dropped_events, + } + } + + /// Wait for events to be available (if there are pending events, then do not wait) + /// and then send them to the ingest API server. + /// + /// If the requests fails, it fails silently. + async fn send_pending_events(&self) { + if let Some(sink) = self.sink.as_ref() { + let payload = self.create_telemetry_payload().await; + sink.send_payload(payload).await; + } + } + + async fn send(&self, event: TelemetryEvent) { + if self.is_disabled() { + return; + } + self.events.push_event(event).await; + } +} + +pub struct TelemetrySender { + pub(crate) inner: Arc, +} + +pub enum TelemetryLoopHandle { + NoLoop, + WithLoop { + join_handle: JoinHandle<()>, + terminate_command_tx: oneshot::Sender<()>, + }, +} + +impl TelemetryLoopHandle { + /// Terminate telemetry will exit the telemetry loop + /// and possibly send the last request, possibly ignoring the + /// telemetry cooldown. + pub async fn terminate_telemetry(self) { + if let Self::WithLoop { + join_handle, + terminate_command_tx, + } = self + { + let _ = terminate_command_tx.send(()); + let _ = tokio::time::timeout(LAST_REQUEST_TIMEOUT, join_handle).await; + } + } +} + +impl TelemetrySender { + pub fn from_quickwit_info(quickwit_info: QuickwitTelemetryInfo) -> Self { + let http_client = create_http_client(); + TelemetrySender::new( + quickwit_info, + http_client, + Clock::periodical(TELEMETRY_PUSH_COOLDOWN), + ) + } + + fn new( + quickwit_info: QuickwitTelemetryInfo, + sink_opt: Option, + clock: Clock, + ) -> Self { + let sink_opt: Option> = if let Some(sink) = sink_opt { + Some(Box::new(sink)) + } else { + None + }; + Self { + inner: Arc::new(Inner { + sink: sink_opt, + client_info: ClientInfo::default(), + quickwit_info, + events: Events::default(), + clock, + is_started: AtomicBool::new(false), + }), + } + } + + pub fn loop_started(&self) -> bool { + self.inner.is_started.load(Ordering::Relaxed) + } + + pub fn start_loop(&self) -> TelemetryLoopHandle { + let (terminate_command_tx, mut terminate_command_rx) = oneshot::channel(); + if self.inner.is_disabled() { + return TelemetryLoopHandle::NoLoop; + } + + assert!( + self.inner + .is_started + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok(), + "The telemetry loop is already started." + ); + + let inner = self.inner.clone(); + start_monitor_if_server_running_task(inner.clone()); + let join_handle = tokio::task::spawn(async move { + // This channel is used to send the command to terminate telemetry. + loop { + let quit_loop = tokio::select! { + _ = (&mut terminate_command_rx) => { true } + _ = inner.clock.tick() => { false } + }; + inner.send_pending_events().await; + if quit_loop { + break; + } + } + }); + TelemetryLoopHandle::WithLoop { + join_handle, + terminate_command_tx, + } + } + + pub async fn send(&self, event: TelemetryEvent) { + self.inner.send(event).await; + } +} + +/// telemetry is disabled in tests. +#[cfg(test)] +pub fn is_telemetry_disabled() -> bool { + true +} +/// Check to see if telemetry is enabled. +#[cfg(not(test))] +pub fn is_telemetry_disabled() -> bool { + quickwit_common::get_bool_from_env(crate::DISABLE_TELEMETRY_ENV_KEY, false) +} + +fn start_monitor_if_server_running_task(telemetry_sender: Arc) { + let mut clock = tokio::time::interval(TELEMETRY_RUNNING_EVENT_INTERVAL); + tokio::spawn(async move { + // Drop the first immediate tick. + clock.tick().await; + loop { + clock.tick().await; + telemetry_sender.send(TelemetryEvent::Running).await; + } + }); +} + +fn create_http_client() -> Option { + if is_telemetry_disabled() { + info!("telemetry to quickwit is disabled"); + return None; + } + let client = HttpClient::try_new()?; + info!("telemetry to {} is enabled", client.endpoint()); + Some(client) +} + +#[cfg(test)] +mod tests { + + use std::env; + + use super::*; + + #[ignore] + #[tokio::test] + async fn test_enabling_and_disabling_telemetry() { + // SAFETY: this test may not be entirely sound if not run with nextest or --test-threads=1 + // as this is only a test, and it would be extremly inconvenient to run it in a different + // way, we are keeping it that way + + // We group the two in a single test to ensure it happens on the same thread. + unsafe { env::set_var(crate::DISABLE_TELEMETRY_ENV_KEY, "") }; + assert_eq!( + TelemetrySender::from_quickwit_info(QuickwitTelemetryInfo::default()) + .inner + .is_disabled(), + true + ); + unsafe { env::remove_var(crate::DISABLE_TELEMETRY_ENV_KEY) }; + assert_eq!( + TelemetrySender::from_quickwit_info(QuickwitTelemetryInfo::default()) + .inner + .is_disabled(), + false + ); + } + + #[tokio::test] + async fn test_telemetry_no_wait_for_first_event() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (_clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop(); + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + let payload_opt = rx.recv().await; + assert!(payload_opt.is_some()); + let payload = payload_opt.unwrap(); + assert_eq!(payload.events.len(), 1); + loop_handler.terminate_telemetry().await; + } + + #[tokio::test] + async fn test_telemetry_two_events() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop(); + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + } + clock_btn.tick().await; + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + } + loop_handler.terminate_telemetry().await; + } + + #[tokio::test] + async fn test_telemetry_uptime_events() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop(); + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + } + clock_btn.tick().await; + tokio::time::sleep(TELEMETRY_RUNNING_EVENT_INTERVAL + Duration::from_secs(1)).await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + assert_eq!(payload.events[0].event, TelemetryEvent::Running); + } + loop_handler.terminate_telemetry().await; + } + + #[tokio::test] + async fn test_telemetry_cooldown_observed() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop(); + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + } + tokio::task::yield_now().await; + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + + let timeout_res = tokio::time::timeout(Duration::from_millis(1), rx.recv()).await; + assert!(timeout_res.is_err()); + + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + clock_btn.tick().await; + { + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 2); + } + loop_handler.terminate_telemetry().await; + } + + #[tokio::test] + async fn test_terminate_telemetry_sends_pending_events() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (_clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop(); + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + telemetry_sender + .send(TelemetryEvent::EndCommand { return_code: 2i32 }) + .await; + loop_handler.terminate_telemetry().await; + let payload = rx.recv().await.unwrap(); + assert_eq!(payload.events.len(), 1); + assert!(matches!( + &payload.events[0].event, + &TelemetryEvent::EndCommand { .. } + )); + } +} diff --git a/quickwit/quickwit-telemetry/src/sink.rs b/quickwit/quickwit-telemetry/src/sink.rs new file mode 100644 index 00000000000..9ca49ebd841 --- /dev/null +++ b/quickwit/quickwit-telemetry/src/sink.rs @@ -0,0 +1,75 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use async_trait::async_trait; +use reqwest::Client; +use reqwest::redirect::Policy; +use tokio::sync::mpsc::UnboundedSender; + +use crate::payload::TelemetryPayload; + +/// Telemetry ingest API URL +const DEFAULT_TELEMETRY_INGEST_API_URL: &str = "https://telemetry.quickwit.io/"; + +fn telemetry_ingest_api_url() -> String { + if let Some(ingest_api_url) = std::env::var_os("TELEMETRY_INGEST_API") { + ingest_api_url.to_string_lossy().to_string() + } else { + DEFAULT_TELEMETRY_INGEST_API_URL.to_string() + } +} + +#[async_trait] +pub trait Sink: Send + Sync + 'static { + async fn send_payload(&self, payload: TelemetryPayload); +} +pub struct HttpClient { + client: Client, + endpoint: String, +} + +impl HttpClient { + pub fn try_new() -> Option { + let client = Client::builder() + .redirect(Policy::limited(3)) + .timeout(Duration::from_secs(10)) + .build() + .ok()?; + Some(HttpClient { + client, + endpoint: telemetry_ingest_api_url(), + }) + } + + pub fn endpoint(&self) -> &str { + &self.endpoint + } +} + +#[async_trait] +impl Sink for UnboundedSender { + async fn send_payload(&self, payload: TelemetryPayload) { + let _ = self.send(payload); + } +} + +#[async_trait] +impl Sink for HttpClient { + async fn send_payload(&self, payload: TelemetryPayload) { + // Note that we swallow the error if any + let _ = self.client.post(&self.endpoint).json(&payload).send().await; + } +} From 1b6de8f44e9f4594f04377462a0befe63e214834 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Mon, 19 Oct 2026 17:02:22 +0000 Subject: [PATCH 02/13] Harden restored telemetry and add a telemetry relay - Add a telemetry relay in distribution/telemetry-relay (Cloudflare Worker). It validates the payload and keeps only documented fields, never reads or forwards client IP addresses or headers, replaces the unsalted MD5 host hash with an HMAC keyed with a relay-only secret, and forwards one Datadog log per event. - Rewrite docs/telemetry.md: Datadog receives the data through the relay, list the events, services and features actually sent, and fix the --help example. - Make QW_DISABLE_TELEMETRY fail closed: any non-empty value other than false, no or 0 disables telemetry. Values such as "on" used to leave telemetry enabled, contrary to the docs. - Report more features from the node config: the storage backend of the default index root URI, Lambda leaf search, split cache and TLS. --- distribution/telemetry-relay/README.md | 84 ++++++ distribution/telemetry-relay/package.json | 9 + .../telemetry-relay/src/dev-server.js | 56 ++++ distribution/telemetry-relay/src/relay.js | 230 +++++++++++++++ .../telemetry-relay/test/relay.test.js | 265 ++++++++++++++++++ distribution/telemetry-relay/wrangler.toml | 19 ++ docs/reference/cli.md | 2 +- docs/telemetry.md | 66 +++-- quickwit/quickwit-cli/src/service.rs | 74 +++++ quickwit/quickwit-telemetry/Cargo.toml | 2 +- quickwit/quickwit-telemetry/src/lib.rs | 5 +- quickwit/quickwit-telemetry/src/payload.rs | 41 ++- quickwit/quickwit-telemetry/src/sender.rs | 39 ++- 13 files changed, 864 insertions(+), 28 deletions(-) create mode 100644 distribution/telemetry-relay/README.md create mode 100644 distribution/telemetry-relay/package.json create mode 100644 distribution/telemetry-relay/src/dev-server.js create mode 100644 distribution/telemetry-relay/src/relay.js create mode 100644 distribution/telemetry-relay/test/relay.test.js create mode 100644 distribution/telemetry-relay/wrangler.toml diff --git a/distribution/telemetry-relay/README.md b/distribution/telemetry-relay/README.md new file mode 100644 index 00000000000..3d7ddd87419 --- /dev/null +++ b/distribution/telemetry-relay/README.md @@ -0,0 +1,84 @@ +# Quickwit telemetry relay + +Quickwit nodes send anonymous usage data to `https://telemetry.quickwit.io/` (see [docs/telemetry.md](../../docs/telemetry.md)). This relay receives it and forwards it to Datadog Logs. It runs as a Cloudflare Worker and has no dependencies. + +For each request, the relay: + +1. Rejects it unless it is a `POST /` whose body matches the payload format of `quickwit/quickwit-telemetry/src/payload.rs`, and copies only the documented fields. +2. Ignores the client IP address and request headers. Nothing from them is stored or forwarded. +3. Replaces `client_info.hashed_host_username`, an unsalted MD5 of `hostname:username`, with `client.install_id`, an HMAC-SHA256 keyed with `INSTALL_ID_SECRET`. Without the secret, the stored value can't be matched against guessed host or user names. +4. Sends one log per event to the Datadog Logs intake API. + +Every release that still sends telemetry (0.6.0 to 0.8.x, and the releases that restored it) uses this payload format, so the relay accepts all of them. + +## Log format + +```json +{ + "ddsource": "quickwit-telemetry", + "service": "quickwit-telemetry", + "message": "end_command", + "event": { "type": "end_command", "client_unixtime": 1760000060, "return_code": 0 }, + "client": { + "session_uuid": "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", + "install_id": "<64 hex characters>", + "os": "linux", + "arch": "x86_64", + "kubernetes": true + }, + "quickwit": { + "version": "0.9.1", + "services": ["indexer", "searcher"], + "features": ["otlp", "s3_storage"] + }, + "num_dropped_events": 0 +} +``` + +The log timestamp is the time Datadog receives the log. The client clock may be wrong, so its time is kept in `event.client_unixtime`. + +## Settings + +| Name | Kind | Description | +| --- | --- | --- | +| `DD_LOGS_URL` | variable | Datadog Logs intake URL, for example `https://http-intake.logs.datadoghq.com/api/v2/logs`. | +| `DD_API_KEY` | secret | Datadog API key used to send the logs. | +| `INSTALL_ID_SECRET` | secret | HMAC key for `install_id`. Changing it changes every install ID. | + +The relay returns HTTP 500 and forwards nothing if a setting is missing. + +## Test + +```bash +npm test +``` + +## Run locally + +Start the relay, then point a Quickwit node at it with `TELEMETRY_INGEST_API`: + +```bash +DD_LOGS_URL=https://http-intake.logs.datadoghq.com/api/v2/logs \ +DD_API_KEY= \ +INSTALL_ID_SECRET= \ +npm run dev +``` + +```bash +TELEMETRY_INGEST_API=http://127.0.0.1:8787/ quickwit run +``` + +## Deploy + +```bash +npx wrangler secret put DD_API_KEY +npx wrangler secret put INSTALL_ID_SECRET +npx wrangler deploy +``` + +Uncomment the `routes` entry in `wrangler.toml` to serve the relay on `telemetry.quickwit.io`. + +Not handled by the relay: + +- Rate limiting. Use a Cloudflare rate limiting rule on the route. +- Platform logs. `wrangler.toml` turns Workers Logs off. Don't turn on Logpush or other request logs that record client IP addresses. diff --git a/distribution/telemetry-relay/package.json b/distribution/telemetry-relay/package.json new file mode 100644 index 00000000000..cc00214d068 --- /dev/null +++ b/distribution/telemetry-relay/package.json @@ -0,0 +1,9 @@ +{ + "name": "quickwit-telemetry-relay", + "private": true, + "type": "module", + "scripts": { + "dev": "node src/dev-server.js", + "test": "node --test" + } +} diff --git a/distribution/telemetry-relay/src/dev-server.js b/distribution/telemetry-relay/src/dev-server.js new file mode 100644 index 00000000000..9961c0ee634 --- /dev/null +++ b/distribution/telemetry-relay/src/dev-server.js @@ -0,0 +1,56 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Runs the relay locally with Node.js, for testing against a local Quickwit node. + +import { createServer } from "node:http"; +import { Readable } from "node:stream"; + +import relay from "./relay.js"; + +const port = Number(process.env.PORT || 8787); +const env = { + DD_API_KEY: process.env.DD_API_KEY, + DD_LOGS_URL: process.env.DD_LOGS_URL, + INSTALL_ID_SECRET: process.env.INSTALL_ID_SECRET, +}; + +const server = createServer(async (nodeRequest, nodeResponse) => { + try { + const hasBody = nodeRequest.method !== "GET" && nodeRequest.method !== "HEAD"; + const headers = {}; + for (const name of ["content-length", "content-type"]) { + if (nodeRequest.headers[name] !== undefined) { + headers[name] = nodeRequest.headers[name]; + } + } + const request = new Request(`http://localhost:${port}${nodeRequest.url}`, { + method: nodeRequest.method, + headers, + body: hasBody ? Readable.toWeb(nodeRequest) : undefined, + duplex: "half", + }); + const response = await relay.fetch(request, env); + nodeResponse.writeHead(response.status, Object.fromEntries(response.headers)); + nodeResponse.end(Buffer.from(await response.arrayBuffer())); + } catch (error) { + console.error(error); + nodeResponse.writeHead(500); + nodeResponse.end(); + } +}); + +server.listen(port, "127.0.0.1", () => { + console.log(`telemetry relay listening on http://127.0.0.1:${port}`); +}); diff --git a/distribution/telemetry-relay/src/relay.js b/distribution/telemetry-relay/src/relay.js new file mode 100644 index 00000000000..adc21cf1701 --- /dev/null +++ b/distribution/telemetry-relay/src/relay.js @@ -0,0 +1,230 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Quickwit telemetry relay. +// +// Quickwit nodes POST their usage payload (`quickwit/quickwit-telemetry/src/payload.rs`, documented +// in `docs/telemetry.md`) to `telemetry.quickwit.io`. The relay: +// - rejects anything that does not match that format, and keeps only the documented fields; +// - never reads or forwards the client's IP address or request headers; +// - replaces `hashed_host_username`, an unsalted MD5 of `hostname:username` that can be reversed +// by guessing, with an HMAC-SHA256 keyed with a secret only the relay knows; +// - forwards one log per event to the Datadog Logs intake API. + +const MAX_BODY_NUM_BYTES = 16 * 1024; +// Quickwit queues at most `MAX_NUM_EVENTS_IN_QUEUE` (10) events per request. +const MAX_NUM_EVENTS = 10; +const MAX_NUM_NAMES = 32; +const EVENT_TYPES = new Set(["run_command", "end_command", "running", "ui_index_page_load"]); +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +const MD5_HEX_RE = /^[0-9a-f]{32}$/; +// OS, CPU architecture, service and feature names. +const NAME_RE = /^[a-z0-9_-]{1,64}$/; +const VERSION_RE = /^[0-9A-Za-z.+-]{1,64}$/; +const REQUIRED_SETTINGS = ["DD_API_KEY", "DD_LOGS_URL", "INSTALL_ID_SECRET"]; + +export class InvalidPayloadError extends Error {} + +class BodyTooLargeError extends Error {} + +function check(condition, message) { + if (!condition) { + throw new InvalidPayloadError(message); + } +} + +function isObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isNonNegativeInteger(value) { + return Number.isSafeInteger(value) && value >= 0; +} + +function isMatch(value, regex) { + return typeof value === "string" && regex.test(value); +} + +function parseNames(value, field) { + check(Array.isArray(value), `${field} must be a list`); + check(value.length <= MAX_NUM_NAMES, `${field} has more than ${MAX_NUM_NAMES} items`); + for (const name of value) { + check(isMatch(name, NAME_RE), `${field} contains an invalid name`); + } + return [...new Set(value)].sort(); +} + +function parseEvent(event) { + check(isObject(event), "events must be objects"); + check(EVENT_TYPES.has(event.type), "unknown event type"); + check(isNonNegativeInteger(event.unixtime), "invalid event unixtime"); + // The client clock may be off, so the client time is kept as an attribute and the log + // timestamp is the intake time. + const parsedEvent = { type: event.type, client_unixtime: event.unixtime }; + if (event.type === "end_command") { + check(Number.isSafeInteger(event.return_code), "invalid end_command return_code"); + parsedEvent.return_code = event.return_code; + } + return parsedEvent; +} + +// Validates a telemetry payload and returns a copy that holds only the documented fields. +export function parsePayload(payload) { + check(isObject(payload), "payload must be an object"); + const clientInfo = payload.client_info; + check(isObject(clientInfo), "missing client_info"); + check(isMatch(clientInfo.session_uuid, UUID_RE), "invalid client_info.session_uuid"); + check(isMatch(clientInfo.os, NAME_RE), "invalid client_info.os"); + check(isMatch(clientInfo.arch, NAME_RE), "invalid client_info.arch"); + check( + isMatch(clientInfo.hashed_host_username, MD5_HEX_RE), + "invalid client_info.hashed_host_username", + ); + check(typeof clientInfo.kubernetes === "boolean", "invalid client_info.kubernetes"); + + const quickwitInfo = payload.quickwit_info; + check(isObject(quickwitInfo), "missing quickwit_info"); + check(isMatch(quickwitInfo.version, VERSION_RE), "invalid quickwit_info.version"); + + check(Array.isArray(payload.events), "events must be a list"); + check(payload.events.length <= MAX_NUM_EVENTS, `more than ${MAX_NUM_EVENTS} events`); + check(isNonNegativeInteger(payload.num_dropped_events), "invalid num_dropped_events"); + + return { + client: { + session_uuid: clientInfo.session_uuid, + os: clientInfo.os, + arch: clientInfo.arch, + hashed_host_username: clientInfo.hashed_host_username, + kubernetes: clientInfo.kubernetes, + }, + quickwit: { + version: quickwitInfo.version, + services: parseNames(quickwitInfo.services, "quickwit_info.services"), + features: parseNames(quickwitInfo.features, "quickwit_info.features"), + }, + events: payload.events.map(parseEvent), + num_dropped_events: payload.num_dropped_events, + }; +} + +// Returns the hex HMAC-SHA256 of `hashedHostUsername` keyed with `secret`. +export async function computeInstallId(secret, hashedHostUsername) { + const encoder = new TextEncoder(); + const key = await crypto.subtle.importKey( + "raw", + encoder.encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"], + ); + const signature = await crypto.subtle.sign("HMAC", key, encoder.encode(hashedHostUsername)); + return Array.from(new Uint8Array(signature), (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); +} + +function toDatadogLogs(parsedPayload, installId) { + const { hashed_host_username: _hashedHostUsername, ...client } = parsedPayload.client; + return parsedPayload.events.map((event) => ({ + ddsource: "quickwit-telemetry", + service: "quickwit-telemetry", + message: event.type, + event, + client: { ...client, install_id: installId }, + quickwit: parsedPayload.quickwit, + num_dropped_events: parsedPayload.num_dropped_events, + })); +} + +async function readBody(request) { + const contentLength = Number(request.headers.get("content-length")); + if (contentLength > MAX_BODY_NUM_BYTES) { + throw new BodyTooLargeError(); + } + if (request.body === null) { + return ""; + } + const reader = request.body.getReader(); + const chunks = []; + let numBytes = 0; + while (true) { + const { done, value: chunk } = await reader.read(); + if (done) { + break; + } + numBytes += chunk.byteLength; + if (numBytes > MAX_BODY_NUM_BYTES) { + await reader.cancel(); + throw new BodyTooLargeError(); + } + chunks.push(chunk); + } + const body = new Uint8Array(numBytes); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(body); +} + +function textResponse(status, text, headers = {}) { + return new Response(`${text}\n`, { status, headers }); +} + +export default { + async fetch(request, env) { + const missingSettings = REQUIRED_SETTINGS.filter((name) => !env[name]); + if (missingSettings.length > 0) { + console.error(`telemetry relay is missing settings: ${missingSettings.join(", ")}`); + return textResponse(500, "relay is misconfigured"); + } + if (new URL(request.url).pathname !== "/") { + return textResponse(404, "not found"); + } + if (request.method !== "POST") { + return textResponse(405, "method not allowed", { Allow: "POST" }); + } + let parsedPayload; + try { + parsedPayload = parsePayload(JSON.parse(await readBody(request))); + } catch (error) { + if (error instanceof BodyTooLargeError) { + return textResponse(413, "payload too large"); + } + if (error instanceof SyntaxError || error instanceof InvalidPayloadError) { + return textResponse(400, `invalid payload: ${error.message}`); + } + throw error; + } + if (parsedPayload.events.length === 0) { + return new Response(null, { status: 202 }); + } + const installId = await computeInstallId( + env.INSTALL_ID_SECRET, + parsedPayload.client.hashed_host_username, + ); + const intakeResponse = await fetch(env.DD_LOGS_URL, { + method: "POST", + headers: { "Content-Type": "application/json", "DD-API-KEY": env.DD_API_KEY }, + body: JSON.stringify(toDatadogLogs(parsedPayload, installId)), + }); + if (!intakeResponse.ok) { + console.error(`Datadog logs intake returned HTTP ${intakeResponse.status}`); + return textResponse(502, "failed to forward telemetry"); + } + return new Response(null, { status: 202 }); + }, +}; diff --git a/distribution/telemetry-relay/test/relay.test.js b/distribution/telemetry-relay/test/relay.test.js new file mode 100644 index 00000000000..d2fd268dbad --- /dev/null +++ b/distribution/telemetry-relay/test/relay.test.js @@ -0,0 +1,265 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import assert from "node:assert/strict"; +import { createHmac } from "node:crypto"; +import { afterEach, beforeEach, describe, it } from "node:test"; + +import relay, { computeInstallId } from "../src/relay.js"; + +const ENV = { + DD_API_KEY: "test-api-key", + DD_LOGS_URL: "https://http-intake.logs.example.com/api/v2/logs", + INSTALL_ID_SECRET: "test-secret", +}; +const HASHED_HOST_USERNAME = "5d41402abc4b2a76b9719d911017c592"; +const CLIENT_IP = "203.0.113.7"; + +// The shape `quickwit-telemetry` serializes (`TelemetryPayload`). +function makePayload() { + return { + client_info: { + session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", + os: "linux", + arch: "x86_64", + hashed_host_username: HASHED_HOST_USERNAME, + kubernetes: true, + }, + quickwit_info: { + version: "0.9.1", + services: ["searcher", "control-plane", "metastore_read_replica"], + features: ["s3_storage", "otlp"], + }, + events: [ + { unixtime: 1760000000, type: "run_command" }, + { unixtime: 1760000060, type: "end_command", return_code: 1 }, + ], + num_dropped_events: 0, + }; +} + +function makeRequest(body, { method = "POST", path = "/" } = {}) { + return new Request(`https://telemetry.quickwit.io${path}`, { + method, + headers: { + "Content-Type": "application/json", + "CF-Connecting-IP": CLIENT_IP, + "X-Forwarded-For": CLIENT_IP, + }, + body: typeof body === "string" ? body : JSON.stringify(body), + }); +} + +describe("telemetry relay", () => { + let intakeCalls; + let intakeStatus; + const originalFetch = globalThis.fetch; + + beforeEach(() => { + intakeCalls = []; + intakeStatus = 202; + globalThis.fetch = async (url, init) => { + intakeCalls.push({ url, init }); + return new Response(null, { status: intakeStatus }); + }; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("forwards one log per event to the Datadog logs intake", async () => { + const response = await relay.fetch(makeRequest(makePayload()), ENV); + + assert.equal(response.status, 202); + assert.equal(intakeCalls.length, 1); + const { url, init } = intakeCalls[0]; + assert.equal(url, ENV.DD_LOGS_URL); + assert.deepEqual(init.headers, { + "Content-Type": "application/json", + "DD-API-KEY": ENV.DD_API_KEY, + }); + const expectedInstallId = createHmac("sha256", ENV.INSTALL_ID_SECRET) + .update(HASHED_HOST_USERNAME) + .digest("hex"); + const commonAttributes = { + ddsource: "quickwit-telemetry", + service: "quickwit-telemetry", + client: { + session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", + os: "linux", + arch: "x86_64", + kubernetes: true, + install_id: expectedInstallId, + }, + quickwit: { + version: "0.9.1", + services: ["control-plane", "metastore_read_replica", "searcher"], + features: ["otlp", "s3_storage"], + }, + num_dropped_events: 0, + }; + assert.deepEqual(JSON.parse(init.body), [ + { + ...commonAttributes, + message: "run_command", + event: { type: "run_command", client_unixtime: 1760000000 }, + }, + { + ...commonAttributes, + message: "end_command", + event: { type: "end_command", client_unixtime: 1760000060, return_code: 1 }, + }, + ]); + }); + + it("never forwards the client IP address or the unkeyed host hash", async () => { + await relay.fetch(makeRequest(makePayload()), ENV); + + const forwarded = JSON.stringify(intakeCalls[0]); + assert.ok(!forwarded.includes(CLIENT_IP)); + assert.ok(!forwarded.includes(HASHED_HOST_USERNAME)); + }); + + it("drops fields that are not part of the documented payload", async () => { + const payload = makePayload(); + payload.client_info.email = "someone@example.com"; + payload.quickwit_info.index_ids = ["secret-index"]; + payload.events[0].query = "secret query"; + payload.extra = "extra"; + + const response = await relay.fetch(makeRequest(payload), ENV); + + assert.equal(response.status, 202); + const forwarded = intakeCalls[0].init.body; + for (const value of ["someone@example.com", "secret-index", "secret query", "extra"]) { + assert.ok(!forwarded.includes(value), `${value} was forwarded`); + } + }); + + it("keys the install ID with the secret", async () => { + const installId = await computeInstallId("secret-a", HASHED_HOST_USERNAME); + + assert.match(installId, /^[0-9a-f]{64}$/); + assert.notEqual(installId, await computeInstallId("secret-b", HASHED_HOST_USERNAME)); + assert.equal(installId, await computeInstallId("secret-a", HASHED_HOST_USERNAME)); + }); + + it("accepts a payload without events and forwards nothing", async () => { + const payload = makePayload(); + payload.events = []; + + const response = await relay.fetch(makeRequest(payload), ENV); + + assert.equal(response.status, 202); + assert.equal(intakeCalls.length, 0); + }); + + const invalidPayloads = { + "malformed JSON": "{", + "a list": [], + "missing client_info": { ...makePayload(), client_info: undefined }, + "an invalid session UUID": (() => { + const payload = makePayload(); + payload.client_info.session_uuid = "not-a-uuid"; + return payload; + })(), + "a host hash that is not an MD5": (() => { + const payload = makePayload(); + payload.client_info.hashed_host_username = "myhost:root"; + return payload; + })(), + "a free-form feature name": (() => { + const payload = makePayload(); + payload.quickwit_info.features = ["Index my-logs on s3://bucket"]; + return payload; + })(), + "an unknown event type": (() => { + const payload = makePayload(); + payload.events = [{ unixtime: 1760000000, type: "search", query: "secret" }]; + return payload; + })(), + "an end_command without return code": (() => { + const payload = makePayload(); + payload.events = [{ unixtime: 1760000000, type: "end_command" }]; + return payload; + })(), + "too many events": (() => { + const payload = makePayload(); + payload.events = Array.from({ length: 11 }, () => ({ unixtime: 1, type: "running" })); + return payload; + })(), + }; + + for (const [description, payload] of Object.entries(invalidPayloads)) { + it(`rejects ${description}`, async () => { + const response = await relay.fetch(makeRequest(payload), ENV); + + assert.equal(response.status, 400); + assert.equal(intakeCalls.length, 0); + }); + } + + it("rejects payloads larger than 16 KiB", async () => { + const payload = makePayload(); + payload.padding = "x".repeat(17 * 1024); + + const response = await relay.fetch(makeRequest(payload), ENV); + + assert.equal(response.status, 413); + assert.equal(intakeCalls.length, 0); + }); + + it("only accepts POST requests on /", async () => { + const getResponse = await relay.fetch( + new Request("https://telemetry.quickwit.io/", { method: "GET" }), + ENV, + ); + assert.equal(getResponse.status, 405); + + const otherPathResponse = await relay.fetch( + makeRequest(makePayload(), { path: "/api/v1/ingest" }), + ENV, + ); + assert.equal(otherPathResponse.status, 404); + assert.equal(intakeCalls.length, 0); + }); + + it("fails without forwarding when a setting is missing", async () => { + const { INSTALL_ID_SECRET: _secret, ...envWithoutSecret } = ENV; + const originalConsoleError = console.error; + console.error = () => {}; + try { + const response = await relay.fetch(makeRequest(makePayload()), envWithoutSecret); + + assert.equal(response.status, 500); + assert.equal(intakeCalls.length, 0); + } finally { + console.error = originalConsoleError; + } + }); + + it("returns 502 when the Datadog logs intake fails", async () => { + intakeStatus = 403; + const originalConsoleError = console.error; + console.error = () => {}; + try { + const response = await relay.fetch(makeRequest(makePayload()), ENV); + + assert.equal(response.status, 502); + } finally { + console.error = originalConsoleError; + } + }); +}); diff --git a/distribution/telemetry-relay/wrangler.toml b/distribution/telemetry-relay/wrangler.toml new file mode 100644 index 00000000000..6b529a4ac17 --- /dev/null +++ b/distribution/telemetry-relay/wrangler.toml @@ -0,0 +1,19 @@ +name = "quickwit-telemetry-relay" +main = "src/relay.js" +compatibility_date = "2026-06-01" + +# Serve the relay on the hostname Quickwit sends telemetry to. This requires the `quickwit.io` +# zone to be on the Cloudflare account. +# routes = [{ pattern = "telemetry.quickwit.io", custom_domain = true }] + +# Workers Logs would retain request metadata, including client IP addresses. Keep it off. +[observability] +enabled = false + +[vars] +DD_LOGS_URL = "https://http-intake.logs.datadoghq.com/api/v2/logs" + +# Secrets, set with `npx wrangler secret put `: +# - DD_API_KEY: Datadog API key used to send the logs. +# - INSTALL_ID_SECRET: key of the HMAC that replaces `hashed_host_username`. Changing it changes +# every install ID, so set it once and keep it. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ad14b3a5fbc..7cfdf0b81a7 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -857,7 +857,7 @@ Specifies the path to the [quickwit config](../configuration/node-config.md). Co ### QW_DISABLE_TELEMETRY -Disables [telemetry](../telemetry.md) when set to any non-empty value. +Disables [telemetry](../telemetry.md) when set to any non-empty value other than `false`, `no` or `0`. *Example* diff --git a/docs/telemetry.md b/docs/telemetry.md index 21ec1aef219..5bec6a1d692 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -3,42 +3,68 @@ title: Telemetry sidebar_position: 12 --- -Quickwit, Inc. collects anonymous data regarding general usage to help us drive our development. Privacy and transparency are at the heart of Quickwit values and we only collect the minimal useful data and don't use any third party tool for the collection. +Quickwit collects anonymous usage data to learn which versions are running, on which platforms, and which features are in use. Quickwit is maintained by Datadog, Inc., which receives and stores this data. -## Disabling data collection +## Disabling telemetry -Data collection are opt-out. To disable them, just set the environment variable `QW_DISABLE_TELEMETRY` to whatever value. +Telemetry is on by default. To turn it off, set `QW_DISABLE_TELEMETRY` before starting Quickwit: ```bash export QW_DISABLE_TELEMETRY=1 ``` -Look at `--help` command output to check whether telemetry is enabled or not: +Any non-empty value other than `false`, `no` or `0` disables telemetry. + +To check whether telemetry is on, run `quickwit --help`. While it is on, the output includes a `Telemetry: enabled` line: + ```bash quickwit --help -Quickwit 0.7 Sub-second search & analytics engine on cloud storage. Find more information at https://quickwit.io/docs -Telemetry enabled +Telemetry: enabled ``` -The line `Telemetry enabled` disappears when you disable it. +The line disappears when telemetry is off. + +Only `quickwit run` sends telemetry. Other CLI commands never do. + +## What is sent + +A node sends at most one request per minute, with up to 10 events per request. + +Events: + +- `run_command`: the node started. +- `end_command`: the node stopped, with its exit code. +- `running`: the node is still up. Sent every 12 hours. +- `ui_index_page_load`: someone opened the Quickwit UI. + +Each request also includes: + +- The Quickwit version. +- The services the node runs, such as `indexer`, `searcher` or `metastore`. +- The features in use: + - the metastore type, file-backed or PostgreSQL, if the node runs the metastore; + - where the default index root URI points: S3, Azure Blob Storage, Google Cloud Storage or the local file system; + - whether the OTLP endpoints and the Jaeger endpoint are enabled; + - whether the searcher uses AWS Lambda for leaf searches or a local split cache; + - whether the REST or gRPC server uses TLS. +- The OS and the CPU architecture. +- Whether the node runs on Kubernetes, based on whether `KUBERNETES_SERVICE_HOST` is set. +- A session ID, generated at random each time the node starts. +- An MD5 hash of the host name and the user name, used to count distinct installations. The relay replaces it with a keyed hash before storage. See below. +- The number of events dropped because too many were queued. -## Which data are collected? +Quickwit never sends index names, documents, queries, configuration values or credentials. -We collect the minimum amount of information to respect privacy. Here are the data collected: -- type of events among create, index, delete and serve events -- client information: - - session uuid: uuid generated on the fly - - quickwit version - - os (linux, macos, freebsd, android...) - - architecture of the CPU - - md5 hash of host and username - - a boolean to know if `KUBERNETES_SERVICE_HOST` is set. +## Where the data goes -All data are sent to `telemetry.quickwit.io`. +Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, a small relay operated by Datadog. Its source code is in [`distribution/telemetry-relay`](https://github.com/quickwit-oss/quickwit/tree/main/distribution/telemetry-relay). The relay: -## No third party +- rejects any request that doesn't match the format above, and keeps only the fields listed on this page; +- doesn't store or forward your IP address; +- replaces the host and user name hash with a keyed hash computed with a secret that only the relay knows, so the stored value can't be matched against guessed host or user names; +- forwards the events to Datadog, where they are stored as logs. -We did not want to add any untrusted third party tool in the collection so we decided to implement and host our own metric collection server. +Releases before 0.9.0 send the same data to the same address. diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index e1c6e1ddab0..23b9570745a 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -178,6 +178,28 @@ fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { }; features.insert(feature); } + let storage_feature_opt = match config.default_index_root_uri.protocol() { + Protocol::Azure => Some(QuickwitFeature::AzureStorage), + Protocol::File => Some(QuickwitFeature::FileStorage), + Protocol::Google => Some(QuickwitFeature::GoogleCloudStorage), + Protocol::S3 => Some(QuickwitFeature::S3Storage), + Protocol::Actor | Protocol::Grpc | Protocol::PostgreSQL | Protocol::Ram => None, + }; + if let Some(storage_feature) = storage_feature_opt { + features.insert(storage_feature); + } + // The Lambda and split cache settings are only relevant if the searcher is enabled. + if config.is_service_enabled(QuickwitService::Searcher) { + if config.searcher_config.lambda.is_some() { + features.insert(QuickwitFeature::LambdaLeafSearch); + } + if config.searcher_config.split_cache.is_some() { + features.insert(QuickwitFeature::SplitCache); + } + } + if config.rest_config.tls_config.is_some() || config.grpc_config.tls_config.is_some() { + features.insert(QuickwitFeature::Tls); + } let services = config .enabled_services .iter() @@ -189,6 +211,7 @@ fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { #[cfg(test)] mod tests { + use quickwit_config::LambdaConfig; use tempfile::tempdir; use super::*; @@ -218,6 +241,57 @@ mod tests { Ok(()) } + #[test] + fn test_quickwit_telemetry_info_features() { + let mut node_config = NodeConfig::for_test(); + node_config.enabled_services = HashSet::from([QuickwitService::Metastore]); + node_config.metastore_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); + node_config.default_index_root_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); + node_config.indexer_config.enable_otlp_endpoint = false; + node_config.jaeger_config.enable_endpoint = false; + node_config.searcher_config.lambda = Some(LambdaConfig::for_test()); + node_config.searcher_config.split_cache = + Some(serde_json::from_value(serde_json::json!({"max_num_bytes": "1GB"})).unwrap()); + + // The searcher settings are ignored because the searcher is not enabled. + let telemetry_info = quickwit_telemetry_info(&node_config); + assert_eq!( + telemetry_info.features, + HashSet::from([ + QuickwitFeature::FileBackedMetastore, + QuickwitFeature::FileStorage + ]) + ); + + node_config.enabled_services = + HashSet::from([QuickwitService::Metastore, QuickwitService::Searcher]); + node_config.metastore_uri = Uri::from_str("postgresql://localhost/metastore").unwrap(); + node_config.default_index_root_uri = Uri::from_str("s3://bucket/indexes").unwrap(); + node_config.indexer_config.enable_otlp_endpoint = true; + node_config.jaeger_config.enable_endpoint = true; + node_config.grpc_config.tls_config = Some( + serde_json::from_value(serde_json::json!({ + "cert_path": "/certs/cert.pem", + "key_path": "/certs/key.pem", + })) + .unwrap(), + ); + + let telemetry_info = quickwit_telemetry_info(&node_config); + assert_eq!( + telemetry_info.features, + HashSet::from([ + QuickwitFeature::Jaeger, + QuickwitFeature::LambdaLeafSearch, + QuickwitFeature::Otlp, + QuickwitFeature::PostgresqMetastore, + QuickwitFeature::S3Storage, + QuickwitFeature::SplitCache, + QuickwitFeature::Tls, + ]) + ); + } + #[test] fn test_parse_service_run_args_all_services() -> anyhow::Result<()> { let command = build_cli().no_binary_name(true); diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index 3027fddf8a6..3b99c35c89c 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quickwit-telemetry" -description = "Open Telemetry services" +description = "Anonymous usage telemetry" version.workspace = true edition.workspace = true diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 72f821005e4..055aed23d40 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -16,7 +16,8 @@ #![deny(clippy::disallowed_methods)] pub mod payload; -/// This crate contains the code responsible for sending usage data to Quickwit inc's server. +/// This crate contains the code responsible for sending anonymous usage data to the telemetry +/// relay. See `docs/telemetry.md` for the data that is sent. mod sender; pub(crate) mod sink; @@ -44,7 +45,7 @@ pub fn start_telemetry_loop(quickwit_info: QuickwitTelemetryInfo) -> Option for EventWithTimestamp { } } -/// Represents a Telemetry Event send to Quickwit's telemetry server for usage information. +/// Represents a telemetry event sent to the telemetry relay for usage information. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] #[serde(rename_all = "snake_case")] @@ -115,10 +115,24 @@ impl Default for QuickwitTelemetryInfo { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum QuickwitFeature { + /// The default index root URI is on Azure Blob Storage. + AzureStorage, FileBackedMetastore, + /// The default index root URI is on the local file system. + FileStorage, + /// The default index root URI is on Google Cloud Storage. + GoogleCloudStorage, Jaeger, + /// The searcher offloads leaf searches to AWS Lambda. + LambdaLeafSearch, Otlp, PostgresqMetastore, + /// The default index root URI is on S3 or an S3-compatible object store. + S3Storage, + /// The searcher caches splits on local disk. + SplitCache, + /// The REST or gRPC server uses TLS. + Tls, } fn hashed_host_username() -> String { @@ -147,7 +161,30 @@ impl Default for ClientInfo { mod tests { use serde_json; - use super::{EventWithTimestamp, TelemetryEvent}; + use super::{EventWithTimestamp, QuickwitFeature, TelemetryEvent}; + + #[test] + fn test_serialize_features_as_expected() { + // These names are part of the telemetry payload documented in `docs/telemetry.md`. + let features = [ + QuickwitFeature::AzureStorage, + QuickwitFeature::FileBackedMetastore, + QuickwitFeature::FileStorage, + QuickwitFeature::GoogleCloudStorage, + QuickwitFeature::Jaeger, + QuickwitFeature::LambdaLeafSearch, + QuickwitFeature::Otlp, + QuickwitFeature::PostgresqMetastore, + QuickwitFeature::S3Storage, + QuickwitFeature::SplitCache, + QuickwitFeature::Tls, + ]; + let json = serde_json::to_string(&features).unwrap(); + assert_eq!( + json, + r#"["azure_storage","file_backed_metastore","file_storage","google_cloud_storage","jaeger","lambda_leaf_search","otlp","postgresq_metastore","s3_storage","split_cache","tls"]"# + ); + } #[test] fn test_serialize_payload_as_expected() { diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 68c9b5deed4..555e8950e64 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::ffi::OsStr; use std::mem; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -304,7 +305,24 @@ pub fn is_telemetry_disabled() -> bool { /// Check to see if telemetry is enabled. #[cfg(not(test))] pub fn is_telemetry_disabled() -> bool { - quickwit_common::get_bool_from_env(crate::DISABLE_TELEMETRY_ENV_KEY, false) + let env_value_opt = std::env::var_os(crate::DISABLE_TELEMETRY_ENV_KEY); + is_telemetry_disabled_by_env_value(env_value_opt.as_deref()) +} + +/// Telemetry is disabled when `QW_DISABLE_TELEMETRY` is set to any non-empty value other than +/// `false`, `no` or `0`. Unrecognized values disable telemetry so that a mistyped opt-out never +/// leaves telemetry on. +fn is_telemetry_disabled_by_env_value(env_value_opt: Option<&OsStr>) -> bool { + let Some(env_value) = env_value_opt else { + return false; + }; + let Some(env_value_str) = env_value.to_str() else { + return true; + }; + if env_value_str.trim().is_empty() { + return false; + } + quickwit_common::parse_bool_lenient(env_value_str) != Some(false) } fn start_monitor_if_server_running_task(telemetry_sender: Arc) { @@ -321,7 +339,7 @@ fn start_monitor_if_server_running_task(telemetry_sender: Arc) { fn create_http_client() -> Option { if is_telemetry_disabled() { - info!("telemetry to quickwit is disabled"); + info!("telemetry is disabled"); return None; } let client = HttpClient::try_new()?; @@ -336,6 +354,23 @@ mod tests { use super::*; + #[test] + fn test_is_telemetry_disabled_by_env_value() { + assert!(!is_telemetry_disabled_by_env_value(None)); + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new("")))); + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new(" ")))); + for enabled_value in ["false", "FALSE", "no", "0"] { + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new( + enabled_value + )))); + } + for disabled_value in ["1", "true", "yes", "on", "disabled", "anything"] { + assert!(is_telemetry_disabled_by_env_value(Some(OsStr::new( + disabled_value + )))); + } + } + #[ignore] #[tokio::test] async fn test_enabling_and_disabling_telemetry() { From 9985cfdea673ab6e320f56b5686e12c6203e89f0 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 22 Oct 2026 17:26:39 +0000 Subject: [PATCH 03/13] Report more telemetry and run the relay on AWS Lambda Add the machine size, build info, a persistent install ID, a hashed cluster ID, more features, API request counts and a cluster summary. The relay runs behind CloudFront, which provides the country and city. --- distribution/telemetry-relay/README.md | 74 +++--- distribution/telemetry-relay/package.json | 1 - .../telemetry-relay/src/dev-server.js | 56 ----- distribution/telemetry-relay/src/lambda.js | 122 +++++++++ distribution/telemetry-relay/src/relay.js | 191 ++++++++++---- .../telemetry-relay/test/lambda.test.js | 111 ++++++++ .../telemetry-relay/test/relay.test.js | 212 +++++++--------- distribution/telemetry-relay/wrangler.toml | 19 -- docs/telemetry.md | 33 ++- quickwit/Cargo.lock | 2 + quickwit/quickwit-cli/src/lib.rs | 1 + quickwit/quickwit-cli/src/service.rs | 115 +-------- quickwit/quickwit-cli/src/telemetry.rs | 236 ++++++++++++++++++ quickwit/quickwit-control-plane/Cargo.toml | 1 + .../src/control_plane.rs | 2 + quickwit/quickwit-control-plane/src/lib.rs | 1 + .../quickwit-control-plane/src/telemetry.rs | 85 +++++++ quickwit/quickwit-serve/src/api_usage.rs | 119 +++++++++ quickwit/quickwit-serve/src/grpc.rs | 22 +- quickwit/quickwit-serve/src/lib.rs | 1 + quickwit/quickwit-serve/src/rest.rs | 4 + quickwit/quickwit-telemetry/Cargo.toml | 1 + quickwit/quickwit-telemetry/src/install.rs | 76 ++++++ quickwit/quickwit-telemetry/src/lib.rs | 57 ++++- quickwit/quickwit-telemetry/src/machine.rs | 105 ++++++++ quickwit/quickwit-telemetry/src/payload.rs | 113 ++++++++- quickwit/quickwit-telemetry/src/sender.rs | 60 +---- quickwit/quickwit-telemetry/src/usage.rs | 188 ++++++++++++++ 28 files changed, 1535 insertions(+), 473 deletions(-) delete mode 100644 distribution/telemetry-relay/src/dev-server.js create mode 100644 distribution/telemetry-relay/src/lambda.js create mode 100644 distribution/telemetry-relay/test/lambda.test.js delete mode 100644 distribution/telemetry-relay/wrangler.toml create mode 100644 quickwit/quickwit-cli/src/telemetry.rs create mode 100644 quickwit/quickwit-control-plane/src/telemetry.rs create mode 100644 quickwit/quickwit-serve/src/api_usage.rs create mode 100644 quickwit/quickwit-telemetry/src/install.rs create mode 100644 quickwit/quickwit-telemetry/src/machine.rs create mode 100644 quickwit/quickwit-telemetry/src/usage.rs diff --git a/distribution/telemetry-relay/README.md b/distribution/telemetry-relay/README.md index 3d7ddd87419..31d141fa96f 100644 --- a/distribution/telemetry-relay/README.md +++ b/distribution/telemetry-relay/README.md @@ -1,16 +1,14 @@ # Quickwit telemetry relay -Quickwit nodes send anonymous usage data to `https://telemetry.quickwit.io/` (see [docs/telemetry.md](../../docs/telemetry.md)). This relay receives it and forwards it to Datadog Logs. It runs as a Cloudflare Worker and has no dependencies. +Quickwit nodes send anonymous usage data to `https://telemetry.quickwit.io/` (see [docs/telemetry.md](../../docs/telemetry.md)). This relay receives it and forwards it to Datadog Logs. It runs on AWS Lambda, behind CloudFront and an API Gateway HTTP API, and has no dependencies. For each request, the relay: -1. Rejects it unless it is a `POST /` whose body matches the payload format of `quickwit/quickwit-telemetry/src/payload.rs`, and copies only the documented fields. -2. Ignores the client IP address and request headers. Nothing from them is stored or forwarded. -3. Replaces `client_info.hashed_host_username`, an unsalted MD5 of `hostname:username`, with `client.install_id`, an HMAC-SHA256 keyed with `INSTALL_ID_SECRET`. Without the secret, the stored value can't be matched against guessed host or user names. +1. Rejects it unless its body matches the payload format of `quickwit/quickwit-telemetry/src/payload.rs`, and keeps only the documented fields. Releases 0.6.0 and later all send this format. +2. Ignores the client IP address and request headers, except the country and city that CloudFront derives from the IP address. +3. Replaces `hashed_host_username` and `hashed_cluster_id`, unsalted MD5 hashes that can be reversed by guessing, with HMAC-SHA256 hashes keyed with `INSTALL_ID_SECRET`: `client.install_id` and `quickwit.cluster_id`. 4. Sends one log per event to the Datadog Logs intake API. -Every release that still sends telemetry (0.6.0 to 0.8.x, and the releases that restored it) uses this payload format, so the relay accepts all of them. - ## Log format ```json @@ -18,67 +16,51 @@ Every release that still sends telemetry (0.6.0 to 0.8.x, and the releases that "ddsource": "quickwit-telemetry", "service": "quickwit-telemetry", "message": "end_command", - "event": { "type": "end_command", "client_unixtime": 1760000060, "return_code": 0 }, + "schema_version": 2, + "event": { + "type": "end_command", + "client_unixtime": 1760000060, + "return_code": 0, + "api_usage": { "elasticsearch_ingest": "100-999", "native_search": "1-9" }, + "cluster_summary": { "num_indexes": "2-10", "source_types": ["kafka"] } + }, "client": { "session_uuid": "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", "install_id": "<64 hex characters>", + "install_uuid": "8c4f2a5e-3b1d-4e7f-9a6c-2d5e8f1b3c7a", "os": "linux", "arch": "x86_64", - "kubernetes": true + "kubernetes": true, + "cpus": "5-8", + "memory": "8-16GiB" }, "quickwit": { "version": "0.9.1", "services": ["indexer", "searcher"], - "features": ["otlp", "s3_storage"] + "features": ["ingest_v2", "otlp", "s3_storage"], + "build": { "commit": "4f20e4f", "target": "x86_64-unknown-linux-gnu", "release": true }, + "cluster_id": "<64 hex characters>" }, + "location": { "country": "US", "city": "Ashburn" }, "num_dropped_events": 0 } ``` -The log timestamp is the time Datadog receives the log. The client clock may be wrong, so its time is kept in `event.client_unixtime`. +Fields that older releases don't send are left out. `event.cluster_summary` only comes from the node running the control plane. ## Settings -| Name | Kind | Description | -| --- | --- | --- | -| `DD_LOGS_URL` | variable | Datadog Logs intake URL, for example `https://http-intake.logs.datadoghq.com/api/v2/logs`. | -| `DD_API_KEY` | secret | Datadog API key used to send the logs. | -| `INSTALL_ID_SECRET` | secret | HMAC key for `install_id`. Changing it changes every install ID. | +| Name | Description | +| --- | --- | +| `DD_LOGS_URL` | Datadog Logs intake URL, for example `https://http-intake.logs.datadoghq.com/api/v2/logs`. | +| `DD_API_KEY_PARAMETER` | SSM parameter holding the Datadog API key. | +| `INSTALL_ID_SECRET_PARAMETER` | SSM parameter holding the HMAC key. Changing it changes every install and cluster ID. | +| `ORIGIN_VERIFY_SECRET` | Value of the `X-Origin-Verify` header that CloudFront adds. Requests without it get HTTP 403. | -The relay returns HTTP 500 and forwards nothing if a setting is missing. +The Lambda entry point is `src/lambda.handler`, on Node.js 22. ## Test ```bash npm test ``` - -## Run locally - -Start the relay, then point a Quickwit node at it with `TELEMETRY_INGEST_API`: - -```bash -DD_LOGS_URL=https://http-intake.logs.datadoghq.com/api/v2/logs \ -DD_API_KEY= \ -INSTALL_ID_SECRET= \ -npm run dev -``` - -```bash -TELEMETRY_INGEST_API=http://127.0.0.1:8787/ quickwit run -``` - -## Deploy - -```bash -npx wrangler secret put DD_API_KEY -npx wrangler secret put INSTALL_ID_SECRET -npx wrangler deploy -``` - -Uncomment the `routes` entry in `wrangler.toml` to serve the relay on `telemetry.quickwit.io`. - -Not handled by the relay: - -- Rate limiting. Use a Cloudflare rate limiting rule on the route. -- Platform logs. `wrangler.toml` turns Workers Logs off. Don't turn on Logpush or other request logs that record client IP addresses. diff --git a/distribution/telemetry-relay/package.json b/distribution/telemetry-relay/package.json index cc00214d068..859a3364ac4 100644 --- a/distribution/telemetry-relay/package.json +++ b/distribution/telemetry-relay/package.json @@ -3,7 +3,6 @@ "private": true, "type": "module", "scripts": { - "dev": "node src/dev-server.js", "test": "node --test" } } diff --git a/distribution/telemetry-relay/src/dev-server.js b/distribution/telemetry-relay/src/dev-server.js deleted file mode 100644 index 9961c0ee634..00000000000 --- a/distribution/telemetry-relay/src/dev-server.js +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Runs the relay locally with Node.js, for testing against a local Quickwit node. - -import { createServer } from "node:http"; -import { Readable } from "node:stream"; - -import relay from "./relay.js"; - -const port = Number(process.env.PORT || 8787); -const env = { - DD_API_KEY: process.env.DD_API_KEY, - DD_LOGS_URL: process.env.DD_LOGS_URL, - INSTALL_ID_SECRET: process.env.INSTALL_ID_SECRET, -}; - -const server = createServer(async (nodeRequest, nodeResponse) => { - try { - const hasBody = nodeRequest.method !== "GET" && nodeRequest.method !== "HEAD"; - const headers = {}; - for (const name of ["content-length", "content-type"]) { - if (nodeRequest.headers[name] !== undefined) { - headers[name] = nodeRequest.headers[name]; - } - } - const request = new Request(`http://localhost:${port}${nodeRequest.url}`, { - method: nodeRequest.method, - headers, - body: hasBody ? Readable.toWeb(nodeRequest) : undefined, - duplex: "half", - }); - const response = await relay.fetch(request, env); - nodeResponse.writeHead(response.status, Object.fromEntries(response.headers)); - nodeResponse.end(Buffer.from(await response.arrayBuffer())); - } catch (error) { - console.error(error); - nodeResponse.writeHead(500); - nodeResponse.end(); - } -}); - -server.listen(port, "127.0.0.1", () => { - console.log(`telemetry relay listening on http://127.0.0.1:${port}`); -}); diff --git a/distribution/telemetry-relay/src/lambda.js b/distribution/telemetry-relay/src/lambda.js new file mode 100644 index 00000000000..842239b153e --- /dev/null +++ b/distribution/telemetry-relay/src/lambda.js @@ -0,0 +1,122 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { timingSafeEqual } from "node:crypto"; + +import relay from "./relay.js"; + +// Requests without CloudFront's secret came straight to API Gateway and are rejected, so that +// nobody can bypass CloudFront or forge its location headers. +const ORIGIN_VERIFY_HEADER = "x-origin-verify"; +// Only these headers reach the relay, never the client IP address. +const FORWARDED_HEADERS = [ + "content-length", + "content-type", + "cloudfront-viewer-country", + "cloudfront-viewer-city", +]; + +// The AWS SDK ships with the Lambda runtime: importing it lazily keeps the relay testable without +// it. +async function loadSettingsFromSsm() { + const ddApiKeyParameter = process.env.DD_API_KEY_PARAMETER; + const installIdSecretParameter = process.env.INSTALL_ID_SECRET_PARAMETER; + if (!ddApiKeyParameter || !installIdSecretParameter) { + throw new Error("DD_API_KEY_PARAMETER and INSTALL_ID_SECRET_PARAMETER must be set"); + } + const { GetParametersCommand, SSMClient } = await import("@aws-sdk/client-ssm"); + const output = await new SSMClient({}).send( + new GetParametersCommand({ + Names: [ddApiKeyParameter, installIdSecretParameter], + WithDecryption: true, + }), + ); + const invalidParameters = output.InvalidParameters ?? []; + if (invalidParameters.length > 0) { + throw new Error(`missing SSM parameters: ${invalidParameters.join(", ")}`); + } + const parameters = output.Parameters ?? []; + const values = new Map(parameters.map((parameter) => [parameter.Name, parameter.Value])); + return { + DD_API_KEY: values.get(ddApiKeyParameter), + DD_LOGS_URL: process.env.DD_LOGS_URL, + INSTALL_ID_SECRET: values.get(installIdSecretParameter), + }; +} + +function toRequest(event) { + const method = event.requestContext.http.method; + const query = event.rawQueryString ? `?${event.rawQueryString}` : ""; + const headers = {}; + for (const name of FORWARDED_HEADERS) { + if (event.headers?.[name] !== undefined) { + headers[name] = event.headers[name]; + } + } + const hasBody = method !== "GET" && method !== "HEAD" && event.body !== undefined; + const body = hasBody + ? Buffer.from(event.body, event.isBase64Encoded ? "base64" : "utf8") + : undefined; + return new Request(`https://${event.requestContext.domainName}${event.rawPath}${query}`, { + method, + headers, + body, + }); +} + +function isFromCloudFront(event, originVerifySecret) { + const header = event.headers?.[ORIGIN_VERIFY_HEADER]; + if (header === undefined) { + return false; + } + const expected = Buffer.from(originVerifySecret); + const actual = Buffer.from(header); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +export function createHandler({ loadSettings, originVerifySecret }) { + let settingsPromise = null; + + return async function handler(event) { + if (!originVerifySecret) { + console.error("ORIGIN_VERIFY_SECRET must be set"); + return { statusCode: 500, body: "relay is misconfigured\n" }; + } + if (!isFromCloudFront(event, originVerifySecret)) { + return { statusCode: 403, body: "forbidden\n" }; + } + if (settingsPromise === null) { + settingsPromise = loadSettings(); + } + let settings; + try { + settings = await settingsPromise; + } catch (error) { + settingsPromise = null; + console.error(`failed to load relay settings: ${error.message}`); + return { statusCode: 500, body: "relay is misconfigured\n" }; + } + const response = await relay.fetch(toRequest(event), settings); + return { + statusCode: response.status, + headers: Object.fromEntries(response.headers), + body: await response.text(), + }; + }; +} + +export const handler = createHandler({ + loadSettings: loadSettingsFromSsm, + originVerifySecret: process.env.ORIGIN_VERIFY_SECRET, +}); diff --git a/distribution/telemetry-relay/src/relay.js b/distribution/telemetry-relay/src/relay.js index adc21cf1701..c9c07bbf72d 100644 --- a/distribution/telemetry-relay/src/relay.js +++ b/distribution/telemetry-relay/src/relay.js @@ -12,29 +12,42 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Quickwit telemetry relay. -// -// Quickwit nodes POST their usage payload (`quickwit/quickwit-telemetry/src/payload.rs`, documented -// in `docs/telemetry.md`) to `telemetry.quickwit.io`. The relay: -// - rejects anything that does not match that format, and keeps only the documented fields; -// - never reads or forwards the client's IP address or request headers; -// - replaces `hashed_host_username`, an unsalted MD5 of `hostname:username` that can be reversed -// by guessing, with an HMAC-SHA256 keyed with a secret only the relay knows; -// - forwards one log per event to the Datadog Logs intake API. - const MAX_BODY_NUM_BYTES = 16 * 1024; // Quickwit queues at most `MAX_NUM_EVENTS_IN_QUEUE` (10) events per request. const MAX_NUM_EVENTS = 10; const MAX_NUM_NAMES = 32; -const EVENT_TYPES = new Set(["run_command", "end_command", "running", "ui_index_page_load"]); +const EVENT_TYPES = new Set([ + "run_command", + "end_command", + "running", + "ui_index_page_load", + "install", +]); +const ACTIVITY_EVENT_TYPES = new Set(["end_command", "running"]); +const API_KINDS = new Set([ + "elasticsearch_search", + "elasticsearch_ingest", + "elasticsearch_other", + "native_search", + "native_ingest", + "otlp", + "jaeger", +]); +const MAX_SCHEMA_VERSION = 2; const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; const MD5_HEX_RE = /^[0-9a-f]{32}$/; -// OS, CPU architecture, service and feature names. const NAME_RE = /^[a-z0-9_-]{1,64}$/; const VERSION_RE = /^[0-9A-Za-z.+-]{1,64}$/; +const BUCKET_RE = /^[0-9A-Za-z<+-]{1,16}$/; +const COMMIT_RE = /^([0-9a-f]{7,40}|unknown)$/; +const TARGET_RE = /^[a-z0-9_.-]{1,64}$/; +const COUNTRY_HEADER = "cloudfront-viewer-country"; +const CITY_HEADER = "cloudfront-viewer-city"; +const COUNTRY_RE = /^[A-Z]{2}$/; +const CITY_RE = /^[^\p{Cc}]{1,100}$/u; const REQUIRED_SETTINGS = ["DD_API_KEY", "DD_LOGS_URL", "INSTALL_ID_SECRET"]; -export class InvalidPayloadError extends Error {} +class InvalidPayloadError extends Error {} class BodyTooLargeError extends Error {} @@ -65,23 +78,69 @@ function parseNames(value, field) { return [...new Set(value)].sort(); } +function parseBucket(value, field) { + check(isMatch(value, BUCKET_RE), `invalid ${field}`); + return value; +} + +function parseApiUsage(apiUsage) { + check(isObject(apiUsage), "api_usage must be an object"); + const parsedApiUsage = {}; + for (const [apiKind, bucket] of Object.entries(apiUsage)) { + check(API_KINDS.has(apiKind), "unknown api_usage key"); + parsedApiUsage[apiKind] = parseBucket(bucket, "api_usage bucket"); + } + return parsedApiUsage; +} + +function parseClusterSummary(clusterSummary) { + check(isObject(clusterSummary), "cluster_summary must be an object"); + return { + num_indexes: parseBucket(clusterSummary.num_indexes, "cluster_summary.num_indexes"), + source_types: parseNames(clusterSummary.source_types, "cluster_summary.source_types"), + }; +} + function parseEvent(event) { check(isObject(event), "events must be objects"); check(EVENT_TYPES.has(event.type), "unknown event type"); check(isNonNegativeInteger(event.unixtime), "invalid event unixtime"); - // The client clock may be off, so the client time is kept as an attribute and the log - // timestamp is the intake time. + // The client clock may be off, so the log timestamp is the intake time. const parsedEvent = { type: event.type, client_unixtime: event.unixtime }; if (event.type === "end_command") { check(Number.isSafeInteger(event.return_code), "invalid end_command return_code"); parsedEvent.return_code = event.return_code; } + if (ACTIVITY_EVENT_TYPES.has(event.type)) { + if (event.api_usage !== undefined) { + parsedEvent.api_usage = parseApiUsage(event.api_usage); + } + if (event.cluster_summary !== undefined) { + parsedEvent.cluster_summary = parseClusterSummary(event.cluster_summary); + } + } return parsedEvent; } -// Validates a telemetry payload and returns a copy that holds only the documented fields. -export function parsePayload(payload) { +function parseBuild(build) { + check(isObject(build), "quickwit_info.build must be an object"); + check(isMatch(build.commit, COMMIT_RE), "invalid quickwit_info.build.commit"); + check( + build.target === "unknown" || isMatch(build.target, TARGET_RE), + "invalid quickwit_info.build.target", + ); + check(typeof build.release === "boolean", "invalid quickwit_info.build.release"); + return { commit: build.commit, target: build.target, release: build.release }; +} + +function parsePayload(payload) { check(isObject(payload), "payload must be an object"); + const schemaVersion = payload.schema_version ?? 1; + const isKnownSchemaVersion = + Number.isSafeInteger(schemaVersion) && + schemaVersion >= 1 && + schemaVersion <= MAX_SCHEMA_VERSION; + check(isKnownSchemaVersion, "unsupported schema_version"); const clientInfo = payload.client_info; check(isObject(clientInfo), "missing client_info"); check(isMatch(clientInfo.session_uuid, UUID_RE), "invalid client_info.session_uuid"); @@ -92,35 +151,57 @@ export function parsePayload(payload) { "invalid client_info.hashed_host_username", ); check(typeof clientInfo.kubernetes === "boolean", "invalid client_info.kubernetes"); + const client = { + session_uuid: clientInfo.session_uuid, + os: clientInfo.os, + arch: clientInfo.arch, + hashed_host_username: clientInfo.hashed_host_username, + kubernetes: clientInfo.kubernetes, + }; + if (clientInfo.cpus !== undefined) { + client.cpus = parseBucket(clientInfo.cpus, "client_info.cpus"); + } + if (clientInfo.memory !== undefined) { + client.memory = parseBucket(clientInfo.memory, "client_info.memory"); + } const quickwitInfo = payload.quickwit_info; check(isObject(quickwitInfo), "missing quickwit_info"); check(isMatch(quickwitInfo.version, VERSION_RE), "invalid quickwit_info.version"); + const quickwit = { + version: quickwitInfo.version, + services: parseNames(quickwitInfo.services, "quickwit_info.services"), + features: parseNames(quickwitInfo.features, "quickwit_info.features"), + }; + if (quickwitInfo.build !== undefined) { + quickwit.build = parseBuild(quickwitInfo.build); + } + if (quickwitInfo.install_uuid !== undefined) { + check(isMatch(quickwitInfo.install_uuid, UUID_RE), "invalid quickwit_info.install_uuid"); + client.install_uuid = quickwitInfo.install_uuid; + } + if (quickwitInfo.hashed_cluster_id !== undefined) { + check( + isMatch(quickwitInfo.hashed_cluster_id, MD5_HEX_RE), + "invalid quickwit_info.hashed_cluster_id", + ); + quickwit.hashed_cluster_id = quickwitInfo.hashed_cluster_id; + } check(Array.isArray(payload.events), "events must be a list"); check(payload.events.length <= MAX_NUM_EVENTS, `more than ${MAX_NUM_EVENTS} events`); check(isNonNegativeInteger(payload.num_dropped_events), "invalid num_dropped_events"); return { - client: { - session_uuid: clientInfo.session_uuid, - os: clientInfo.os, - arch: clientInfo.arch, - hashed_host_username: clientInfo.hashed_host_username, - kubernetes: clientInfo.kubernetes, - }, - quickwit: { - version: quickwitInfo.version, - services: parseNames(quickwitInfo.services, "quickwit_info.services"), - features: parseNames(quickwitInfo.features, "quickwit_info.features"), - }, + schema_version: schemaVersion, + client, + quickwit, events: payload.events.map(parseEvent), num_dropped_events: payload.num_dropped_events, }; } -// Returns the hex HMAC-SHA256 of `hashedHostUsername` keyed with `secret`. -export async function computeInstallId(secret, hashedHostUsername) { +async function computeKeyedHash(secret, value) { const encoder = new TextEncoder(); const key = await crypto.subtle.importKey( "raw", @@ -129,22 +210,48 @@ export async function computeInstallId(secret, hashedHostUsername) { false, ["sign"], ); - const signature = await crypto.subtle.sign("HMAC", key, encoder.encode(hashedHostUsername)); + const signature = await crypto.subtle.sign("HMAC", key, encoder.encode(value)); return Array.from(new Uint8Array(signature), (byte) => byte.toString(16).padStart(2, "0")).join( "", ); } -function toDatadogLogs(parsedPayload, installId) { - const { hashed_host_username: _hashedHostUsername, ...client } = parsedPayload.client; - return parsedPayload.events.map((event) => ({ +function parseLocation(headers) { + const location = {}; + const country = headers.get(COUNTRY_HEADER); + if (country !== null && COUNTRY_RE.test(country)) { + location.country = country; + } + const city = headers.get(CITY_HEADER); + if (city !== null && CITY_RE.test(city)) { + location.city = city; + } + return location; +} + +// The prefix keeps a cluster ID from ever equaling an install ID. +async function replaceHashes(parsedPayload, secret) { + const { hashed_host_username: hashedHostUsername, ...client } = parsedPayload.client; + client.install_id = await computeKeyedHash(secret, hashedHostUsername); + const { hashed_cluster_id: hashedClusterId, ...quickwit } = parsedPayload.quickwit; + if (hashedClusterId !== undefined) { + quickwit.cluster_id = await computeKeyedHash(secret, `cluster\n${hashedClusterId}`); + } + return { ...parsedPayload, client, quickwit }; +} + +function toDatadogLogs(keyedPayload, location) { + const hasLocation = Object.keys(location).length > 0; + return keyedPayload.events.map((event) => ({ ddsource: "quickwit-telemetry", service: "quickwit-telemetry", message: event.type, + schema_version: keyedPayload.schema_version, event, - client: { ...client, install_id: installId }, - quickwit: parsedPayload.quickwit, - num_dropped_events: parsedPayload.num_dropped_events, + client: keyedPayload.client, + quickwit: keyedPayload.quickwit, + ...(hasLocation ? { location } : {}), + num_dropped_events: keyedPayload.num_dropped_events, })); } @@ -212,14 +319,12 @@ export default { if (parsedPayload.events.length === 0) { return new Response(null, { status: 202 }); } - const installId = await computeInstallId( - env.INSTALL_ID_SECRET, - parsedPayload.client.hashed_host_username, - ); + const keyedPayload = await replaceHashes(parsedPayload, env.INSTALL_ID_SECRET); + const logs = toDatadogLogs(keyedPayload, parseLocation(request.headers)); const intakeResponse = await fetch(env.DD_LOGS_URL, { method: "POST", headers: { "Content-Type": "application/json", "DD-API-KEY": env.DD_API_KEY }, - body: JSON.stringify(toDatadogLogs(parsedPayload, installId)), + body: JSON.stringify(logs), }); if (!intakeResponse.ok) { console.error(`Datadog logs intake returned HTTP ${intakeResponse.status}`); diff --git a/distribution/telemetry-relay/test/lambda.test.js b/distribution/telemetry-relay/test/lambda.test.js new file mode 100644 index 00000000000..8df0ba4ec69 --- /dev/null +++ b/distribution/telemetry-relay/test/lambda.test.js @@ -0,0 +1,111 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import assert from "node:assert/strict"; +import { afterEach, beforeEach, describe, it } from "node:test"; + +import { createHandler } from "../src/lambda.js"; + +const SETTINGS = { + DD_API_KEY: "test-api-key", + DD_LOGS_URL: "https://http-intake.logs.example.com/api/v2/logs", + INSTALL_ID_SECRET: "test-secret", +}; +const ORIGIN_VERIFY_SECRET = "test-origin-secret"; +const CLIENT_IP = "198.51.100.23"; +const PAYLOAD = { + client_info: { + session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", + os: "linux", + arch: "aarch64", + hashed_host_username: "5d41402abc4b2a76b9719d911017c592", + kubernetes: false, + }, + quickwit_info: { version: "0.9.1", services: ["searcher"], features: ["otlp"] }, + events: [{ unixtime: 1760000000, type: "running" }], + num_dropped_events: 0, +}; + +function makeEvent({ base64 = false, headers = {} } = {}) { + const body = JSON.stringify(PAYLOAD); + return { + rawPath: "/", + rawQueryString: "", + headers: { + "content-type": "application/json", + "x-forwarded-for": CLIENT_IP, + "x-origin-verify": ORIGIN_VERIFY_SECRET, + ...headers, + }, + requestContext: { + domainName: "abc123.execute-api.us-east-1.amazonaws.com", + http: { method: "POST", path: "/", sourceIp: CLIENT_IP }, + }, + body: base64 ? Buffer.from(body).toString("base64") : body, + isBase64Encoded: base64, + }; +} + +describe("lambda handler", () => { + let intakeCalls; + const originalFetch = globalThis.fetch; + const handler = createHandler({ + loadSettings: async () => SETTINGS, + originVerifySecret: ORIGIN_VERIFY_SECRET, + }); + + beforeEach(() => { + intakeCalls = []; + globalThis.fetch = async (url, init) => { + intakeCalls.push({ url, init }); + return new Response(null, { status: 202 }); + }; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("relays the request", async () => { + const result = await handler(makeEvent()); + + assert.equal(result.statusCode, 202); + assert.equal(intakeCalls[0].init.headers["DD-API-KEY"], SETTINGS.DD_API_KEY); + }); + + it("decodes base64 bodies", async () => { + const result = await handler(makeEvent({ base64: true })); + + assert.equal(result.statusCode, 202); + }); + + it("rejects requests that don't come from CloudFront", async () => { + const result = await handler(makeEvent({ headers: { "x-origin-verify": "guess" } })); + + assert.equal(result.statusCode, 403); + assert.equal(intakeCalls.length, 0); + }); + + it("passes the location on, but not the client IP or the origin secret", async () => { + const headers = { "cloudfront-viewer-country": "FR", "cloudfront-viewer-city": "Lyon" }; + + await handler(makeEvent({ headers })); + + const [log] = JSON.parse(intakeCalls[0].init.body); + assert.deepEqual(log.location, { country: "FR", city: "Lyon" }); + const forwarded = JSON.stringify(intakeCalls); + assert.ok(!forwarded.includes(CLIENT_IP)); + assert.ok(!forwarded.includes(ORIGIN_VERIFY_SECRET)); + }); +}); diff --git a/distribution/telemetry-relay/test/relay.test.js b/distribution/telemetry-relay/test/relay.test.js index d2fd268dbad..5332851965a 100644 --- a/distribution/telemetry-relay/test/relay.test.js +++ b/distribution/telemetry-relay/test/relay.test.js @@ -16,7 +16,7 @@ import assert from "node:assert/strict"; import { createHmac } from "node:crypto"; import { afterEach, beforeEach, describe, it } from "node:test"; -import relay, { computeInstallId } from "../src/relay.js"; +import relay from "../src/relay.js"; const ENV = { DD_API_KEY: "test-api-key", @@ -24,9 +24,14 @@ const ENV = { INSTALL_ID_SECRET: "test-secret", }; const HASHED_HOST_USERNAME = "5d41402abc4b2a76b9719d911017c592"; +const HASHED_CLUSTER_ID = "9e107d9d372bb6826bd81d3542a419d6"; +const INSTALL_UUID = "8c4f2a5e-3b1d-4e7f-9a6c-2d5e8f1b3c7a"; const CLIENT_IP = "203.0.113.7"; -// The shape `quickwit-telemetry` serializes (`TelemetryPayload`). +function keyedHash(value) { + return createHmac("sha256", ENV.INSTALL_ID_SECRET).update(value).digest("hex"); +} + function makePayload() { return { client_info: { @@ -38,7 +43,7 @@ function makePayload() { }, quickwit_info: { version: "0.9.1", - services: ["searcher", "control-plane", "metastore_read_replica"], + services: ["searcher", "control-plane"], features: ["s3_storage", "otlp"], }, events: [ @@ -49,29 +54,46 @@ function makePayload() { }; } -function makeRequest(body, { method = "POST", path = "/" } = {}) { - return new Request(`https://telemetry.quickwit.io${path}`, { - method, - headers: { - "Content-Type": "application/json", - "CF-Connecting-IP": CLIENT_IP, - "X-Forwarded-For": CLIENT_IP, +function makeV2Payload() { + const payload = makePayload(); + payload.schema_version = 2; + payload.client_info.cpus = "5-8"; + payload.client_info.memory = "8-16GiB"; + payload.quickwit_info.build = { + commit: "4f20e4f", + target: "aarch64-unknown-linux-gnu", + release: false, + }; + payload.quickwit_info.install_uuid = INSTALL_UUID; + payload.quickwit_info.hashed_cluster_id = HASHED_CLUSTER_ID; + payload.events = [ + { + unixtime: 1760043200, + type: "running", + api_usage: { native_search: "1-9", elasticsearch_ingest: "100-999" }, + cluster_summary: { num_indexes: "2-10", source_types: ["vec", "kafka"] }, }, + ]; + return payload; +} + +function makeRequest(body, headers = {}) { + return new Request("https://telemetry.quickwit.io/", { + method: "POST", + headers: { "Content-Type": "application/json", "X-Forwarded-For": CLIENT_IP, ...headers }, body: typeof body === "string" ? body : JSON.stringify(body), }); } describe("telemetry relay", () => { let intakeCalls; - let intakeStatus; const originalFetch = globalThis.fetch; beforeEach(() => { intakeCalls = []; - intakeStatus = 202; globalThis.fetch = async (url, init) => { intakeCalls.push({ url, init }); - return new Response(null, { status: intakeStatus }); + return new Response(null, { status: 202 }); }; }); @@ -79,38 +101,35 @@ describe("telemetry relay", () => { globalThis.fetch = originalFetch; }); - it("forwards one log per event to the Datadog logs intake", async () => { + function forwardedLogs() { + return JSON.parse(intakeCalls[0].init.body); + } + + it("forwards one log per event", async () => { const response = await relay.fetch(makeRequest(makePayload()), ENV); assert.equal(response.status, 202); - assert.equal(intakeCalls.length, 1); - const { url, init } = intakeCalls[0]; - assert.equal(url, ENV.DD_LOGS_URL); - assert.deepEqual(init.headers, { - "Content-Type": "application/json", - "DD-API-KEY": ENV.DD_API_KEY, - }); - const expectedInstallId = createHmac("sha256", ENV.INSTALL_ID_SECRET) - .update(HASHED_HOST_USERNAME) - .digest("hex"); + assert.equal(intakeCalls[0].url, ENV.DD_LOGS_URL); + assert.equal(intakeCalls[0].init.headers["DD-API-KEY"], ENV.DD_API_KEY); const commonAttributes = { ddsource: "quickwit-telemetry", service: "quickwit-telemetry", + schema_version: 1, client: { session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", os: "linux", arch: "x86_64", kubernetes: true, - install_id: expectedInstallId, + install_id: keyedHash(HASHED_HOST_USERNAME), }, quickwit: { version: "0.9.1", - services: ["control-plane", "metastore_read_replica", "searcher"], + services: ["control-plane", "searcher"], features: ["otlp", "s3_storage"], }, num_dropped_events: 0, }; - assert.deepEqual(JSON.parse(init.body), [ + assert.deepEqual(forwardedLogs(), [ { ...commonAttributes, message: "run_command", @@ -124,82 +143,66 @@ describe("telemetry relay", () => { ]); }); - it("never forwards the client IP address or the unkeyed host hash", async () => { - await relay.fetch(makeRequest(makePayload()), ENV); - - const forwarded = JSON.stringify(intakeCalls[0]); - assert.ok(!forwarded.includes(CLIENT_IP)); - assert.ok(!forwarded.includes(HASHED_HOST_USERNAME)); + it("forwards the schema version 2 fields", async () => { + await relay.fetch(makeRequest(makeV2Payload()), ENV); + + const [log] = forwardedLogs(); + assert.equal(log.schema_version, 2); + assert.equal(log.client.cpus, "5-8"); + assert.equal(log.client.memory, "8-16GiB"); + assert.equal(log.client.install_uuid, INSTALL_UUID); + assert.deepEqual(log.quickwit.build, { + commit: "4f20e4f", + target: "aarch64-unknown-linux-gnu", + release: false, + }); + assert.equal(log.quickwit.cluster_id, keyedHash(`cluster\n${HASHED_CLUSTER_ID}`)); + assert.deepEqual(log.event, { + type: "running", + client_unixtime: 1760043200, + api_usage: { elasticsearch_ingest: "100-999", native_search: "1-9" }, + cluster_summary: { num_indexes: "2-10", source_types: ["kafka", "vec"] }, + }); }); - it("drops fields that are not part of the documented payload", async () => { - const payload = makePayload(); + it("never forwards the client IP, unkeyed hashes or undocumented fields", async () => { + const payload = makeV2Payload(); payload.client_info.email = "someone@example.com"; - payload.quickwit_info.index_ids = ["secret-index"]; payload.events[0].query = "secret query"; - payload.extra = "extra"; - const response = await relay.fetch(makeRequest(payload), ENV); + await relay.fetch(makeRequest(payload), ENV); - assert.equal(response.status, 202); - const forwarded = intakeCalls[0].init.body; - for (const value of ["someone@example.com", "secret-index", "secret query", "extra"]) { + const forwarded = JSON.stringify(intakeCalls[0]); + for (const value of [ + CLIENT_IP, + HASHED_HOST_USERNAME, + HASHED_CLUSTER_ID, + "someone@example.com", + "secret query", + ]) { assert.ok(!forwarded.includes(value), `${value} was forwarded`); } }); - it("keys the install ID with the secret", async () => { - const installId = await computeInstallId("secret-a", HASHED_HOST_USERNAME); - - assert.match(installId, /^[0-9a-f]{64}$/); - assert.notEqual(installId, await computeInstallId("secret-b", HASHED_HOST_USERNAME)); - assert.equal(installId, await computeInstallId("secret-a", HASHED_HOST_USERNAME)); - }); - - it("accepts a payload without events and forwards nothing", async () => { - const payload = makePayload(); - payload.events = []; + it("adds the country and city set by CloudFront", async () => { + const headers = { "CloudFront-Viewer-Country": "FR", "CloudFront-Viewer-City": "Lyon" }; - const response = await relay.fetch(makeRequest(payload), ENV); + await relay.fetch(makeRequest(makePayload(), headers), ENV); - assert.equal(response.status, 202); - assert.equal(intakeCalls.length, 0); + for (const log of forwardedLogs()) { + assert.deepEqual(log.location, { country: "FR", city: "Lyon" }); + } }); const invalidPayloads = { "malformed JSON": "{", - "a list": [], - "missing client_info": { ...makePayload(), client_info: undefined }, - "an invalid session UUID": (() => { - const payload = makePayload(); - payload.client_info.session_uuid = "not-a-uuid"; - return payload; - })(), - "a host hash that is not an MD5": (() => { - const payload = makePayload(); - payload.client_info.hashed_host_username = "myhost:root"; - return payload; - })(), + "an unknown event type": { ...makePayload(), events: [{ unixtime: 1, type: "search" }] }, "a free-form feature name": (() => { const payload = makePayload(); payload.quickwit_info.features = ["Index my-logs on s3://bucket"]; return payload; })(), - "an unknown event type": (() => { - const payload = makePayload(); - payload.events = [{ unixtime: 1760000000, type: "search", query: "secret" }]; - return payload; - })(), - "an end_command without return code": (() => { - const payload = makePayload(); - payload.events = [{ unixtime: 1760000000, type: "end_command" }]; - return payload; - })(), - "too many events": (() => { - const payload = makePayload(); - payload.events = Array.from({ length: 11 }, () => ({ unixtime: 1, type: "running" })); - return payload; - })(), + "an unsupported schema version": { ...makeV2Payload(), schema_version: 3 }, }; for (const [description, payload] of Object.entries(invalidPayloads)) { @@ -212,54 +215,11 @@ describe("telemetry relay", () => { } it("rejects payloads larger than 16 KiB", async () => { - const payload = makePayload(); - payload.padding = "x".repeat(17 * 1024); + const payload = { ...makePayload(), padding: "x".repeat(17 * 1024) }; const response = await relay.fetch(makeRequest(payload), ENV); assert.equal(response.status, 413); assert.equal(intakeCalls.length, 0); }); - - it("only accepts POST requests on /", async () => { - const getResponse = await relay.fetch( - new Request("https://telemetry.quickwit.io/", { method: "GET" }), - ENV, - ); - assert.equal(getResponse.status, 405); - - const otherPathResponse = await relay.fetch( - makeRequest(makePayload(), { path: "/api/v1/ingest" }), - ENV, - ); - assert.equal(otherPathResponse.status, 404); - assert.equal(intakeCalls.length, 0); - }); - - it("fails without forwarding when a setting is missing", async () => { - const { INSTALL_ID_SECRET: _secret, ...envWithoutSecret } = ENV; - const originalConsoleError = console.error; - console.error = () => {}; - try { - const response = await relay.fetch(makeRequest(makePayload()), envWithoutSecret); - - assert.equal(response.status, 500); - assert.equal(intakeCalls.length, 0); - } finally { - console.error = originalConsoleError; - } - }); - - it("returns 502 when the Datadog logs intake fails", async () => { - intakeStatus = 403; - const originalConsoleError = console.error; - console.error = () => {}; - try { - const response = await relay.fetch(makeRequest(makePayload()), ENV); - - assert.equal(response.status, 502); - } finally { - console.error = originalConsoleError; - } - }); }); diff --git a/distribution/telemetry-relay/wrangler.toml b/distribution/telemetry-relay/wrangler.toml deleted file mode 100644 index 6b529a4ac17..00000000000 --- a/distribution/telemetry-relay/wrangler.toml +++ /dev/null @@ -1,19 +0,0 @@ -name = "quickwit-telemetry-relay" -main = "src/relay.js" -compatibility_date = "2026-06-01" - -# Serve the relay on the hostname Quickwit sends telemetry to. This requires the `quickwit.io` -# zone to be on the Cloudflare account. -# routes = [{ pattern = "telemetry.quickwit.io", custom_domain = true }] - -# Workers Logs would retain request metadata, including client IP addresses. Keep it off. -[observability] -enabled = false - -[vars] -DD_LOGS_URL = "https://http-intake.logs.datadoghq.com/api/v2/logs" - -# Secrets, set with `npx wrangler secret put `: -# - DD_API_KEY: Datadog API key used to send the logs. -# - INSTALL_ID_SECRET: key of the HMAC that replaces `hashed_host_username`. Changing it changes -# every install ID, so set it once and keep it. diff --git a/docs/telemetry.md b/docs/telemetry.md index 5bec6a1d692..59e81aa7329 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -35,6 +35,7 @@ A node sends at most one request per minute, with up to 10 events per request. Events: +- `install`: the node started with a data directory that it had never used, so this counts new installations. - `run_command`: the node started. - `end_command`: the node stopped, with its exit code. - `running`: the node is still up. Sent every 12 hours. @@ -42,29 +43,43 @@ Events: Each request also includes: -- The Quickwit version. +- The version of the request format. +- The Quickwit version, and how the binary was built: the commit, the target platform, and whether it was built from a release tag. - The services the node runs, such as `indexer`, `searcher` or `metastore`. - The features in use: - the metastore type, file-backed or PostgreSQL, if the node runs the metastore; - where the default index root URI points: S3, Azure Blob Storage, Google Cloud Storage or the local file system; + - whether the S3 storage config targets an S3-compatible store, through a custom endpoint or a flavor such as MinIO, Garage, DigitalOcean Spaces or Google Cloud Storage; - whether the OTLP endpoints and the Jaeger endpoint are enabled; + - whether ingest API v1 and v2 are enabled; - whether the searcher uses AWS Lambda for leaf searches or a local split cache; + - whether document clustering is configured; - whether the REST or gRPC server uses TLS. - The OS and the CPU architecture. +- The number of CPUs and the amount of memory available to the node, as ranges such as `5-8` CPUs or `8-16GiB`. - Whether the node runs on Kubernetes, based on whether `KUBERNETES_SERVICE_HOST` is set. - A session ID, generated at random each time the node starts. -- An MD5 hash of the host name and the user name, used to count distinct installations. The relay replaces it with a keyed hash before storage. See below. +- An install ID, generated at random the first time the node uses its data directory and stored there in the `telemetry_install_id` file. It is only created while telemetry is on. +- An MD5 hash of the host name and the user name, used to count distinct installations from older releases, which have no install ID. The relay replaces it with a keyed hash before storage. See below. +- An MD5 hash of the cluster ID and the metastore URI, without credentials, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. - The number of events dropped because too many were queued. -Quickwit never sends index names, documents, queries, configuration values or credentials. +`running` and `end_command` events also include: + +- How many requests each API received since the previous report, as ranges such as `10-99`: search and bulk ingestion on the Elasticsearch-compatible API, other Elasticsearch-compatible requests, search and ingestion on the native API, OTLP, and Jaeger. Searches from the Quickwit UI count as native searches. +- On the node that runs the control plane only, so once per cluster: the number of indexes, as a range such as `2-10`, and the types of the sources created by users, such as `kafka` or `kinesis`. + +Quickwit never sends index or field names, documents, queries, configuration values or credentials. ## Where the data goes -Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, a small relay operated by Datadog. Its source code is in [`distribution/telemetry-relay`](https://github.com/quickwit-oss/quickwit/tree/main/distribution/telemetry-relay). The relay: +Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog operates on AWS. Its source code is in [`distribution/telemetry-relay`](https://github.com/quickwit-oss/quickwit/tree/main/distribution/telemetry-relay). + +- The request goes through Amazon CloudFront, which looks up the country and city of your IP address and passes them on. The IP address itself goes no further. +- A small relay checks the request against the format above, rejects anything else, and keeps only the fields listed on this page. +- The relay replaces the host and cluster hashes with keyed hashes, computed with a secret that only the relay knows, so the stored values can't be matched against guessed host names, user names or cluster IDs. +- The relay forwards the events, with the country and city, to Datadog, where they are stored as logs. Your IP address is not stored. -- rejects any request that doesn't match the format above, and keeps only the fields listed on this page; -- doesn't store or forward your IP address; -- replaces the host and user name hash with a keyed hash computed with a secret that only the relay knows, so the stored value can't be matched against guessed host or user names; -- forwards the events to Datadog, where they are stored as logs. +For nodes running in the cloud, the country and city are usually those of the data center. -Releases before 0.9.0 send the same data to the same address. +Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, without the API counts or cluster summary, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, and the metastore, OTLP and Jaeger features. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index cdb5db4cc91..f64328b90a7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -7404,6 +7404,7 @@ dependencies = [ "quickwit-metastore", "quickwit-metrics", "quickwit-proto", + "quickwit-telemetry", "rand 0.10.3", "serde", "serde_json", @@ -8149,6 +8150,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "tempfile", "tokio", "tracing", "username", diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs index c3dfd952cc3..4113f165056 100644 --- a/quickwit/quickwit-cli/src/lib.rs +++ b/quickwit/quickwit-cli/src/lib.rs @@ -56,6 +56,7 @@ pub mod service; pub mod source; pub mod split; pub mod stats; +mod telemetry; pub mod tool; /// Throughput calculation window size. diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index 23b9570745a..c0af34f61c2 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -21,16 +21,16 @@ use colored::Colorize; use futures::future::select; use itertools::Itertools; use quickwit_common::runtimes::RuntimesConfig; -use quickwit_common::uri::{Protocol, Uri}; -use quickwit_config::NodeConfig; +use quickwit_common::uri::Uri; use quickwit_config::service::QuickwitService; use quickwit_serve::tcp_listener::DefaultTcpListenerResolver; use quickwit_serve::{BuildInfo, EnvFilterReloadFn, reload_tls_cert, serve_quickwit}; -use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent}; +use quickwit_telemetry::payload::TelemetryEvent; use tokio::signal; use tracing::{debug, info}; use crate::checklist::{BLUE_COLOR, RED_COLOR}; +use crate::telemetry::{quickwit_telemetry_info, set_install_uuid}; use crate::{config_cli_arg, get_resolvers, load_node_config, start_actor_runtimes}; pub fn build_run_command() -> Command { @@ -126,8 +126,12 @@ impl RunCliCommand { let (storage_resolver, metastore_resolver) = get_resolvers(&node_config.storage_configs, &node_config.metastore_configs); crate::busy_detector::set_enabled(true); - let telemetry_handle_opt = - quickwit_telemetry::start_telemetry_loop(quickwit_telemetry_info(&node_config)); + let mut telemetry_info = quickwit_telemetry_info(&node_config); + let is_new_install = set_install_uuid(&mut telemetry_info, &node_config.data_dir_path); + let telemetry_handle_opt = quickwit_telemetry::start_telemetry_loop(telemetry_info); + if is_new_install { + quickwit_telemetry::send_telemetry_event(TelemetryEvent::Install).await; + } quickwit_telemetry::send_telemetry_event(TelemetryEvent::RunCommand).await; // TODO move in serve quickwit? let runtimes_config = RuntimesConfig::default(); @@ -151,7 +155,7 @@ impl RunCliCommand { Ok(_) => 0, Err(_) => 1, }; - quickwit_telemetry::send_telemetry_event(TelemetryEvent::EndCommand { return_code }).await; + quickwit_telemetry::send_telemetry_event(TelemetryEvent::end_command(return_code)).await; if let Some(telemetry_handle) = telemetry_handle_opt { telemetry_handle.terminate_telemetry().await; } @@ -161,57 +165,9 @@ impl RunCliCommand { } } -fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { - let mut features = HashSet::new(); - if config.indexer_config.enable_otlp_endpoint { - features.insert(QuickwitFeature::Otlp); - } - if config.jaeger_config.enable_endpoint { - features.insert(QuickwitFeature::Jaeger); - } - // The metastore URI is only relevant if the metastore is enabled. - if config.is_service_enabled(QuickwitService::Metastore) { - let feature = if config.metastore_uri.protocol() == Protocol::PostgreSQL { - QuickwitFeature::PostgresqMetastore - } else { - QuickwitFeature::FileBackedMetastore - }; - features.insert(feature); - } - let storage_feature_opt = match config.default_index_root_uri.protocol() { - Protocol::Azure => Some(QuickwitFeature::AzureStorage), - Protocol::File => Some(QuickwitFeature::FileStorage), - Protocol::Google => Some(QuickwitFeature::GoogleCloudStorage), - Protocol::S3 => Some(QuickwitFeature::S3Storage), - Protocol::Actor | Protocol::Grpc | Protocol::PostgreSQL | Protocol::Ram => None, - }; - if let Some(storage_feature) = storage_feature_opt { - features.insert(storage_feature); - } - // The Lambda and split cache settings are only relevant if the searcher is enabled. - if config.is_service_enabled(QuickwitService::Searcher) { - if config.searcher_config.lambda.is_some() { - features.insert(QuickwitFeature::LambdaLeafSearch); - } - if config.searcher_config.split_cache.is_some() { - features.insert(QuickwitFeature::SplitCache); - } - } - if config.rest_config.tls_config.is_some() || config.grpc_config.tls_config.is_some() { - features.insert(QuickwitFeature::Tls); - } - let services = config - .enabled_services - .iter() - .map(|service| service.to_string()) - .collect(); - QuickwitTelemetryInfo::new(services, features) -} - #[cfg(test)] mod tests { - use quickwit_config::LambdaConfig; use tempfile::tempdir; use super::*; @@ -241,57 +197,6 @@ mod tests { Ok(()) } - #[test] - fn test_quickwit_telemetry_info_features() { - let mut node_config = NodeConfig::for_test(); - node_config.enabled_services = HashSet::from([QuickwitService::Metastore]); - node_config.metastore_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); - node_config.default_index_root_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); - node_config.indexer_config.enable_otlp_endpoint = false; - node_config.jaeger_config.enable_endpoint = false; - node_config.searcher_config.lambda = Some(LambdaConfig::for_test()); - node_config.searcher_config.split_cache = - Some(serde_json::from_value(serde_json::json!({"max_num_bytes": "1GB"})).unwrap()); - - // The searcher settings are ignored because the searcher is not enabled. - let telemetry_info = quickwit_telemetry_info(&node_config); - assert_eq!( - telemetry_info.features, - HashSet::from([ - QuickwitFeature::FileBackedMetastore, - QuickwitFeature::FileStorage - ]) - ); - - node_config.enabled_services = - HashSet::from([QuickwitService::Metastore, QuickwitService::Searcher]); - node_config.metastore_uri = Uri::from_str("postgresql://localhost/metastore").unwrap(); - node_config.default_index_root_uri = Uri::from_str("s3://bucket/indexes").unwrap(); - node_config.indexer_config.enable_otlp_endpoint = true; - node_config.jaeger_config.enable_endpoint = true; - node_config.grpc_config.tls_config = Some( - serde_json::from_value(serde_json::json!({ - "cert_path": "/certs/cert.pem", - "key_path": "/certs/key.pem", - })) - .unwrap(), - ); - - let telemetry_info = quickwit_telemetry_info(&node_config); - assert_eq!( - telemetry_info.features, - HashSet::from([ - QuickwitFeature::Jaeger, - QuickwitFeature::LambdaLeafSearch, - QuickwitFeature::Otlp, - QuickwitFeature::PostgresqMetastore, - QuickwitFeature::S3Storage, - QuickwitFeature::SplitCache, - QuickwitFeature::Tls, - ]) - ); - } - #[test] fn test_parse_service_run_args_all_services() -> anyhow::Result<()> { let command = build_cli().no_binary_name(true); diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs new file mode 100644 index 00000000000..ae3880230da --- /dev/null +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -0,0 +1,236 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; +use std::path::Path; + +use quickwit_common::uri::Protocol; +use quickwit_config::service::QuickwitService; +use quickwit_config::{NodeConfig, StorageBackendFlavor, disable_ingest_v1, enable_ingest_v2}; +use quickwit_serve::BuildInfo; +use quickwit_telemetry::payload::{ + BuildTelemetryInfo, QuickwitFeature, QuickwitTelemetryInfo, hash_cluster_identity, +}; +use tracing::warn; + +pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { + let services = config + .enabled_services + .iter() + .map(|service| service.to_string()) + .collect(); + let build_info = BuildInfo::get(); + let mut redacted_metastore_uri = config.metastore_uri.clone(); + redacted_metastore_uri.redact(); + QuickwitTelemetryInfo { + build: BuildTelemetryInfo { + commit: build_info.commit_short_hash.to_string(), + target: build_info.build_target.to_string(), + release: !build_info.version.ends_with("-nightly"), + }, + hashed_cluster_id: Some(hash_cluster_identity( + &config.cluster_id, + redacted_metastore_uri.as_str(), + )), + ..QuickwitTelemetryInfo::new(services, telemetry_features(config)) + } +} + +/// Returns `true` when the install ID was just created. Does nothing when telemetry is disabled, so +/// that a node with telemetry off never writes the ID. +pub(crate) fn set_install_uuid( + telemetry_info: &mut QuickwitTelemetryInfo, + data_dir: &Path, +) -> bool { + if quickwit_telemetry::is_telemetry_disabled() { + return false; + } + match quickwit_telemetry::load_or_create_install_id(data_dir) { + Ok((install_uuid, created)) => { + telemetry_info.install_uuid = Some(install_uuid); + created + } + Err(error) => { + warn!(%error, "failed to load or create the telemetry install ID"); + false + } + } +} + +fn telemetry_features(config: &NodeConfig) -> HashSet { + let mut features = HashSet::new(); + if config.indexer_config.enable_otlp_endpoint { + features.insert(QuickwitFeature::Otlp); + } + if config.jaeger_config.enable_endpoint { + features.insert(QuickwitFeature::Jaeger); + } + // The metastore URI is only relevant if the metastore is enabled. + if config.is_service_enabled(QuickwitService::Metastore) { + let feature = if config.metastore_uri.protocol() == Protocol::PostgreSQL { + QuickwitFeature::PostgresqMetastore + } else { + QuickwitFeature::FileBackedMetastore + }; + features.insert(feature); + } + let storage_feature_opt = match config.default_index_root_uri.protocol() { + Protocol::Azure => Some(QuickwitFeature::AzureStorage), + Protocol::File => Some(QuickwitFeature::FileStorage), + Protocol::Google => Some(QuickwitFeature::GoogleCloudStorage), + Protocol::S3 => Some(QuickwitFeature::S3Storage), + Protocol::Actor | Protocol::Grpc | Protocol::PostgreSQL | Protocol::Ram => None, + }; + if let Some(storage_feature) = storage_feature_opt { + features.insert(storage_feature); + } + if let Some(s3_storage_config) = config.storage_configs.find_s3() { + if s3_storage_config.endpoint().is_some() { + features.insert(QuickwitFeature::S3CustomEndpoint); + } + let flavor_feature_opt = match s3_storage_config.flavor { + Some(StorageBackendFlavor::DigitalOcean) => Some(QuickwitFeature::S3FlavorDigitalOcean), + Some(StorageBackendFlavor::Garage) => Some(QuickwitFeature::S3FlavorGarage), + Some(StorageBackendFlavor::Gcs) => Some(QuickwitFeature::S3FlavorGcs), + Some(StorageBackendFlavor::MinIO) => Some(QuickwitFeature::S3FlavorMinio), + None => None, + }; + if let Some(flavor_feature) = flavor_feature_opt { + features.insert(flavor_feature); + } + } + if config.is_service_enabled(QuickwitService::Searcher) { + if config.searcher_config.lambda.is_some() { + features.insert(QuickwitFeature::LambdaLeafSearch); + } + if config.searcher_config.split_cache.is_some() { + features.insert(QuickwitFeature::SplitCache); + } + } + if config.rest_config.tls_config.is_some() || config.grpc_config.tls_config.is_some() { + features.insert(QuickwitFeature::Tls); + } + if !disable_ingest_v1() { + features.insert(QuickwitFeature::IngestV1); + } + if enable_ingest_v2() { + features.insert(QuickwitFeature::IngestV2); + } + if config.docs_clustering_config.is_some() { + features.insert(QuickwitFeature::DocsClustering); + } + features +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use quickwit_common::uri::Uri; + use quickwit_config::{ + DocsClusteringConfig, LambdaConfig, S3StorageConfig, StorageConfig, StorageConfigs, + }; + + use super::*; + + /// The ingest features depend on environment variables, which CI may set. + fn ingest_features() -> Vec { + let mut features = Vec::new(); + if !disable_ingest_v1() { + features.push(QuickwitFeature::IngestV1); + } + if enable_ingest_v2() { + features.push(QuickwitFeature::IngestV2); + } + features + } + + #[test] + fn test_telemetry_features() { + let mut node_config = NodeConfig::for_test(); + node_config.enabled_services = HashSet::from([QuickwitService::Metastore]); + node_config.metastore_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); + node_config.default_index_root_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); + // No S3 config, so that `QW_S3_ENDPOINT`, which CI may set, has no effect. + node_config.storage_configs = StorageConfigs::new(Vec::new()); + node_config.indexer_config.enable_otlp_endpoint = false; + node_config.jaeger_config.enable_endpoint = false; + node_config.searcher_config.lambda = Some(LambdaConfig::for_test()); + node_config.searcher_config.split_cache = + Some(serde_json::from_value(serde_json::json!({"max_num_bytes": "1GB"})).unwrap()); + + let mut expected_features = HashSet::from([ + QuickwitFeature::FileBackedMetastore, + QuickwitFeature::FileStorage, + ]); + expected_features.extend(ingest_features()); + assert_eq!(telemetry_features(&node_config), expected_features); + + node_config.enabled_services = + HashSet::from([QuickwitService::Metastore, QuickwitService::Searcher]); + node_config.metastore_uri = Uri::from_str("postgresql://localhost/metastore").unwrap(); + node_config.default_index_root_uri = Uri::from_str("s3://bucket/indexes").unwrap(); + node_config.storage_configs = + StorageConfigs::new(vec![StorageConfig::S3(S3StorageConfig { + flavor: Some(StorageBackendFlavor::MinIO), + endpoint: Some("http://minio:9000".to_string()), + ..Default::default() + })]); + node_config.indexer_config.enable_otlp_endpoint = true; + node_config.jaeger_config.enable_endpoint = true; + node_config.grpc_config.tls_config = Some( + serde_json::from_value(serde_json::json!({ + "cert_path": "/certs/cert.pem", + "key_path": "/certs/key.pem", + })) + .unwrap(), + ); + node_config.docs_clustering_config = Some(DocsClusteringConfig { + policies: Vec::new(), + }); + + let mut expected_features = HashSet::from([ + QuickwitFeature::DocsClustering, + QuickwitFeature::Jaeger, + QuickwitFeature::LambdaLeafSearch, + QuickwitFeature::Otlp, + QuickwitFeature::PostgresqMetastore, + QuickwitFeature::S3CustomEndpoint, + QuickwitFeature::S3FlavorMinio, + QuickwitFeature::S3Storage, + QuickwitFeature::SplitCache, + QuickwitFeature::Tls, + ]); + expected_features.extend(ingest_features()); + assert_eq!(telemetry_features(&node_config), expected_features); + } + + #[test] + fn test_quickwit_telemetry_info_hashes_cluster_identity() { + let mut node_config = NodeConfig::for_test(); + node_config.metastore_uri = + Uri::from_str("postgresql://quickwit:secret@db:5432/metastore").unwrap(); + let telemetry_info = quickwit_telemetry_info(&node_config); + let hashed_cluster_id = telemetry_info.hashed_cluster_id.unwrap(); + + node_config.metastore_uri = + Uri::from_str("postgresql://quickwit:rotated@db:5432/metastore").unwrap(); + let telemetry_info = quickwit_telemetry_info(&node_config); + assert_eq!(telemetry_info.hashed_cluster_id.unwrap(), hashed_cluster_id); + + node_config.cluster_id = "other-cluster".to_string(); + let telemetry_info = quickwit_telemetry_info(&node_config); + assert_ne!(telemetry_info.hashed_cluster_id.unwrap(), hashed_cluster_id); + } +} diff --git a/quickwit/quickwit-control-plane/Cargo.toml b/quickwit/quickwit-control-plane/Cargo.toml index e0c8e951e98..61b7848cfa7 100644 --- a/quickwit/quickwit-control-plane/Cargo.toml +++ b/quickwit/quickwit-control-plane/Cargo.toml @@ -35,6 +35,7 @@ quickwit-config = { workspace = true } quickwit-ingest = { workspace = true } quickwit-metastore = { workspace = true } quickwit-proto = { workspace = true } +quickwit-telemetry = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs index 66ede136770..bdc0896d211 100644 --- a/quickwit/quickwit-control-plane/src/control_plane.rs +++ b/quickwit/quickwit-control-plane/src/control_plane.rs @@ -62,6 +62,7 @@ use crate::ingest::IngestController; use crate::ingest::ingest_controller::{IngestControllerStats, RebalanceShardsCallback}; use crate::metrics::{METASTORE_ERROR_ABORTED, METASTORE_ERROR_MAYBE_EXECUTED, RESTART_TOTAL}; use crate::model::ControlPlaneModel; +use crate::telemetry::update_telemetry_cluster_summary; /// Interval between two controls (or checks) of the desired plan VS running plan. pub(crate) const CONTROL_PLAN_LOOP_INTERVAL: Duration = if cfg!(any(test, feature = "testsuite")) { @@ -506,6 +507,7 @@ impl Handler for ControlPlane { } } self.indexing_scheduler.control_running_plan(&self.model); + update_telemetry_cluster_summary(&self.model); ctx.schedule_self_msg(CONTROL_PLAN_LOOP_INTERVAL, ControlPlaneLoop); Ok(()) } diff --git a/quickwit/quickwit-control-plane/src/lib.rs b/quickwit/quickwit-control-plane/src/lib.rs index 45ee0275d21..df8e3531296 100644 --- a/quickwit/quickwit-control-plane/src/lib.rs +++ b/quickwit/quickwit-control-plane/src/lib.rs @@ -18,6 +18,7 @@ pub mod indexing_scheduler; pub mod ingest; pub(crate) mod metrics; pub(crate) mod model; +mod telemetry; use quickwit_common::tower::Pool; use quickwit_proto::indexing::{CpuCapacity, IndexingServiceClient, IndexingTask}; diff --git a/quickwit/quickwit-control-plane/src/telemetry.rs b/quickwit/quickwit-control-plane/src/telemetry.rs new file mode 100644 index 00000000000..bf022366f4c --- /dev/null +++ b/quickwit/quickwit-control-plane/src/telemetry.rs @@ -0,0 +1,85 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeSet; + +use quickwit_config::{CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID}; +use quickwit_proto::metastore::SourceType; + +use crate::model::ControlPlaneModel; + +const RESERVED_SOURCE_IDS: [&str; 3] = [CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID]; + +pub(crate) fn update_telemetry_cluster_summary(model: &ControlPlaneModel) { + let (num_indexes, source_types) = cluster_summary(model); + quickwit_telemetry::set_cluster_summary(num_indexes, source_types); +} + +fn cluster_summary(model: &ControlPlaneModel) -> (usize, BTreeSet) { + let source_types = model + .source_configs() + .filter(|(source_uid, _)| !RESERVED_SOURCE_IDS.contains(&source_uid.source_id.as_str())) + .filter_map(|(_, source_config)| source_type_name(source_config.source_type())) + .map(str::to_string) + .collect(); + (model.num_indexes(), source_types) +} + +fn source_type_name(source_type: SourceType) -> Option<&'static str> { + match source_type { + SourceType::Cli => Some("cli"), + SourceType::File => Some("file"), + SourceType::IngestV1 => Some("ingest_v1"), + SourceType::IngestV2 => Some("ingest_v2"), + SourceType::Kafka => Some("kafka"), + SourceType::Kinesis => Some("kinesis"), + SourceType::Nats => Some("nats"), + SourceType::PubSub => Some("pubsub"), + SourceType::Pulsar => Some("pulsar"), + SourceType::Stdin => Some("stdin"), + SourceType::Vec => Some("vec"), + SourceType::Void => Some("void"), + SourceType::Unspecified => None, + } +} + +#[cfg(test)] +mod tests { + use quickwit_config::{SourceConfig, SourceParams}; + use quickwit_metastore::IndexMetadata; + + use super::*; + + #[test] + fn test_cluster_summary_leaves_out_reserved_sources() { + let mut model = ControlPlaneModel::default(); + assert_eq!(cluster_summary(&model), (0, BTreeSet::new())); + + let mut index_metadata = IndexMetadata::for_test("test-index", "ram:///indexes"); + index_metadata.add_source(SourceConfig::cli()).unwrap(); + index_metadata + .add_source(SourceConfig::ingest_v2()) + .unwrap(); + index_metadata + .add_source(SourceConfig::for_test("my-source", SourceParams::void())) + .unwrap(); + model.add_index(index_metadata); + model.add_index(IndexMetadata::for_test("other-index", "ram:///indexes")); + + assert_eq!( + cluster_summary(&model), + (2, BTreeSet::from(["void".to_string()])) + ); + } +} diff --git a/quickwit/quickwit-serve/src/api_usage.rs b/quickwit/quickwit-serve/src/api_usage.rs new file mode 100644 index 00000000000..f10f9139515 --- /dev/null +++ b/quickwit/quickwit-serve/src/api_usage.rs @@ -0,0 +1,119 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use quickwit_telemetry::ApiKind; + +pub(crate) fn classify_rest_request(path: &str) -> Option { + let api_path = path.strip_prefix("/api/v1/")?; + let segments: Vec<&str> = api_path.split('/').collect(); + if segments[0] == "_elastic" { + return Some(classify_elasticsearch_request(&segments[1..])); + } + // `/api/v1/otlp/v1/{logs,traces}`, before the index routes since an index may be named `otlp`. + if segments[0] == "otlp" && segments.get(1) == Some(&"v1") { + return Some(ApiKind::Otlp); + } + match segments.get(1) { + Some(&"search") => Some(ApiKind::NativeSearch), + Some(&"ingest") => Some(ApiKind::NativeIngest), + Some(&"otlp") => Some(ApiKind::Otlp), + Some(&"jaeger") => Some(ApiKind::Jaeger), + _ => None, + } +} + +fn classify_elasticsearch_request(segments: &[&str]) -> ApiKind { + if segments.contains(&"_bulk") { + return ApiKind::ElasticsearchIngest; + } + let is_search = segments + .iter() + .any(|segment| matches!(*segment, "_search" | "_msearch" | "_count")); + if is_search { + return ApiKind::ElasticsearchSearch; + } + ApiKind::ElasticsearchOther +} + +pub(crate) fn count_otlp_grpc_request( + request: tonic::Request<()>, +) -> Result, tonic::Status> { + quickwit_telemetry::record_api_request(ApiKind::Otlp); + Ok(request) +} + +pub(crate) fn count_jaeger_grpc_request( + request: tonic::Request<()>, +) -> Result, tonic::Status> { + quickwit_telemetry::record_api_request(ApiKind::Jaeger); + Ok(request) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_classify_rest_request() { + let cases = [ + ("/api/v1/_elastic/_bulk", Some(ApiKind::ElasticsearchIngest)), + ( + "/api/v1/_elastic/logs/_bulk", + Some(ApiKind::ElasticsearchIngest), + ), + ( + "/api/v1/_elastic/_search", + Some(ApiKind::ElasticsearchSearch), + ), + ( + "/api/v1/_elastic/logs/_search", + Some(ApiKind::ElasticsearchSearch), + ), + ( + "/api/v1/_elastic/_msearch", + Some(ApiKind::ElasticsearchSearch), + ), + ( + "/api/v1/_elastic/_search/scroll", + Some(ApiKind::ElasticsearchSearch), + ), + ( + "/api/v1/_elastic/logs/_count", + Some(ApiKind::ElasticsearchSearch), + ), + ( + "/api/v1/_elastic/_field_caps", + Some(ApiKind::ElasticsearchOther), + ), + ("/api/v1/_elastic", Some(ApiKind::ElasticsearchOther)), + ("/api/v1/logs/search", Some(ApiKind::NativeSearch)), + ("/api/v1/logs/ingest", Some(ApiKind::NativeIngest)), + ("/api/v1/otlp/v1/logs", Some(ApiKind::Otlp)), + ("/api/v1/otlp/v1/traces", Some(ApiKind::Otlp)), + ("/api/v1/my-logs/otlp/v1/logs", Some(ApiKind::Otlp)), + ("/api/v1/otlp/search", Some(ApiKind::NativeSearch)), + ( + "/api/v1/otel-traces-v0_9/jaeger/api/services", + Some(ApiKind::Jaeger), + ), + ("/api/v1/indexes", None), + ("/api/v1/indexes/logs/sources", None), + ("/health/livez", None), + ("/ui/search", None), + ]; + for (path, expected_api_kind) in cases { + assert_eq!(classify_rest_request(path), expected_api_kind, "{path}"); + } + } +} diff --git a/quickwit/quickwit-serve/src/grpc.rs b/quickwit/quickwit-serve/src/grpc.rs index ae33f21cde0..8435d0576f9 100644 --- a/quickwit/quickwit-serve/src/grpc.rs +++ b/quickwit/quickwit-serve/src/grpc.rs @@ -32,12 +32,14 @@ use quickwit_proto::tonic::transport::Server; use quickwit_proto::tonic::transport::server::TcpIncoming; use tokio::net::TcpListener; use tokio_rustls::TlsAcceptor; +use tonic::service::interceptor::InterceptedService; use tonic_health::pb::FILE_DESCRIPTOR_SET as HEALTH_FILE_DESCRIPTOR_SET; use tonic_health::pb::health_server::{Health, HealthServer}; use tonic_reflection::pb::v1::FILE_DESCRIPTOR_SET as REFLECTION_FILE_DESCRIPTOR_SET; use tonic_reflection::server::v1::{ServerReflection, ServerReflectionServer}; use tracing::*; +use crate::api_usage::{count_jaeger_grpc_request, count_otlp_grpc_request}; use crate::developer_api::DeveloperApiServer; use crate::search_api::GrpcSearchAdapter; use crate::{INDEXING_GRPC_SERVER_METRICS_LAYER, QuickwitServices}; @@ -175,7 +177,10 @@ pub(crate) async fn start_grpc_server( .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(grpc_config.max_message_size.0 as usize) .max_encoding_message_size(grpc_config.max_message_size.0 as usize); - Some(trace_service) + Some(InterceptedService::new( + trace_service, + count_otlp_grpc_request, + )) } else { None }; @@ -187,7 +192,10 @@ pub(crate) async fn start_grpc_server( .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(grpc_config.max_message_size.0 as usize) .max_encoding_message_size(grpc_config.max_message_size.0 as usize); - Some(logs_service) + Some(InterceptedService::new( + logs_service, + count_otlp_grpc_request, + )) } else { None }; @@ -213,7 +221,10 @@ pub(crate) async fn start_grpc_server( // Mount gRPC jaeger service if present. let jaeger_grpc_service = if let Some(jaeger_service) = services.jaeger_service_opt.clone() { enabled_grpc_services.insert("jaeger"); - Some(SpanReaderPluginServer::new(jaeger_service)) + Some(InterceptedService::new( + SpanReaderPluginServer::new(jaeger_service), + count_jaeger_grpc_request, + )) } else { None }; @@ -221,7 +232,10 @@ pub(crate) async fn start_grpc_server( // Mount gRPC jaeger v2 service (TraceReader) if present. let jaeger_v2_grpc_service = if let Some(jaeger_service) = services.jaeger_service_opt.clone() { enabled_grpc_services.insert("jaeger-v2"); - Some(TraceReaderServer::new(jaeger_service)) + Some(InterceptedService::new( + TraceReaderServer::new(jaeger_service), + count_jaeger_grpc_request, + )) } else { None }; diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 3fcbb2d5593..a2437cbc102 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -14,6 +14,7 @@ #![recursion_limit = "256"] +mod api_usage; mod build_info; mod cluster_api; mod decompression; diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index 56bc33f91b2..c50190d0f57 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -44,6 +44,7 @@ use warp::hyper::http::HeaderValue; use warp::hyper::{Method, StatusCode, http}; use warp::{Filter, Rejection, Reply, redirect}; +use crate::api_usage::classify_rest_request; use crate::cluster_api::cluster_handler; use crate::decompression::{CorruptedData, UnsupportedEncoding}; use crate::delete_task_api::delete_task_api_handlers; @@ -167,6 +168,9 @@ pub(crate) async fn start_rest_server( labels: [labels], ) .inc(); + if let Some(api_kind) = classify_rest_request(info.path()) { + quickwit_telemetry::record_api_request(api_kind); + } }); // Docs routes let api_doc = warp::path("openapi.json") diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index 3b99c35c89c..b94e21cf0a4 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -29,6 +29,7 @@ quickwit-common = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +tempfile = { workspace = true } [package.metadata.cargo-machete] # see above diff --git a/quickwit/quickwit-telemetry/src/install.rs b/quickwit/quickwit-telemetry/src/install.rs new file mode 100644 index 00000000000..514d9beb12a --- /dev/null +++ b/quickwit/quickwit-telemetry/src/install.rs @@ -0,0 +1,76 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io; +use std::path::Path; + +use uuid::Uuid; + +const INSTALL_ID_FILE_NAME: &str = "telemetry_install_id"; + +/// Returns the install ID stored in `data_dir`, creating it if needed, and whether it was just +/// created. An invalid ID file is an error rather than being replaced. +pub fn load_or_create_install_id(data_dir: &Path) -> io::Result<(Uuid, bool)> { + let install_id_path = data_dir.join(INSTALL_ID_FILE_NAME); + match std::fs::read_to_string(&install_id_path) { + Ok(install_id_str) => { + let install_id = Uuid::parse_str(install_id_str.trim()).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "invalid install ID in `{}`: {error}", + install_id_path.display() + ), + ) + })?; + return Ok((install_id, false)); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let install_id = Uuid::new_v4(); + std::fs::create_dir_all(data_dir)?; + // Write to a temporary file, then rename it, so that a crash never leaves a truncated ID. + let tmp_install_id_path = data_dir.join(format!("{INSTALL_ID_FILE_NAME}.tmp")); + std::fs::write(&tmp_install_id_path, install_id.to_string())?; + std::fs::rename(&tmp_install_id_path, &install_id_path)?; + Ok((install_id, true)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_load_or_create_install_id() { + let temp_dir = tempfile::tempdir().unwrap(); + let data_dir = temp_dir.path().join("qwdata"); + + let (install_id, created) = load_or_create_install_id(&data_dir).unwrap(); + assert!(created); + + let (reloaded_install_id, created) = load_or_create_install_id(&data_dir).unwrap(); + assert!(!created); + assert_eq!(reloaded_install_id, install_id); + } + + #[test] + fn test_load_or_create_install_id_rejects_invalid_file() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::write(temp_dir.path().join(INSTALL_ID_FILE_NAME), "not-a-uuid").unwrap(); + + let error = load_or_create_install_id(temp_dir.path()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } +} diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 055aed23d40..76892d80e41 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -15,20 +15,25 @@ #![allow(clippy::bool_assert_comparison)] #![deny(clippy::disallowed_methods)] +mod install; +mod machine; pub mod payload; /// This crate contains the code responsible for sending anonymous usage data to the telemetry /// relay. See `docs/telemetry.md` for the data that is sent. mod sender; pub(crate) mod sink; +mod usage; +use std::ffi::OsStr; use std::sync::OnceLock; use payload::QuickwitTelemetryInfo; use tracing::info; +pub use crate::install::load_or_create_install_id; use crate::payload::TelemetryEvent; -pub use crate::sender::is_telemetry_disabled; use crate::sender::{TelemetryLoopHandle, TelemetrySender}; +pub use crate::usage::{ApiKind, record_api_request, set_cluster_summary}; static TELEMETRY_SENDER: OnceLock = OnceLock::new(); @@ -64,3 +69,53 @@ pub async fn send_telemetry_event(event: TelemetryEvent) { /// This environment variable can be set to disable sending telemetry events. pub const DISABLE_TELEMETRY_ENV_KEY: &str = "QW_DISABLE_TELEMETRY"; + +/// telemetry is disabled in tests. +#[cfg(test)] +pub fn is_telemetry_disabled() -> bool { + true +} +/// Check to see if telemetry is enabled. +#[cfg(not(test))] +pub fn is_telemetry_disabled() -> bool { + let env_value_opt = std::env::var_os(crate::DISABLE_TELEMETRY_ENV_KEY); + is_telemetry_disabled_by_env_value(env_value_opt.as_deref()) +} + +/// Telemetry is disabled when `QW_DISABLE_TELEMETRY` is set to any non-empty value other than +/// `false`, `no` or `0`. Unrecognized values disable telemetry so that a mistyped opt-out never +/// leaves telemetry on. +fn is_telemetry_disabled_by_env_value(env_value_opt: Option<&OsStr>) -> bool { + let Some(env_value) = env_value_opt else { + return false; + }; + let Some(env_value_str) = env_value.to_str() else { + return true; + }; + if env_value_str.trim().is_empty() { + return false; + } + quickwit_common::parse_bool_lenient(env_value_str) != Some(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_telemetry_disabled_by_env_value() { + assert!(!is_telemetry_disabled_by_env_value(None)); + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new("")))); + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new(" ")))); + for enabled_value in ["false", "FALSE", "no", "0"] { + assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new( + enabled_value + )))); + } + for disabled_value in ["1", "true", "yes", "on", "disabled", "anything"] { + assert!(is_telemetry_disabled_by_env_value(Some(OsStr::new( + disabled_value + )))); + } + } +} diff --git a/quickwit/quickwit-telemetry/src/machine.rs b/quickwit/quickwit-telemetry/src/machine.rs new file mode 100644 index 00000000000..7165345dc6a --- /dev/null +++ b/quickwit/quickwit-telemetry/src/machine.rs @@ -0,0 +1,105 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const GIB: u64 = 1 << 30; + +pub(crate) fn cpu_count_bucket(num_cpus: usize) -> &'static str { + match num_cpus { + 0 | 1 => "1", + 2 => "2", + 3..=4 => "3-4", + 5..=8 => "5-8", + 9..=16 => "9-16", + 17..=32 => "17-32", + 33..=64 => "33-64", + _ => "65+", + } +} + +pub(crate) fn memory_bucket(num_bytes: u64) -> &'static str { + match num_bytes { + num_bytes if num_bytes < 2 * GIB => "<2GiB", + num_bytes if num_bytes < 4 * GIB => "2-4GiB", + num_bytes if num_bytes < 8 * GIB => "4-8GiB", + num_bytes if num_bytes < 16 * GIB => "8-16GiB", + num_bytes if num_bytes < 32 * GIB => "16-32GiB", + num_bytes if num_bytes < 64 * GIB => "32-64GiB", + num_bytes if num_bytes < 128 * GIB => "64-128GiB", + _ => "128GiB+", + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn available_memory_num_bytes() -> Option { + let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?; + let physical_memory_num_bytes = parse_mem_total_num_bytes(&meminfo)?; + let Some(cgroup_limit_num_bytes) = cgroup_memory_limit_num_bytes() else { + return Some(physical_memory_num_bytes); + }; + Some(physical_memory_num_bytes.min(cgroup_limit_num_bytes)) +} + +#[cfg(not(target_os = "linux"))] +pub(crate) fn available_memory_num_bytes() -> Option { + None +} + +#[cfg(any(target_os = "linux", test))] +fn parse_mem_total_num_bytes(meminfo: &str) -> Option { + let mem_total_kib: u64 = meminfo + .lines() + .find_map(|line| line.strip_prefix("MemTotal:"))? + .trim() + .strip_suffix("kB")? + .trim() + .parse() + .ok()?; + Some(mem_total_kib * 1024) +} + +/// Without a limit, cgroup v2 holds `max`, which doesn't parse, and cgroup v1 a huge number, which +/// the caller caps with the physical memory. +#[cfg(target_os = "linux")] +fn cgroup_memory_limit_num_bytes() -> Option { + for path in [ + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + ] { + if let Ok(limit) = std::fs::read_to_string(path) { + return limit.trim().parse().ok(); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_buckets() { + assert_eq!(cpu_count_bucket(4), "3-4"); + assert_eq!(cpu_count_bucket(96), "65+"); + assert_eq!(memory_bucket(GIB), "<2GiB"); + assert_eq!(memory_bucket(31 * GIB), "16-32GiB"); + assert_eq!(memory_bucket(512 * GIB), "128GiB+"); + } + + #[test] + fn test_parse_mem_total_num_bytes() { + let meminfo = "MemTotal: 16303516 kB\nMemFree: 1102768 kB\n"; + assert_eq!(parse_mem_total_num_bytes(meminfo), Some(16_303_516 * 1024)); + assert_eq!(parse_mem_total_num_bytes("MemFree: 1 kB\n"), None); + } +} diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index fe3f2d7f3e7..d406285c9a0 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -12,16 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::env; use std::time::UNIX_EPOCH; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::machine::{available_memory_num_bytes, cpu_count_bucket, memory_bucket}; +use crate::usage::{ApiKind, ClusterSummary}; + +/// Version of the payload format. Payloads without one are version 1. +pub(crate) const TELEMETRY_SCHEMA_VERSION: u32 = 2; + /// Represents the payload of the request sent with telemetry requests. #[derive(Debug, Serialize, Deserialize)] pub struct TelemetryPayload { + pub schema_version: u32, /// Client information. See details in `[ClientInformation]`. pub client_info: ClientInfo, /// Quickwit information. See details in `[QuickwitInfo]`. @@ -69,11 +76,22 @@ pub enum TelemetryEvent { /// EndCommand (with the return code). EndCommand { return_code: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + api_usage: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + cluster_summary: Option, }, /// Event sent every 12h to signal the server is running. - Running, + Running { + #[serde(default, skip_serializing_if = "Option::is_none")] + api_usage: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + cluster_summary: Option, + }, /// UI index.html was requested. UiIndexPageLoad, + /// The install ID was just created. + Install, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -83,6 +101,26 @@ pub struct ClientInfo { arch: String, hashed_host_username: String, kubernetes: bool, + cpus: String, + #[serde(skip_serializing_if = "Option::is_none")] + memory: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BuildTelemetryInfo { + pub commit: String, + pub target: String, + pub release: bool, +} + +impl Default for BuildTelemetryInfo { + fn default() -> Self { + Self { + commit: "unknown".to_string(), + target: "unknown".to_string(), + release: false, + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -90,14 +128,19 @@ pub struct QuickwitTelemetryInfo { pub version: String, pub services: HashSet, pub features: HashSet, + pub build: BuildTelemetryInfo, + #[serde(skip_serializing_if = "Option::is_none")] + pub install_uuid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hashed_cluster_id: Option, } impl QuickwitTelemetryInfo { pub fn new(services: HashSet, features: HashSet) -> Self { Self { features, - version: env!("CARGO_PKG_VERSION").to_string(), services, + ..Default::default() } } } @@ -108,6 +151,9 @@ impl Default for QuickwitTelemetryInfo { features: HashSet::new(), version: env!("CARGO_PKG_VERSION").to_string(), services: HashSet::new(), + build: BuildTelemetryInfo::default(), + install_uuid: None, + hashed_cluster_id: None, } } } @@ -115,26 +161,33 @@ impl Default for QuickwitTelemetryInfo { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum QuickwitFeature { - /// The default index root URI is on Azure Blob Storage. AzureStorage, + DocsClustering, FileBackedMetastore, - /// The default index root URI is on the local file system. FileStorage, - /// The default index root URI is on Google Cloud Storage. GoogleCloudStorage, + IngestV1, + IngestV2, Jaeger, - /// The searcher offloads leaf searches to AWS Lambda. LambdaLeafSearch, Otlp, PostgresqMetastore, - /// The default index root URI is on S3 or an S3-compatible object store. + S3CustomEndpoint, + S3FlavorDigitalOcean, + S3FlavorGarage, + S3FlavorGcs, + S3FlavorMinio, S3Storage, - /// The searcher caches splits on local disk. SplitCache, - /// The REST or gRPC server uses TLS. Tls, } +/// Callers must strip the credentials from the metastore URI first. +pub fn hash_cluster_identity(cluster_id: &str, redacted_metastore_uri: &str) -> String { + let digest = md5::compute(format!("{cluster_id}\n{redacted_metastore_uri}")); + format!("{digest:x}") +} + fn hashed_host_username() -> String { let hostname = hostname::get() .map(|hostname| hostname.to_string_lossy().to_string()) @@ -153,6 +206,9 @@ impl Default for ClientInfo { arch: env::consts::ARCH.to_string(), hashed_host_username: hashed_host_username(), kubernetes: std::env::var_os("KUBERNETES_SERVICE_HOST").is_some(), + cpus: cpu_count_bucket(quickwit_common::num_cpus()).to_string(), + memory: available_memory_num_bytes() + .map(|num_bytes| memory_bucket(num_bytes).to_string()), } } } @@ -161,20 +217,28 @@ impl Default for ClientInfo { mod tests { use serde_json; - use super::{EventWithTimestamp, QuickwitFeature, TelemetryEvent}; + use super::*; #[test] fn test_serialize_features_as_expected() { // These names are part of the telemetry payload documented in `docs/telemetry.md`. let features = [ QuickwitFeature::AzureStorage, + QuickwitFeature::DocsClustering, QuickwitFeature::FileBackedMetastore, QuickwitFeature::FileStorage, QuickwitFeature::GoogleCloudStorage, + QuickwitFeature::IngestV1, + QuickwitFeature::IngestV2, QuickwitFeature::Jaeger, QuickwitFeature::LambdaLeafSearch, QuickwitFeature::Otlp, QuickwitFeature::PostgresqMetastore, + QuickwitFeature::S3CustomEndpoint, + QuickwitFeature::S3FlavorDigitalOcean, + QuickwitFeature::S3FlavorGarage, + QuickwitFeature::S3FlavorGcs, + QuickwitFeature::S3FlavorMinio, QuickwitFeature::S3Storage, QuickwitFeature::SplitCache, QuickwitFeature::Tls, @@ -182,7 +246,7 @@ mod tests { let json = serde_json::to_string(&features).unwrap(); assert_eq!( json, - r#"["azure_storage","file_backed_metastore","file_storage","google_cloud_storage","jaeger","lambda_leaf_search","otlp","postgresq_metastore","s3_storage","split_cache","tls"]"# + r#"["azure_storage","docs_clustering","file_backed_metastore","file_storage","google_cloud_storage","ingest_v1","ingest_v2","jaeger","lambda_leaf_search","otlp","postgresq_metastore","s3_custom_endpoint","s3_flavor_digital_ocean","s3_flavor_garage","s3_flavor_gcs","s3_flavor_minio","s3_storage","split_cache","tls"]"# ); } @@ -190,7 +254,11 @@ mod tests { fn test_serialize_payload_as_expected() { let event = EventWithTimestamp { unixtime: 0, - event: TelemetryEvent::EndCommand { return_code: 0 }, + event: TelemetryEvent::EndCommand { + return_code: 0, + api_usage: None, + cluster_summary: None, + }, }; let json = serde_json::to_string(&event).unwrap(); assert_eq!( @@ -198,4 +266,23 @@ mod tests { r#"{"unixtime":0,"type":"end_command","return_code":0}"# ); } + + #[test] + fn test_serialize_activity_as_expected() { + let event = EventWithTimestamp { + unixtime: 0, + event: TelemetryEvent::Running { + api_usage: Some(BTreeMap::from([(ApiKind::NativeSearch, "1-9".to_string())])), + cluster_summary: Some(ClusterSummary { + num_indexes: "2-10".to_string(), + source_types: ["kafka".to_string()].into(), + }), + }, + }; + let json = serde_json::to_string(&event).unwrap(); + assert_eq!( + json, + r#"{"unixtime":0,"type":"running","api_usage":{"native_search":"1-9"},"cluster_summary":{"num_indexes":"2-10","source_types":["kafka"]}}"# + ); + } } diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 555e8950e64..82f83218e4e 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ffi::OsStr; use std::mem; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -24,8 +23,10 @@ use tokio::task::JoinHandle; use tokio::time::Interval; use tracing::info; +use crate::is_telemetry_disabled; use crate::payload::{ - ClientInfo, EventWithTimestamp, QuickwitTelemetryInfo, TelemetryEvent, TelemetryPayload, + ClientInfo, EventWithTimestamp, QuickwitTelemetryInfo, TELEMETRY_SCHEMA_VERSION, + TelemetryEvent, TelemetryPayload, }; use crate::sink::{HttpClient, Sink}; @@ -167,6 +168,7 @@ impl Inner { async fn create_telemetry_payload(&self) -> TelemetryPayload { let events_state = self.events.drain_events().await; TelemetryPayload { + schema_version: TELEMETRY_SCHEMA_VERSION, client_info: self.client_info.clone(), quickwit_info: self.quickwit_info.clone(), events: events_state.events, @@ -297,34 +299,6 @@ impl TelemetrySender { } } -/// telemetry is disabled in tests. -#[cfg(test)] -pub fn is_telemetry_disabled() -> bool { - true -} -/// Check to see if telemetry is enabled. -#[cfg(not(test))] -pub fn is_telemetry_disabled() -> bool { - let env_value_opt = std::env::var_os(crate::DISABLE_TELEMETRY_ENV_KEY); - is_telemetry_disabled_by_env_value(env_value_opt.as_deref()) -} - -/// Telemetry is disabled when `QW_DISABLE_TELEMETRY` is set to any non-empty value other than -/// `false`, `no` or `0`. Unrecognized values disable telemetry so that a mistyped opt-out never -/// leaves telemetry on. -fn is_telemetry_disabled_by_env_value(env_value_opt: Option<&OsStr>) -> bool { - let Some(env_value) = env_value_opt else { - return false; - }; - let Some(env_value_str) = env_value.to_str() else { - return true; - }; - if env_value_str.trim().is_empty() { - return false; - } - quickwit_common::parse_bool_lenient(env_value_str) != Some(false) -} - fn start_monitor_if_server_running_task(telemetry_sender: Arc) { let mut clock = tokio::time::interval(TELEMETRY_RUNNING_EVENT_INTERVAL); tokio::spawn(async move { @@ -332,7 +306,7 @@ fn start_monitor_if_server_running_task(telemetry_sender: Arc) { clock.tick().await; loop { clock.tick().await; - telemetry_sender.send(TelemetryEvent::Running).await; + telemetry_sender.send(TelemetryEvent::running()).await; } }); } @@ -354,23 +328,6 @@ mod tests { use super::*; - #[test] - fn test_is_telemetry_disabled_by_env_value() { - assert!(!is_telemetry_disabled_by_env_value(None)); - assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new("")))); - assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new(" ")))); - for enabled_value in ["false", "FALSE", "no", "0"] { - assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new( - enabled_value - )))); - } - for disabled_value in ["1", "true", "yes", "on", "disabled", "anything"] { - assert!(is_telemetry_disabled_by_env_value(Some(OsStr::new( - disabled_value - )))); - } - } - #[ignore] #[tokio::test] async fn test_enabling_and_disabling_telemetry() { @@ -448,7 +405,10 @@ mod tests { { let payload = rx.recv().await.unwrap(); assert_eq!(payload.events.len(), 1); - assert_eq!(payload.events[0].event, TelemetryEvent::Running); + assert!(matches!( + payload.events[0].event, + TelemetryEvent::Running { .. } + )); } loop_handler.terminate_telemetry().await; } @@ -491,7 +451,7 @@ mod tests { let payload = rx.recv().await.unwrap(); assert_eq!(payload.events.len(), 1); telemetry_sender - .send(TelemetryEvent::EndCommand { return_code: 2i32 }) + .send(TelemetryEvent::end_command(2i32)) .await; loop_handler.terminate_telemetry().await; let payload = rx.recv().await.unwrap(); diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs new file mode 100644 index 00000000000..d37e5dcf28d --- /dev/null +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -0,0 +1,188 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, PoisonError}; + +use serde::{Deserialize, Serialize}; + +use crate::payload::TelemetryEvent; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ApiKind { + ElasticsearchSearch, + ElasticsearchIngest, + ElasticsearchOther, + NativeSearch, + NativeIngest, + Otlp, + Jaeger, +} + +impl ApiKind { + /// In declaration order: `ApiRequestCounts` is indexed by `api_kind as usize`. + const ALL: [ApiKind; 7] = [ + ApiKind::ElasticsearchSearch, + ApiKind::ElasticsearchIngest, + ApiKind::ElasticsearchOther, + ApiKind::NativeSearch, + ApiKind::NativeIngest, + ApiKind::Otlp, + ApiKind::Jaeger, + ]; +} + +struct ApiRequestCounts([AtomicU64; ApiKind::ALL.len()]); + +impl ApiRequestCounts { + const fn new() -> Self { + Self([const { AtomicU64::new(0) }; ApiKind::ALL.len()]) + } + + fn record(&self, api_kind: ApiKind) { + self.0[api_kind as usize].fetch_add(1, Ordering::Relaxed); + } + + /// Returns the bucketed counts of the APIs that received requests, and resets the counts. + fn take(&self) -> Option> { + let mut api_usage = BTreeMap::new(); + for api_kind in ApiKind::ALL { + let num_requests = self.0[api_kind as usize].swap(0, Ordering::Relaxed); + if num_requests > 0 { + api_usage.insert(api_kind, request_count_bucket(num_requests).to_string()); + } + } + if api_usage.is_empty() { + return None; + } + Some(api_usage) + } +} + +static API_REQUEST_COUNTS: ApiRequestCounts = ApiRequestCounts::new(); + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct ClusterSummary { + pub num_indexes: String, + pub source_types: BTreeSet, +} + +static CLUSTER_SUMMARY: Mutex> = Mutex::new(None); + +pub fn record_api_request(api_kind: ApiKind) { + API_REQUEST_COUNTS.record(api_kind); +} + +/// Only the node running the control plane calls it, so each cluster reports its summary once. +pub fn set_cluster_summary(num_indexes: usize, source_types: BTreeSet) { + let cluster_summary = ClusterSummary { + num_indexes: index_count_bucket(num_indexes).to_string(), + source_types, + }; + *CLUSTER_SUMMARY + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(cluster_summary); +} + +impl TelemetryEvent { + /// Takes the API request counts since the previous report. + pub fn running() -> Self { + TelemetryEvent::Running { + api_usage: API_REQUEST_COUNTS.take(), + cluster_summary: cluster_summary(), + } + } + + /// Takes the API request counts since the previous report. + pub fn end_command(return_code: i32) -> Self { + TelemetryEvent::EndCommand { + return_code, + api_usage: API_REQUEST_COUNTS.take(), + cluster_summary: cluster_summary(), + } + } +} + +fn cluster_summary() -> Option { + CLUSTER_SUMMARY + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() +} + +fn request_count_bucket(num_requests: u64) -> &'static str { + match num_requests { + 0 => "0", + 1..=9 => "1-9", + 10..=99 => "10-99", + 100..=999 => "100-999", + 1_000..=9_999 => "1000-9999", + 10_000..=99_999 => "10000-99999", + 100_000..=999_999 => "100000-999999", + _ => "1000000+", + } +} + +fn index_count_bucket(num_indexes: usize) -> &'static str { + match num_indexes { + 0 => "0", + 1 => "1", + 2..=10 => "2-10", + 11..=100 => "11-100", + 101..=1_000 => "101-1000", + _ => "1001+", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_api_kind_all_matches_discriminants() { + for (index, api_kind) in ApiKind::ALL.iter().enumerate() { + assert_eq!(*api_kind as usize, index); + } + } + + #[test] + fn test_api_request_counts_reports_buckets_and_resets() { + let api_request_counts = ApiRequestCounts::new(); + api_request_counts.record(ApiKind::NativeSearch); + for _ in 0..12 { + api_request_counts.record(ApiKind::ElasticsearchIngest); + } + let api_usage = api_request_counts.take().unwrap(); + assert_eq!( + api_usage, + BTreeMap::from([ + (ApiKind::ElasticsearchIngest, "10-99".to_string()), + (ApiKind::NativeSearch, "1-9".to_string()), + ]) + ); + assert_eq!(api_request_counts.take(), None); + } + + #[test] + fn test_buckets() { + assert_eq!(request_count_bucket(9), "1-9"); + assert_eq!(request_count_bucket(10), "10-99"); + assert_eq!(request_count_bucket(1_000_000), "1000000+"); + assert_eq!(index_count_bucket(1), "1"); + assert_eq!(index_count_bucket(2), "2-10"); + assert_eq!(index_count_bucket(1_001), "1001+"); + } +} From e139dec72e80043526777f6ac05807b7a10039ad Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Fri, 23 Oct 2026 18:11:37 +0000 Subject: [PATCH 04/13] Move the telemetry relay out and simplify telemetry The relay now lives with its infrastructure in a separate repository. Read the memory size with sysinfo, replace an invalid install ID instead of failing, and trim the tests. Leave the database user and query parameters out of the cluster hash, take the API request counts only once their event is queued, report the OTLP and Jaeger features only on the nodes that serve them, and disable telemetry in the UI CI. --- .github/workflows/ui-ci.yml | 1 + LICENSE-3rdparty.csv | 1 + distribution/telemetry-relay/README.md | 66 ---- distribution/telemetry-relay/package.json | 8 - distribution/telemetry-relay/src/lambda.js | 122 ------- distribution/telemetry-relay/src/relay.js | 335 ------------------ .../telemetry-relay/test/lambda.test.js | 111 ------ .../telemetry-relay/test/relay.test.js | 225 ------------ docs/telemetry.md | 6 +- quickwit/Cargo.lock | 12 + quickwit/Cargo.toml | 2 +- quickwit/quickwit-cli/src/telemetry.rs | 93 ++--- quickwit/quickwit-serve/src/api_usage.rs | 27 +- quickwit/quickwit-telemetry/Cargo.toml | 1 + quickwit/quickwit-telemetry/src/install.rs | 31 +- quickwit/quickwit-telemetry/src/machine.rs | 56 +-- quickwit/quickwit-telemetry/src/payload.rs | 46 ++- quickwit/quickwit-telemetry/src/sender.rs | 3 +- quickwit/quickwit-telemetry/src/usage.rs | 36 +- 19 files changed, 127 insertions(+), 1055 deletions(-) delete mode 100644 distribution/telemetry-relay/README.md delete mode 100644 distribution/telemetry-relay/package.json delete mode 100644 distribution/telemetry-relay/src/lambda.js delete mode 100644 distribution/telemetry-relay/src/relay.js delete mode 100644 distribution/telemetry-relay/test/lambda.test.js delete mode 100644 distribution/telemetry-relay/test/relay.test.js diff --git a/.github/workflows/ui-ci.yml b/.github/workflows/ui-ci.yml index f9f59e8bdd5..440e0959682 100644 --- a/.github/workflows/ui-ci.yml +++ b/.github/workflows/ui-ci.yml @@ -63,6 +63,7 @@ jobs: --health-retries 5 env: CARGO_INCREMENTAL: 0 + QW_DISABLE_TELEMETRY: 1 RUST_BACKTRACE: 1 RUSTFLAGS: -Dwarnings --cfg tokio_unstable RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 698238bb139..91eadd68792 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -440,6 +440,7 @@ nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nom-language,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs normalize-line-endings,https://github.com/derekdreery/normalize-line-endings,Apache-2.0,Richard Dodd +ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" num,https://github.com/rust-num/num,MIT OR Apache-2.0,The Rust Project Developers num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Project Developers diff --git a/distribution/telemetry-relay/README.md b/distribution/telemetry-relay/README.md deleted file mode 100644 index 31d141fa96f..00000000000 --- a/distribution/telemetry-relay/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Quickwit telemetry relay - -Quickwit nodes send anonymous usage data to `https://telemetry.quickwit.io/` (see [docs/telemetry.md](../../docs/telemetry.md)). This relay receives it and forwards it to Datadog Logs. It runs on AWS Lambda, behind CloudFront and an API Gateway HTTP API, and has no dependencies. - -For each request, the relay: - -1. Rejects it unless its body matches the payload format of `quickwit/quickwit-telemetry/src/payload.rs`, and keeps only the documented fields. Releases 0.6.0 and later all send this format. -2. Ignores the client IP address and request headers, except the country and city that CloudFront derives from the IP address. -3. Replaces `hashed_host_username` and `hashed_cluster_id`, unsalted MD5 hashes that can be reversed by guessing, with HMAC-SHA256 hashes keyed with `INSTALL_ID_SECRET`: `client.install_id` and `quickwit.cluster_id`. -4. Sends one log per event to the Datadog Logs intake API. - -## Log format - -```json -{ - "ddsource": "quickwit-telemetry", - "service": "quickwit-telemetry", - "message": "end_command", - "schema_version": 2, - "event": { - "type": "end_command", - "client_unixtime": 1760000060, - "return_code": 0, - "api_usage": { "elasticsearch_ingest": "100-999", "native_search": "1-9" }, - "cluster_summary": { "num_indexes": "2-10", "source_types": ["kafka"] } - }, - "client": { - "session_uuid": "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", - "install_id": "<64 hex characters>", - "install_uuid": "8c4f2a5e-3b1d-4e7f-9a6c-2d5e8f1b3c7a", - "os": "linux", - "arch": "x86_64", - "kubernetes": true, - "cpus": "5-8", - "memory": "8-16GiB" - }, - "quickwit": { - "version": "0.9.1", - "services": ["indexer", "searcher"], - "features": ["ingest_v2", "otlp", "s3_storage"], - "build": { "commit": "4f20e4f", "target": "x86_64-unknown-linux-gnu", "release": true }, - "cluster_id": "<64 hex characters>" - }, - "location": { "country": "US", "city": "Ashburn" }, - "num_dropped_events": 0 -} -``` - -Fields that older releases don't send are left out. `event.cluster_summary` only comes from the node running the control plane. - -## Settings - -| Name | Description | -| --- | --- | -| `DD_LOGS_URL` | Datadog Logs intake URL, for example `https://http-intake.logs.datadoghq.com/api/v2/logs`. | -| `DD_API_KEY_PARAMETER` | SSM parameter holding the Datadog API key. | -| `INSTALL_ID_SECRET_PARAMETER` | SSM parameter holding the HMAC key. Changing it changes every install and cluster ID. | -| `ORIGIN_VERIFY_SECRET` | Value of the `X-Origin-Verify` header that CloudFront adds. Requests without it get HTTP 403. | - -The Lambda entry point is `src/lambda.handler`, on Node.js 22. - -## Test - -```bash -npm test -``` diff --git a/distribution/telemetry-relay/package.json b/distribution/telemetry-relay/package.json deleted file mode 100644 index 859a3364ac4..00000000000 --- a/distribution/telemetry-relay/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "quickwit-telemetry-relay", - "private": true, - "type": "module", - "scripts": { - "test": "node --test" - } -} diff --git a/distribution/telemetry-relay/src/lambda.js b/distribution/telemetry-relay/src/lambda.js deleted file mode 100644 index 842239b153e..00000000000 --- a/distribution/telemetry-relay/src/lambda.js +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { timingSafeEqual } from "node:crypto"; - -import relay from "./relay.js"; - -// Requests without CloudFront's secret came straight to API Gateway and are rejected, so that -// nobody can bypass CloudFront or forge its location headers. -const ORIGIN_VERIFY_HEADER = "x-origin-verify"; -// Only these headers reach the relay, never the client IP address. -const FORWARDED_HEADERS = [ - "content-length", - "content-type", - "cloudfront-viewer-country", - "cloudfront-viewer-city", -]; - -// The AWS SDK ships with the Lambda runtime: importing it lazily keeps the relay testable without -// it. -async function loadSettingsFromSsm() { - const ddApiKeyParameter = process.env.DD_API_KEY_PARAMETER; - const installIdSecretParameter = process.env.INSTALL_ID_SECRET_PARAMETER; - if (!ddApiKeyParameter || !installIdSecretParameter) { - throw new Error("DD_API_KEY_PARAMETER and INSTALL_ID_SECRET_PARAMETER must be set"); - } - const { GetParametersCommand, SSMClient } = await import("@aws-sdk/client-ssm"); - const output = await new SSMClient({}).send( - new GetParametersCommand({ - Names: [ddApiKeyParameter, installIdSecretParameter], - WithDecryption: true, - }), - ); - const invalidParameters = output.InvalidParameters ?? []; - if (invalidParameters.length > 0) { - throw new Error(`missing SSM parameters: ${invalidParameters.join(", ")}`); - } - const parameters = output.Parameters ?? []; - const values = new Map(parameters.map((parameter) => [parameter.Name, parameter.Value])); - return { - DD_API_KEY: values.get(ddApiKeyParameter), - DD_LOGS_URL: process.env.DD_LOGS_URL, - INSTALL_ID_SECRET: values.get(installIdSecretParameter), - }; -} - -function toRequest(event) { - const method = event.requestContext.http.method; - const query = event.rawQueryString ? `?${event.rawQueryString}` : ""; - const headers = {}; - for (const name of FORWARDED_HEADERS) { - if (event.headers?.[name] !== undefined) { - headers[name] = event.headers[name]; - } - } - const hasBody = method !== "GET" && method !== "HEAD" && event.body !== undefined; - const body = hasBody - ? Buffer.from(event.body, event.isBase64Encoded ? "base64" : "utf8") - : undefined; - return new Request(`https://${event.requestContext.domainName}${event.rawPath}${query}`, { - method, - headers, - body, - }); -} - -function isFromCloudFront(event, originVerifySecret) { - const header = event.headers?.[ORIGIN_VERIFY_HEADER]; - if (header === undefined) { - return false; - } - const expected = Buffer.from(originVerifySecret); - const actual = Buffer.from(header); - return actual.length === expected.length && timingSafeEqual(actual, expected); -} - -export function createHandler({ loadSettings, originVerifySecret }) { - let settingsPromise = null; - - return async function handler(event) { - if (!originVerifySecret) { - console.error("ORIGIN_VERIFY_SECRET must be set"); - return { statusCode: 500, body: "relay is misconfigured\n" }; - } - if (!isFromCloudFront(event, originVerifySecret)) { - return { statusCode: 403, body: "forbidden\n" }; - } - if (settingsPromise === null) { - settingsPromise = loadSettings(); - } - let settings; - try { - settings = await settingsPromise; - } catch (error) { - settingsPromise = null; - console.error(`failed to load relay settings: ${error.message}`); - return { statusCode: 500, body: "relay is misconfigured\n" }; - } - const response = await relay.fetch(toRequest(event), settings); - return { - statusCode: response.status, - headers: Object.fromEntries(response.headers), - body: await response.text(), - }; - }; -} - -export const handler = createHandler({ - loadSettings: loadSettingsFromSsm, - originVerifySecret: process.env.ORIGIN_VERIFY_SECRET, -}); diff --git a/distribution/telemetry-relay/src/relay.js b/distribution/telemetry-relay/src/relay.js deleted file mode 100644 index c9c07bbf72d..00000000000 --- a/distribution/telemetry-relay/src/relay.js +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const MAX_BODY_NUM_BYTES = 16 * 1024; -// Quickwit queues at most `MAX_NUM_EVENTS_IN_QUEUE` (10) events per request. -const MAX_NUM_EVENTS = 10; -const MAX_NUM_NAMES = 32; -const EVENT_TYPES = new Set([ - "run_command", - "end_command", - "running", - "ui_index_page_load", - "install", -]); -const ACTIVITY_EVENT_TYPES = new Set(["end_command", "running"]); -const API_KINDS = new Set([ - "elasticsearch_search", - "elasticsearch_ingest", - "elasticsearch_other", - "native_search", - "native_ingest", - "otlp", - "jaeger", -]); -const MAX_SCHEMA_VERSION = 2; -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; -const MD5_HEX_RE = /^[0-9a-f]{32}$/; -const NAME_RE = /^[a-z0-9_-]{1,64}$/; -const VERSION_RE = /^[0-9A-Za-z.+-]{1,64}$/; -const BUCKET_RE = /^[0-9A-Za-z<+-]{1,16}$/; -const COMMIT_RE = /^([0-9a-f]{7,40}|unknown)$/; -const TARGET_RE = /^[a-z0-9_.-]{1,64}$/; -const COUNTRY_HEADER = "cloudfront-viewer-country"; -const CITY_HEADER = "cloudfront-viewer-city"; -const COUNTRY_RE = /^[A-Z]{2}$/; -const CITY_RE = /^[^\p{Cc}]{1,100}$/u; -const REQUIRED_SETTINGS = ["DD_API_KEY", "DD_LOGS_URL", "INSTALL_ID_SECRET"]; - -class InvalidPayloadError extends Error {} - -class BodyTooLargeError extends Error {} - -function check(condition, message) { - if (!condition) { - throw new InvalidPayloadError(message); - } -} - -function isObject(value) { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isNonNegativeInteger(value) { - return Number.isSafeInteger(value) && value >= 0; -} - -function isMatch(value, regex) { - return typeof value === "string" && regex.test(value); -} - -function parseNames(value, field) { - check(Array.isArray(value), `${field} must be a list`); - check(value.length <= MAX_NUM_NAMES, `${field} has more than ${MAX_NUM_NAMES} items`); - for (const name of value) { - check(isMatch(name, NAME_RE), `${field} contains an invalid name`); - } - return [...new Set(value)].sort(); -} - -function parseBucket(value, field) { - check(isMatch(value, BUCKET_RE), `invalid ${field}`); - return value; -} - -function parseApiUsage(apiUsage) { - check(isObject(apiUsage), "api_usage must be an object"); - const parsedApiUsage = {}; - for (const [apiKind, bucket] of Object.entries(apiUsage)) { - check(API_KINDS.has(apiKind), "unknown api_usage key"); - parsedApiUsage[apiKind] = parseBucket(bucket, "api_usage bucket"); - } - return parsedApiUsage; -} - -function parseClusterSummary(clusterSummary) { - check(isObject(clusterSummary), "cluster_summary must be an object"); - return { - num_indexes: parseBucket(clusterSummary.num_indexes, "cluster_summary.num_indexes"), - source_types: parseNames(clusterSummary.source_types, "cluster_summary.source_types"), - }; -} - -function parseEvent(event) { - check(isObject(event), "events must be objects"); - check(EVENT_TYPES.has(event.type), "unknown event type"); - check(isNonNegativeInteger(event.unixtime), "invalid event unixtime"); - // The client clock may be off, so the log timestamp is the intake time. - const parsedEvent = { type: event.type, client_unixtime: event.unixtime }; - if (event.type === "end_command") { - check(Number.isSafeInteger(event.return_code), "invalid end_command return_code"); - parsedEvent.return_code = event.return_code; - } - if (ACTIVITY_EVENT_TYPES.has(event.type)) { - if (event.api_usage !== undefined) { - parsedEvent.api_usage = parseApiUsage(event.api_usage); - } - if (event.cluster_summary !== undefined) { - parsedEvent.cluster_summary = parseClusterSummary(event.cluster_summary); - } - } - return parsedEvent; -} - -function parseBuild(build) { - check(isObject(build), "quickwit_info.build must be an object"); - check(isMatch(build.commit, COMMIT_RE), "invalid quickwit_info.build.commit"); - check( - build.target === "unknown" || isMatch(build.target, TARGET_RE), - "invalid quickwit_info.build.target", - ); - check(typeof build.release === "boolean", "invalid quickwit_info.build.release"); - return { commit: build.commit, target: build.target, release: build.release }; -} - -function parsePayload(payload) { - check(isObject(payload), "payload must be an object"); - const schemaVersion = payload.schema_version ?? 1; - const isKnownSchemaVersion = - Number.isSafeInteger(schemaVersion) && - schemaVersion >= 1 && - schemaVersion <= MAX_SCHEMA_VERSION; - check(isKnownSchemaVersion, "unsupported schema_version"); - const clientInfo = payload.client_info; - check(isObject(clientInfo), "missing client_info"); - check(isMatch(clientInfo.session_uuid, UUID_RE), "invalid client_info.session_uuid"); - check(isMatch(clientInfo.os, NAME_RE), "invalid client_info.os"); - check(isMatch(clientInfo.arch, NAME_RE), "invalid client_info.arch"); - check( - isMatch(clientInfo.hashed_host_username, MD5_HEX_RE), - "invalid client_info.hashed_host_username", - ); - check(typeof clientInfo.kubernetes === "boolean", "invalid client_info.kubernetes"); - const client = { - session_uuid: clientInfo.session_uuid, - os: clientInfo.os, - arch: clientInfo.arch, - hashed_host_username: clientInfo.hashed_host_username, - kubernetes: clientInfo.kubernetes, - }; - if (clientInfo.cpus !== undefined) { - client.cpus = parseBucket(clientInfo.cpus, "client_info.cpus"); - } - if (clientInfo.memory !== undefined) { - client.memory = parseBucket(clientInfo.memory, "client_info.memory"); - } - - const quickwitInfo = payload.quickwit_info; - check(isObject(quickwitInfo), "missing quickwit_info"); - check(isMatch(quickwitInfo.version, VERSION_RE), "invalid quickwit_info.version"); - const quickwit = { - version: quickwitInfo.version, - services: parseNames(quickwitInfo.services, "quickwit_info.services"), - features: parseNames(quickwitInfo.features, "quickwit_info.features"), - }; - if (quickwitInfo.build !== undefined) { - quickwit.build = parseBuild(quickwitInfo.build); - } - if (quickwitInfo.install_uuid !== undefined) { - check(isMatch(quickwitInfo.install_uuid, UUID_RE), "invalid quickwit_info.install_uuid"); - client.install_uuid = quickwitInfo.install_uuid; - } - if (quickwitInfo.hashed_cluster_id !== undefined) { - check( - isMatch(quickwitInfo.hashed_cluster_id, MD5_HEX_RE), - "invalid quickwit_info.hashed_cluster_id", - ); - quickwit.hashed_cluster_id = quickwitInfo.hashed_cluster_id; - } - - check(Array.isArray(payload.events), "events must be a list"); - check(payload.events.length <= MAX_NUM_EVENTS, `more than ${MAX_NUM_EVENTS} events`); - check(isNonNegativeInteger(payload.num_dropped_events), "invalid num_dropped_events"); - - return { - schema_version: schemaVersion, - client, - quickwit, - events: payload.events.map(parseEvent), - num_dropped_events: payload.num_dropped_events, - }; -} - -async function computeKeyedHash(secret, value) { - const encoder = new TextEncoder(); - const key = await crypto.subtle.importKey( - "raw", - encoder.encode(secret), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"], - ); - const signature = await crypto.subtle.sign("HMAC", key, encoder.encode(value)); - return Array.from(new Uint8Array(signature), (byte) => byte.toString(16).padStart(2, "0")).join( - "", - ); -} - -function parseLocation(headers) { - const location = {}; - const country = headers.get(COUNTRY_HEADER); - if (country !== null && COUNTRY_RE.test(country)) { - location.country = country; - } - const city = headers.get(CITY_HEADER); - if (city !== null && CITY_RE.test(city)) { - location.city = city; - } - return location; -} - -// The prefix keeps a cluster ID from ever equaling an install ID. -async function replaceHashes(parsedPayload, secret) { - const { hashed_host_username: hashedHostUsername, ...client } = parsedPayload.client; - client.install_id = await computeKeyedHash(secret, hashedHostUsername); - const { hashed_cluster_id: hashedClusterId, ...quickwit } = parsedPayload.quickwit; - if (hashedClusterId !== undefined) { - quickwit.cluster_id = await computeKeyedHash(secret, `cluster\n${hashedClusterId}`); - } - return { ...parsedPayload, client, quickwit }; -} - -function toDatadogLogs(keyedPayload, location) { - const hasLocation = Object.keys(location).length > 0; - return keyedPayload.events.map((event) => ({ - ddsource: "quickwit-telemetry", - service: "quickwit-telemetry", - message: event.type, - schema_version: keyedPayload.schema_version, - event, - client: keyedPayload.client, - quickwit: keyedPayload.quickwit, - ...(hasLocation ? { location } : {}), - num_dropped_events: keyedPayload.num_dropped_events, - })); -} - -async function readBody(request) { - const contentLength = Number(request.headers.get("content-length")); - if (contentLength > MAX_BODY_NUM_BYTES) { - throw new BodyTooLargeError(); - } - if (request.body === null) { - return ""; - } - const reader = request.body.getReader(); - const chunks = []; - let numBytes = 0; - while (true) { - const { done, value: chunk } = await reader.read(); - if (done) { - break; - } - numBytes += chunk.byteLength; - if (numBytes > MAX_BODY_NUM_BYTES) { - await reader.cancel(); - throw new BodyTooLargeError(); - } - chunks.push(chunk); - } - const body = new Uint8Array(numBytes); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.byteLength; - } - return new TextDecoder().decode(body); -} - -function textResponse(status, text, headers = {}) { - return new Response(`${text}\n`, { status, headers }); -} - -export default { - async fetch(request, env) { - const missingSettings = REQUIRED_SETTINGS.filter((name) => !env[name]); - if (missingSettings.length > 0) { - console.error(`telemetry relay is missing settings: ${missingSettings.join(", ")}`); - return textResponse(500, "relay is misconfigured"); - } - if (new URL(request.url).pathname !== "/") { - return textResponse(404, "not found"); - } - if (request.method !== "POST") { - return textResponse(405, "method not allowed", { Allow: "POST" }); - } - let parsedPayload; - try { - parsedPayload = parsePayload(JSON.parse(await readBody(request))); - } catch (error) { - if (error instanceof BodyTooLargeError) { - return textResponse(413, "payload too large"); - } - if (error instanceof SyntaxError || error instanceof InvalidPayloadError) { - return textResponse(400, `invalid payload: ${error.message}`); - } - throw error; - } - if (parsedPayload.events.length === 0) { - return new Response(null, { status: 202 }); - } - const keyedPayload = await replaceHashes(parsedPayload, env.INSTALL_ID_SECRET); - const logs = toDatadogLogs(keyedPayload, parseLocation(request.headers)); - const intakeResponse = await fetch(env.DD_LOGS_URL, { - method: "POST", - headers: { "Content-Type": "application/json", "DD-API-KEY": env.DD_API_KEY }, - body: JSON.stringify(logs), - }); - if (!intakeResponse.ok) { - console.error(`Datadog logs intake returned HTTP ${intakeResponse.status}`); - return textResponse(502, "failed to forward telemetry"); - } - return new Response(null, { status: 202 }); - }, -}; diff --git a/distribution/telemetry-relay/test/lambda.test.js b/distribution/telemetry-relay/test/lambda.test.js deleted file mode 100644 index 8df0ba4ec69..00000000000 --- a/distribution/telemetry-relay/test/lambda.test.js +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import assert from "node:assert/strict"; -import { afterEach, beforeEach, describe, it } from "node:test"; - -import { createHandler } from "../src/lambda.js"; - -const SETTINGS = { - DD_API_KEY: "test-api-key", - DD_LOGS_URL: "https://http-intake.logs.example.com/api/v2/logs", - INSTALL_ID_SECRET: "test-secret", -}; -const ORIGIN_VERIFY_SECRET = "test-origin-secret"; -const CLIENT_IP = "198.51.100.23"; -const PAYLOAD = { - client_info: { - session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", - os: "linux", - arch: "aarch64", - hashed_host_username: "5d41402abc4b2a76b9719d911017c592", - kubernetes: false, - }, - quickwit_info: { version: "0.9.1", services: ["searcher"], features: ["otlp"] }, - events: [{ unixtime: 1760000000, type: "running" }], - num_dropped_events: 0, -}; - -function makeEvent({ base64 = false, headers = {} } = {}) { - const body = JSON.stringify(PAYLOAD); - return { - rawPath: "/", - rawQueryString: "", - headers: { - "content-type": "application/json", - "x-forwarded-for": CLIENT_IP, - "x-origin-verify": ORIGIN_VERIFY_SECRET, - ...headers, - }, - requestContext: { - domainName: "abc123.execute-api.us-east-1.amazonaws.com", - http: { method: "POST", path: "/", sourceIp: CLIENT_IP }, - }, - body: base64 ? Buffer.from(body).toString("base64") : body, - isBase64Encoded: base64, - }; -} - -describe("lambda handler", () => { - let intakeCalls; - const originalFetch = globalThis.fetch; - const handler = createHandler({ - loadSettings: async () => SETTINGS, - originVerifySecret: ORIGIN_VERIFY_SECRET, - }); - - beforeEach(() => { - intakeCalls = []; - globalThis.fetch = async (url, init) => { - intakeCalls.push({ url, init }); - return new Response(null, { status: 202 }); - }; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - }); - - it("relays the request", async () => { - const result = await handler(makeEvent()); - - assert.equal(result.statusCode, 202); - assert.equal(intakeCalls[0].init.headers["DD-API-KEY"], SETTINGS.DD_API_KEY); - }); - - it("decodes base64 bodies", async () => { - const result = await handler(makeEvent({ base64: true })); - - assert.equal(result.statusCode, 202); - }); - - it("rejects requests that don't come from CloudFront", async () => { - const result = await handler(makeEvent({ headers: { "x-origin-verify": "guess" } })); - - assert.equal(result.statusCode, 403); - assert.equal(intakeCalls.length, 0); - }); - - it("passes the location on, but not the client IP or the origin secret", async () => { - const headers = { "cloudfront-viewer-country": "FR", "cloudfront-viewer-city": "Lyon" }; - - await handler(makeEvent({ headers })); - - const [log] = JSON.parse(intakeCalls[0].init.body); - assert.deepEqual(log.location, { country: "FR", city: "Lyon" }); - const forwarded = JSON.stringify(intakeCalls); - assert.ok(!forwarded.includes(CLIENT_IP)); - assert.ok(!forwarded.includes(ORIGIN_VERIFY_SECRET)); - }); -}); diff --git a/distribution/telemetry-relay/test/relay.test.js b/distribution/telemetry-relay/test/relay.test.js deleted file mode 100644 index 5332851965a..00000000000 --- a/distribution/telemetry-relay/test/relay.test.js +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import assert from "node:assert/strict"; -import { createHmac } from "node:crypto"; -import { afterEach, beforeEach, describe, it } from "node:test"; - -import relay from "../src/relay.js"; - -const ENV = { - DD_API_KEY: "test-api-key", - DD_LOGS_URL: "https://http-intake.logs.example.com/api/v2/logs", - INSTALL_ID_SECRET: "test-secret", -}; -const HASHED_HOST_USERNAME = "5d41402abc4b2a76b9719d911017c592"; -const HASHED_CLUSTER_ID = "9e107d9d372bb6826bd81d3542a419d6"; -const INSTALL_UUID = "8c4f2a5e-3b1d-4e7f-9a6c-2d5e8f1b3c7a"; -const CLIENT_IP = "203.0.113.7"; - -function keyedHash(value) { - return createHmac("sha256", ENV.INSTALL_ID_SECRET).update(value).digest("hex"); -} - -function makePayload() { - return { - client_info: { - session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", - os: "linux", - arch: "x86_64", - hashed_host_username: HASHED_HOST_USERNAME, - kubernetes: true, - }, - quickwit_info: { - version: "0.9.1", - services: ["searcher", "control-plane"], - features: ["s3_storage", "otlp"], - }, - events: [ - { unixtime: 1760000000, type: "run_command" }, - { unixtime: 1760000060, type: "end_command", return_code: 1 }, - ], - num_dropped_events: 0, - }; -} - -function makeV2Payload() { - const payload = makePayload(); - payload.schema_version = 2; - payload.client_info.cpus = "5-8"; - payload.client_info.memory = "8-16GiB"; - payload.quickwit_info.build = { - commit: "4f20e4f", - target: "aarch64-unknown-linux-gnu", - release: false, - }; - payload.quickwit_info.install_uuid = INSTALL_UUID; - payload.quickwit_info.hashed_cluster_id = HASHED_CLUSTER_ID; - payload.events = [ - { - unixtime: 1760043200, - type: "running", - api_usage: { native_search: "1-9", elasticsearch_ingest: "100-999" }, - cluster_summary: { num_indexes: "2-10", source_types: ["vec", "kafka"] }, - }, - ]; - return payload; -} - -function makeRequest(body, headers = {}) { - return new Request("https://telemetry.quickwit.io/", { - method: "POST", - headers: { "Content-Type": "application/json", "X-Forwarded-For": CLIENT_IP, ...headers }, - body: typeof body === "string" ? body : JSON.stringify(body), - }); -} - -describe("telemetry relay", () => { - let intakeCalls; - const originalFetch = globalThis.fetch; - - beforeEach(() => { - intakeCalls = []; - globalThis.fetch = async (url, init) => { - intakeCalls.push({ url, init }); - return new Response(null, { status: 202 }); - }; - }); - - afterEach(() => { - globalThis.fetch = originalFetch; - }); - - function forwardedLogs() { - return JSON.parse(intakeCalls[0].init.body); - } - - it("forwards one log per event", async () => { - const response = await relay.fetch(makeRequest(makePayload()), ENV); - - assert.equal(response.status, 202); - assert.equal(intakeCalls[0].url, ENV.DD_LOGS_URL); - assert.equal(intakeCalls[0].init.headers["DD-API-KEY"], ENV.DD_API_KEY); - const commonAttributes = { - ddsource: "quickwit-telemetry", - service: "quickwit-telemetry", - schema_version: 1, - client: { - session_uuid: "0d6e4079-e2c8-4bd5-a1a1-a6bce5ad1c29", - os: "linux", - arch: "x86_64", - kubernetes: true, - install_id: keyedHash(HASHED_HOST_USERNAME), - }, - quickwit: { - version: "0.9.1", - services: ["control-plane", "searcher"], - features: ["otlp", "s3_storage"], - }, - num_dropped_events: 0, - }; - assert.deepEqual(forwardedLogs(), [ - { - ...commonAttributes, - message: "run_command", - event: { type: "run_command", client_unixtime: 1760000000 }, - }, - { - ...commonAttributes, - message: "end_command", - event: { type: "end_command", client_unixtime: 1760000060, return_code: 1 }, - }, - ]); - }); - - it("forwards the schema version 2 fields", async () => { - await relay.fetch(makeRequest(makeV2Payload()), ENV); - - const [log] = forwardedLogs(); - assert.equal(log.schema_version, 2); - assert.equal(log.client.cpus, "5-8"); - assert.equal(log.client.memory, "8-16GiB"); - assert.equal(log.client.install_uuid, INSTALL_UUID); - assert.deepEqual(log.quickwit.build, { - commit: "4f20e4f", - target: "aarch64-unknown-linux-gnu", - release: false, - }); - assert.equal(log.quickwit.cluster_id, keyedHash(`cluster\n${HASHED_CLUSTER_ID}`)); - assert.deepEqual(log.event, { - type: "running", - client_unixtime: 1760043200, - api_usage: { elasticsearch_ingest: "100-999", native_search: "1-9" }, - cluster_summary: { num_indexes: "2-10", source_types: ["kafka", "vec"] }, - }); - }); - - it("never forwards the client IP, unkeyed hashes or undocumented fields", async () => { - const payload = makeV2Payload(); - payload.client_info.email = "someone@example.com"; - payload.events[0].query = "secret query"; - - await relay.fetch(makeRequest(payload), ENV); - - const forwarded = JSON.stringify(intakeCalls[0]); - for (const value of [ - CLIENT_IP, - HASHED_HOST_USERNAME, - HASHED_CLUSTER_ID, - "someone@example.com", - "secret query", - ]) { - assert.ok(!forwarded.includes(value), `${value} was forwarded`); - } - }); - - it("adds the country and city set by CloudFront", async () => { - const headers = { "CloudFront-Viewer-Country": "FR", "CloudFront-Viewer-City": "Lyon" }; - - await relay.fetch(makeRequest(makePayload(), headers), ENV); - - for (const log of forwardedLogs()) { - assert.deepEqual(log.location, { country: "FR", city: "Lyon" }); - } - }); - - const invalidPayloads = { - "malformed JSON": "{", - "an unknown event type": { ...makePayload(), events: [{ unixtime: 1, type: "search" }] }, - "a free-form feature name": (() => { - const payload = makePayload(); - payload.quickwit_info.features = ["Index my-logs on s3://bucket"]; - return payload; - })(), - "an unsupported schema version": { ...makeV2Payload(), schema_version: 3 }, - }; - - for (const [description, payload] of Object.entries(invalidPayloads)) { - it(`rejects ${description}`, async () => { - const response = await relay.fetch(makeRequest(payload), ENV); - - assert.equal(response.status, 400); - assert.equal(intakeCalls.length, 0); - }); - } - - it("rejects payloads larger than 16 KiB", async () => { - const payload = { ...makePayload(), padding: "x".repeat(17 * 1024) }; - - const response = await relay.fetch(makeRequest(payload), ENV); - - assert.equal(response.status, 413); - assert.equal(intakeCalls.length, 0); - }); -}); diff --git a/docs/telemetry.md b/docs/telemetry.md index 59e81aa7329..aeb6db1a887 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -50,7 +50,7 @@ Each request also includes: - the metastore type, file-backed or PostgreSQL, if the node runs the metastore; - where the default index root URI points: S3, Azure Blob Storage, Google Cloud Storage or the local file system; - whether the S3 storage config targets an S3-compatible store, through a custom endpoint or a flavor such as MinIO, Garage, DigitalOcean Spaces or Google Cloud Storage; - - whether the OTLP endpoints and the Jaeger endpoint are enabled; + - whether the OTLP endpoints are enabled on an indexer, and the Jaeger endpoint on a searcher; - whether ingest API v1 and v2 are enabled; - whether the searcher uses AWS Lambda for leaf searches or a local split cache; - whether document clustering is configured; @@ -61,7 +61,7 @@ Each request also includes: - A session ID, generated at random each time the node starts. - An install ID, generated at random the first time the node uses its data directory and stored there in the `telemetry_install_id` file. It is only created while telemetry is on. - An MD5 hash of the host name and the user name, used to count distinct installations from older releases, which have no install ID. The relay replaces it with a keyed hash before storage. See below. -- An MD5 hash of the cluster ID and the metastore URI, without credentials, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. +- An MD5 hash of the cluster ID and the metastore URI, without the user name, password or query parameters, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. - The number of events dropped because too many were queued. `running` and `end_command` events also include: @@ -73,7 +73,7 @@ Quickwit never sends index or field names, documents, queries, configuration val ## Where the data goes -Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog operates on AWS. Its source code is in [`distribution/telemetry-relay`](https://github.com/quickwit-oss/quickwit/tree/main/distribution/telemetry-relay). +Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog operates on AWS. - The request goes through Amazon CloudFront, which looks up the country and city of your IP address and passes them on. The IP address itself goes no further. - A small relay checks the request against the format above, rejects anything else, and keeps only the fields listed on this page. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index f64328b90a7..11c2456334a 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -5332,6 +5332,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -8150,6 +8159,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "sysinfo", "tempfile", "tokio", "tracing", @@ -10288,6 +10298,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c8bd2130a9b60bee2581bf82cfe89ee836424d1f37dcfa4ce21509611684673" dependencies = [ "libc", + "memchr", + "ntapi", "objc2-core-foundation", "objc2-io-kit", "windows", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 9427e024fa1..e478b2d6ed6 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -233,7 +233,7 @@ sqlx = { version = "0.8", features = [ ] } syn = { version = "2.0", features = ["extra-traits", "full", "parsing"] } sync_wrapper = "1" -sysinfo = { version = "0.39", default-features = false, features = ["disk"] } +sysinfo = { version = "0.39", default-features = false, features = ["disk", "system"] } tabled = { version = "0.21", features = ["ansi"] } tempfile = "3" thiserror = "2" diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs index ae3880230da..63e41a012e2 100644 --- a/quickwit/quickwit-cli/src/telemetry.rs +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -31,8 +31,6 @@ pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryI .map(|service| service.to_string()) .collect(); let build_info = BuildInfo::get(); - let mut redacted_metastore_uri = config.metastore_uri.clone(); - redacted_metastore_uri.redact(); QuickwitTelemetryInfo { build: BuildTelemetryInfo { commit: build_info.commit_short_hash.to_string(), @@ -41,7 +39,7 @@ pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryI }, hashed_cluster_id: Some(hash_cluster_identity( &config.cluster_id, - redacted_metastore_uri.as_str(), + &config.metastore_uri, )), ..QuickwitTelemetryInfo::new(services, telemetry_features(config)) } @@ -70,12 +68,11 @@ pub(crate) fn set_install_uuid( fn telemetry_features(config: &NodeConfig) -> HashSet { let mut features = HashSet::new(); - if config.indexer_config.enable_otlp_endpoint { + if config.is_service_enabled(QuickwitService::Indexer) + && config.indexer_config.enable_otlp_endpoint + { features.insert(QuickwitFeature::Otlp); } - if config.jaeger_config.enable_endpoint { - features.insert(QuickwitFeature::Jaeger); - } // The metastore URI is only relevant if the metastore is enabled. if config.is_service_enabled(QuickwitService::Metastore) { let feature = if config.metastore_uri.protocol() == Protocol::PostgreSQL { @@ -111,6 +108,9 @@ fn telemetry_features(config: &NodeConfig) -> HashSet { } } if config.is_service_enabled(QuickwitService::Searcher) { + if config.jaeger_config.enable_endpoint { + features.insert(QuickwitFeature::Jaeger); + } if config.searcher_config.lambda.is_some() { features.insert(QuickwitFeature::LambdaLeafSearch); } @@ -135,8 +135,6 @@ fn telemetry_features(config: &NodeConfig) -> HashSet { #[cfg(test)] mod tests { - use std::str::FromStr; - use quickwit_common::uri::Uri; use quickwit_config::{ DocsClusteringConfig, LambdaConfig, S3StorageConfig, StorageConfig, StorageConfigs, @@ -144,51 +142,32 @@ mod tests { use super::*; - /// The ingest features depend on environment variables, which CI may set. - fn ingest_features() -> Vec { - let mut features = Vec::new(); - if !disable_ingest_v1() { - features.push(QuickwitFeature::IngestV1); - } - if enable_ingest_v2() { - features.push(QuickwitFeature::IngestV2); - } - features - } - #[test] fn test_telemetry_features() { let mut node_config = NodeConfig::for_test(); node_config.enabled_services = HashSet::from([QuickwitService::Metastore]); - node_config.metastore_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); - node_config.default_index_root_uri = Uri::from_str("file:///qwdata/indexes").unwrap(); - // No S3 config, so that `QW_S3_ENDPOINT`, which CI may set, has no effect. - node_config.storage_configs = StorageConfigs::new(Vec::new()); - node_config.indexer_config.enable_otlp_endpoint = false; - node_config.jaeger_config.enable_endpoint = false; - node_config.searcher_config.lambda = Some(LambdaConfig::for_test()); - node_config.searcher_config.split_cache = - Some(serde_json::from_value(serde_json::json!({"max_num_bytes": "1GB"})).unwrap()); - - let mut expected_features = HashSet::from([ - QuickwitFeature::FileBackedMetastore, - QuickwitFeature::FileStorage, + node_config.indexer_config.enable_otlp_endpoint = true; + node_config.jaeger_config.enable_endpoint = true; + let features = telemetry_features(&node_config); + assert!(!features.contains(&QuickwitFeature::Otlp)); + assert!(!features.contains(&QuickwitFeature::Jaeger)); + + node_config.enabled_services = HashSet::from([ + QuickwitService::Indexer, + QuickwitService::Metastore, + QuickwitService::Searcher, ]); - expected_features.extend(ingest_features()); - assert_eq!(telemetry_features(&node_config), expected_features); - - node_config.enabled_services = - HashSet::from([QuickwitService::Metastore, QuickwitService::Searcher]); - node_config.metastore_uri = Uri::from_str("postgresql://localhost/metastore").unwrap(); - node_config.default_index_root_uri = Uri::from_str("s3://bucket/indexes").unwrap(); + node_config.metastore_uri = Uri::for_test("postgresql://localhost/metastore"); + node_config.default_index_root_uri = Uri::for_test("s3://bucket/indexes"); node_config.storage_configs = StorageConfigs::new(vec![StorageConfig::S3(S3StorageConfig { flavor: Some(StorageBackendFlavor::MinIO), endpoint: Some("http://minio:9000".to_string()), ..Default::default() })]); - node_config.indexer_config.enable_otlp_endpoint = true; - node_config.jaeger_config.enable_endpoint = true; + node_config.searcher_config.lambda = Some(LambdaConfig::for_test()); + node_config.searcher_config.split_cache = + Some(serde_json::from_value(serde_json::json!({"max_num_bytes": "1GB"})).unwrap()); node_config.grpc_config.tls_config = Some( serde_json::from_value(serde_json::json!({ "cert_path": "/certs/cert.pem", @@ -199,8 +178,8 @@ mod tests { node_config.docs_clustering_config = Some(DocsClusteringConfig { policies: Vec::new(), }); - - let mut expected_features = HashSet::from([ + let features = telemetry_features(&node_config); + for feature in [ QuickwitFeature::DocsClustering, QuickwitFeature::Jaeger, QuickwitFeature::LambdaLeafSearch, @@ -211,26 +190,8 @@ mod tests { QuickwitFeature::S3Storage, QuickwitFeature::SplitCache, QuickwitFeature::Tls, - ]); - expected_features.extend(ingest_features()); - assert_eq!(telemetry_features(&node_config), expected_features); - } - - #[test] - fn test_quickwit_telemetry_info_hashes_cluster_identity() { - let mut node_config = NodeConfig::for_test(); - node_config.metastore_uri = - Uri::from_str("postgresql://quickwit:secret@db:5432/metastore").unwrap(); - let telemetry_info = quickwit_telemetry_info(&node_config); - let hashed_cluster_id = telemetry_info.hashed_cluster_id.unwrap(); - - node_config.metastore_uri = - Uri::from_str("postgresql://quickwit:rotated@db:5432/metastore").unwrap(); - let telemetry_info = quickwit_telemetry_info(&node_config); - assert_eq!(telemetry_info.hashed_cluster_id.unwrap(), hashed_cluster_id); - - node_config.cluster_id = "other-cluster".to_string(); - let telemetry_info = quickwit_telemetry_info(&node_config); - assert_ne!(telemetry_info.hashed_cluster_id.unwrap(), hashed_cluster_id); + ] { + assert!(features.contains(&feature), "{feature:?}"); + } } } diff --git a/quickwit/quickwit-serve/src/api_usage.rs b/quickwit/quickwit-serve/src/api_usage.rs index f10f9139515..2b2d2ff2d00 100644 --- a/quickwit/quickwit-serve/src/api_usage.rs +++ b/quickwit/quickwit-serve/src/api_usage.rs @@ -67,49 +67,24 @@ mod tests { #[test] fn test_classify_rest_request() { let cases = [ - ("/api/v1/_elastic/_bulk", Some(ApiKind::ElasticsearchIngest)), ( "/api/v1/_elastic/logs/_bulk", Some(ApiKind::ElasticsearchIngest), ), - ( - "/api/v1/_elastic/_search", - Some(ApiKind::ElasticsearchSearch), - ), - ( - "/api/v1/_elastic/logs/_search", - Some(ApiKind::ElasticsearchSearch), - ), ( "/api/v1/_elastic/_msearch", Some(ApiKind::ElasticsearchSearch), ), - ( - "/api/v1/_elastic/_search/scroll", - Some(ApiKind::ElasticsearchSearch), - ), - ( - "/api/v1/_elastic/logs/_count", - Some(ApiKind::ElasticsearchSearch), - ), ( "/api/v1/_elastic/_field_caps", Some(ApiKind::ElasticsearchOther), ), - ("/api/v1/_elastic", Some(ApiKind::ElasticsearchOther)), ("/api/v1/logs/search", Some(ApiKind::NativeSearch)), ("/api/v1/logs/ingest", Some(ApiKind::NativeIngest)), ("/api/v1/otlp/v1/logs", Some(ApiKind::Otlp)), - ("/api/v1/otlp/v1/traces", Some(ApiKind::Otlp)), - ("/api/v1/my-logs/otlp/v1/logs", Some(ApiKind::Otlp)), ("/api/v1/otlp/search", Some(ApiKind::NativeSearch)), - ( - "/api/v1/otel-traces-v0_9/jaeger/api/services", - Some(ApiKind::Jaeger), - ), + ("/api/v1/traces/jaeger/api/services", Some(ApiKind::Jaeger)), ("/api/v1/indexes", None), - ("/api/v1/indexes/logs/sources", None), - ("/health/livez", None), ("/ui/search", None), ]; for (path, expected_api_kind) in cases { diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index b94e21cf0a4..3a579304ae6 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -16,6 +16,7 @@ hostname = { workspace = true } md5 = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } +sysinfo = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } username = { workspace = true } diff --git a/quickwit/quickwit-telemetry/src/install.rs b/quickwit/quickwit-telemetry/src/install.rs index 514d9beb12a..7a0f23d4bb1 100644 --- a/quickwit/quickwit-telemetry/src/install.rs +++ b/quickwit/quickwit-telemetry/src/install.rs @@ -19,32 +19,22 @@ use uuid::Uuid; const INSTALL_ID_FILE_NAME: &str = "telemetry_install_id"; -/// Returns the install ID stored in `data_dir`, creating it if needed, and whether it was just -/// created. An invalid ID file is an error rather than being replaced. +/// Returns the install ID stored in `data_dir`, creating it if needed or invalid, and whether it +/// was just created. pub fn load_or_create_install_id(data_dir: &Path) -> io::Result<(Uuid, bool)> { let install_id_path = data_dir.join(INSTALL_ID_FILE_NAME); match std::fs::read_to_string(&install_id_path) { Ok(install_id_str) => { - let install_id = Uuid::parse_str(install_id_str.trim()).map_err(|error| { - io::Error::new( - io::ErrorKind::InvalidData, - format!( - "invalid install ID in `{}`: {error}", - install_id_path.display() - ), - ) - })?; - return Ok((install_id, false)); + if let Ok(install_id) = Uuid::parse_str(install_id_str.trim()) { + return Ok((install_id, false)); + } } Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), } let install_id = Uuid::new_v4(); std::fs::create_dir_all(data_dir)?; - // Write to a temporary file, then rename it, so that a crash never leaves a truncated ID. - let tmp_install_id_path = data_dir.join(format!("{INSTALL_ID_FILE_NAME}.tmp")); - std::fs::write(&tmp_install_id_path, install_id.to_string())?; - std::fs::rename(&tmp_install_id_path, &install_id_path)?; + std::fs::write(&install_id_path, install_id.to_string())?; Ok((install_id, true)) } @@ -64,13 +54,4 @@ mod tests { assert!(!created); assert_eq!(reloaded_install_id, install_id); } - - #[test] - fn test_load_or_create_install_id_rejects_invalid_file() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::write(temp_dir.path().join(INSTALL_ID_FILE_NAME), "not-a-uuid").unwrap(); - - let error = load_or_create_install_id(temp_dir.path()).unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - } } diff --git a/quickwit/quickwit-telemetry/src/machine.rs b/quickwit/quickwit-telemetry/src/machine.rs index 7165345dc6a..37e926ba320 100644 --- a/quickwit/quickwit-telemetry/src/machine.rs +++ b/quickwit/quickwit-telemetry/src/machine.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use sysinfo::{MemoryRefreshKind, RefreshKind, System}; + const GIB: u64 = 1 << 30; pub(crate) fn cpu_count_bucket(num_cpus: usize) -> &'static str { @@ -40,47 +42,14 @@ pub(crate) fn memory_bucket(num_bytes: u64) -> &'static str { } } -#[cfg(target_os = "linux")] -pub(crate) fn available_memory_num_bytes() -> Option { - let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?; - let physical_memory_num_bytes = parse_mem_total_num_bytes(&meminfo)?; - let Some(cgroup_limit_num_bytes) = cgroup_memory_limit_num_bytes() else { - return Some(physical_memory_num_bytes); - }; - Some(physical_memory_num_bytes.min(cgroup_limit_num_bytes)) -} - -#[cfg(not(target_os = "linux"))] -pub(crate) fn available_memory_num_bytes() -> Option { - None -} - -#[cfg(any(target_os = "linux", test))] -fn parse_mem_total_num_bytes(meminfo: &str) -> Option { - let mem_total_kib: u64 = meminfo - .lines() - .find_map(|line| line.strip_prefix("MemTotal:"))? - .trim() - .strip_suffix("kB")? - .trim() - .parse() - .ok()?; - Some(mem_total_kib * 1024) -} - -/// Without a limit, cgroup v2 holds `max`, which doesn't parse, and cgroup v1 a huge number, which -/// the caller caps with the physical memory. -#[cfg(target_os = "linux")] -fn cgroup_memory_limit_num_bytes() -> Option { - for path in [ - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - ] { - if let Ok(limit) = std::fs::read_to_string(path) { - return limit.trim().parse().ok(); - } +pub(crate) fn available_memory_num_bytes() -> u64 { + let system = System::new_with_specifics( + RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), + ); + match system.cgroup_limits() { + Some(cgroup_limits) => cgroup_limits.total_memory, + None => system.total_memory(), } - None } #[cfg(test)] @@ -95,11 +64,4 @@ mod tests { assert_eq!(memory_bucket(31 * GIB), "16-32GiB"); assert_eq!(memory_bucket(512 * GIB), "128GiB+"); } - - #[test] - fn test_parse_mem_total_num_bytes() { - let meminfo = "MemTotal: 16303516 kB\nMemFree: 1102768 kB\n"; - assert_eq!(parse_mem_total_num_bytes(meminfo), Some(16_303_516 * 1024)); - assert_eq!(parse_mem_total_num_bytes("MemFree: 1 kB\n"), None); - } } diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index d406285c9a0..84befb3c84e 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -16,6 +16,7 @@ use std::collections::{BTreeMap, HashSet}; use std::env; use std::time::UNIX_EPOCH; +use quickwit_common::uri::Uri; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -102,8 +103,7 @@ pub struct ClientInfo { hashed_host_username: String, kubernetes: bool, cpus: String, - #[serde(skip_serializing_if = "Option::is_none")] - memory: Option, + memory: String, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -182,9 +182,21 @@ pub enum QuickwitFeature { Tls, } -/// Callers must strip the credentials from the metastore URI first. -pub fn hash_cluster_identity(cluster_id: &str, redacted_metastore_uri: &str) -> String { - let digest = md5::compute(format!("{cluster_id}\n{redacted_metastore_uri}")); +/// Leaves out the user, the password and the query parameters of the metastore URI, which may +/// differ between the nodes of a cluster. +pub fn hash_cluster_identity(cluster_id: &str, metastore_uri: &Uri) -> String { + let mut location = metastore_uri.as_str(); + if let Some((_, after_scheme)) = location.split_once("://") { + location = after_scheme; + } + if let Some((before_query, _)) = location.split_once('?') { + location = before_query; + } + let authority_len = location.find('/').unwrap_or(location.len()); + if let Some(at_index) = location[..authority_len].rfind('@') { + location = &location[at_index + 1..]; + } + let digest = md5::compute(format!("{cluster_id}\n{location}")); format!("{digest:x}") } @@ -207,18 +219,38 @@ impl Default for ClientInfo { hashed_host_username: hashed_host_username(), kubernetes: std::env::var_os("KUBERNETES_SERVICE_HOST").is_some(), cpus: cpu_count_bucket(quickwit_common::num_cpus()).to_string(), - memory: available_memory_num_bytes() - .map(|num_bytes| memory_bucket(num_bytes).to_string()), + memory: memory_bucket(available_memory_num_bytes()).to_string(), } } } #[cfg(test)] mod tests { + use std::str::FromStr; + use serde_json; use super::*; + #[test] + fn test_hash_cluster_identity_leaves_out_credentials() { + let hashed_cluster_id = |cluster_id, metastore_uri| { + hash_cluster_identity(cluster_id, &Uri::from_str(metastore_uri).unwrap()) + }; + let expected = hashed_cluster_id("my-cluster", "postgresql://db:5432/metastore"); + assert_eq!( + hashed_cluster_id( + "my-cluster", + "postgresql://node-1:secret@db:5432/metastore?sslmode=require" + ), + expected + ); + assert_ne!( + hashed_cluster_id("other-cluster", "postgresql://db:5432/metastore"), + expected + ); + } + #[test] fn test_serialize_features_as_expected() { // These names are part of the telemetry payload documented in `docs/telemetry.md`. diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 82f83218e4e..82df3682e50 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -106,11 +106,12 @@ impl EventsState { /// Adds an event. /// If the queue is already saturated, (ie. it has reached the len `MAX_NUM_EVENTS_IN_QUEUE`) // Returns true iff it was the first event in the queue. - fn push_event(&mut self, event: TelemetryEvent) -> bool { + fn push_event(&mut self, mut event: TelemetryEvent) -> bool { if self.events.len() >= MAX_NUM_EVENTS_IN_QUEUE { self.num_dropped_events += 1; return false; } + event.add_usage(); let events_was_empty = self.events.is_empty(); self.events.push(EventWithTimestamp::from(event)); events_was_empty diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs index d37e5dcf28d..ac118ac07e3 100644 --- a/quickwit/quickwit-telemetry/src/usage.rs +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -98,29 +98,41 @@ pub fn set_cluster_summary(num_indexes: usize, source_types: BTreeSet) { } impl TelemetryEvent { - /// Takes the API request counts since the previous report. pub fn running() -> Self { TelemetryEvent::Running { - api_usage: API_REQUEST_COUNTS.take(), - cluster_summary: cluster_summary(), + api_usage: None, + cluster_summary: None, } } - /// Takes the API request counts since the previous report. pub fn end_command(return_code: i32) -> Self { TelemetryEvent::EndCommand { return_code, - api_usage: API_REQUEST_COUNTS.take(), - cluster_summary: cluster_summary(), + api_usage: None, + cluster_summary: None, } } -} -fn cluster_summary() -> Option { - CLUSTER_SUMMARY - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone() + /// Adds the API request counts since the previous report and the cluster summary. Only called + /// once the event is queued, so that a dropped event never loses the counts. + pub(crate) fn add_usage(&mut self) { + if let TelemetryEvent::Running { + api_usage, + cluster_summary, + } + | TelemetryEvent::EndCommand { + api_usage, + cluster_summary, + .. + } = self + { + *api_usage = API_REQUEST_COUNTS.take(); + *cluster_summary = CLUSTER_SUMMARY + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + } + } } fn request_count_bucket(num_requests: u64) -> &'static str { From 7bc5eef637c31ac25e2b69381c32c22e3045c437 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 29 Oct 2026 17:22:08 +0000 Subject: [PATCH 05/13] Reject invalid telemetry install IDs and drop URI fragments from the cluster hash --- docs/telemetry.md | 2 +- quickwit/quickwit-telemetry/src/install.rs | 41 ++++++++++++++++++---- quickwit/quickwit-telemetry/src/payload.rs | 12 ++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index aeb6db1a887..9583363f3a9 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -61,7 +61,7 @@ Each request also includes: - A session ID, generated at random each time the node starts. - An install ID, generated at random the first time the node uses its data directory and stored there in the `telemetry_install_id` file. It is only created while telemetry is on. - An MD5 hash of the host name and the user name, used to count distinct installations from older releases, which have no install ID. The relay replaces it with a keyed hash before storage. See below. -- An MD5 hash of the cluster ID and the metastore URI, without the user name, password or query parameters, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. +- An MD5 hash of the cluster ID and the metastore URI, without the user name, password, query parameters or fragment, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. - The number of events dropped because too many were queued. `running` and `end_command` events also include: diff --git a/quickwit/quickwit-telemetry/src/install.rs b/quickwit/quickwit-telemetry/src/install.rs index 7a0f23d4bb1..8bf7c867a67 100644 --- a/quickwit/quickwit-telemetry/src/install.rs +++ b/quickwit/quickwit-telemetry/src/install.rs @@ -12,29 +12,42 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::io; +use std::fs::File; +use std::io::{self, Write}; use std::path::Path; use uuid::Uuid; const INSTALL_ID_FILE_NAME: &str = "telemetry_install_id"; +const INSTALL_ID_TEMP_FILE_NAME: &str = "telemetry_install_id.tmp"; -/// Returns the install ID stored in `data_dir`, creating it if needed or invalid, and whether it -/// was just created. +/// Returns the install ID stored in `data_dir`, creating it if needed, and whether it was just +/// created. Returns an error if the stored install ID is invalid. pub fn load_or_create_install_id(data_dir: &Path) -> io::Result<(Uuid, bool)> { let install_id_path = data_dir.join(INSTALL_ID_FILE_NAME); match std::fs::read_to_string(&install_id_path) { Ok(install_id_str) => { - if let Ok(install_id) = Uuid::parse_str(install_id_str.trim()) { - return Ok((install_id, false)); - } + let install_id = Uuid::parse_str(install_id_str.trim()).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "invalid install ID in `{}`: {error}", + install_id_path.display() + ), + ) + })?; + return Ok((install_id, false)); } Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), } let install_id = Uuid::new_v4(); std::fs::create_dir_all(data_dir)?; - std::fs::write(&install_id_path, install_id.to_string())?; + let temp_install_id_path = data_dir.join(INSTALL_ID_TEMP_FILE_NAME); + let mut temp_install_id_file = File::create(&temp_install_id_path)?; + temp_install_id_file.write_all(install_id.to_string().as_bytes())?; + temp_install_id_file.sync_all()?; + std::fs::rename(&temp_install_id_path, &install_id_path)?; Ok((install_id, true)) } @@ -54,4 +67,18 @@ mod tests { assert!(!created); assert_eq!(reloaded_install_id, install_id); } + + #[test] + fn test_load_or_create_install_id_rejects_invalid_install_id() { + let temp_dir = tempfile::tempdir().unwrap(); + let install_id_path = temp_dir.path().join(INSTALL_ID_FILE_NAME); + std::fs::write(&install_id_path, "not-a-uuid").unwrap(); + + let error = load_or_create_install_id(temp_dir.path()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!( + std::fs::read_to_string(&install_id_path).unwrap(), + "not-a-uuid" + ); + } } diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index 84befb3c84e..77063360120 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -182,15 +182,15 @@ pub enum QuickwitFeature { Tls, } -/// Leaves out the user, the password and the query parameters of the metastore URI, which may -/// differ between the nodes of a cluster. +/// Leaves out the user, the password, the query parameters and the fragment of the metastore URI, +/// which may differ between the nodes of a cluster. pub fn hash_cluster_identity(cluster_id: &str, metastore_uri: &Uri) -> String { let mut location = metastore_uri.as_str(); if let Some((_, after_scheme)) = location.split_once("://") { location = after_scheme; } - if let Some((before_query, _)) = location.split_once('?') { - location = before_query; + if let Some(query_or_fragment_start) = location.find(['?', '#']) { + location = &location[..query_or_fragment_start]; } let authority_len = location.find('/').unwrap_or(location.len()); if let Some(at_index) = location[..authority_len].rfind('@') { @@ -249,6 +249,10 @@ mod tests { hashed_cluster_id("other-cluster", "postgresql://db:5432/metastore"), expected ); + assert_eq!( + hashed_cluster_id("my-cluster", "s3://bucket/metastore#polling_interval=30s"), + hashed_cluster_id("my-cluster", "s3://bucket/metastore") + ); } #[test] From fddaa33931972580a29cbb28783b659069e1b430 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 29 Oct 2026 17:39:00 +0000 Subject: [PATCH 06/13] Stop pinning encoding_rs in quickwit-telemetry --- quickwit/Cargo.lock | 16 +++++++++++++--- quickwit/quickwit-telemetry/Cargo.toml | 8 -------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index b15319c9462..59ab56d8bcb 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -3102,11 +3102,16 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -5452,6 +5457,12 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "murmur3" version = "0.5.2" @@ -8443,7 +8454,6 @@ name = "quickwit-telemetry" version = "0.9.3" dependencies = [ "async-trait", - "encoding_rs", "hostname", "md5", "quickwit-common", diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index 3a579304ae6..ca8270977f4 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -22,16 +22,8 @@ tracing = { workspace = true } username = { workspace = true } uuid = { workspace = true } -# This is actually not used directly the goal is to fix the version -# used by reqwest. 0.8.30 has an unclear license. -encoding_rs = { workspace = true } - quickwit-common = { workspace = true } [dev-dependencies] serde_json = { workspace = true } tempfile = { workspace = true } - -[package.metadata.cargo-machete] -# see above -ignored = ["encoding_rs"] From 91b4e6f894b33521e6573696743c6ab6bde3ebd2 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 29 Oct 2026 18:06:40 +0000 Subject: [PATCH 07/13] Keep telemetry summaries when the queue is full and hash the metastore protocol - Events other than UI page loads replace the latest UI page load when the queue is full, so the shutdown summary and its API counts are not lost. - Report the metastore type on metastore read replica nodes. - Keep the canonical metastore protocol in the cluster hash, so URIs that differ only by protocol no longer hash the same. - Document that the last request at shutdown skips the one-minute cooldown. --- docs/telemetry.md | 4 +-- quickwit/quickwit-cli/src/telemetry.rs | 19 +++++++++++-- quickwit/quickwit-telemetry/src/lib.rs | 3 +- quickwit/quickwit-telemetry/src/payload.rs | 11 +++++++- quickwit/quickwit-telemetry/src/sender.rs | 32 ++++++++++++++++++++-- 5 files changed, 60 insertions(+), 9 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index 9583363f3a9..ef1392b5ffb 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -31,7 +31,7 @@ Only `quickwit run` sends telemetry. Other CLI commands never do. ## What is sent -A node sends at most one request per minute, with up to 10 events per request. +A node sends at most one request per minute, plus one last request with the pending events when it shuts down, with up to 10 events per request. Events: @@ -47,7 +47,7 @@ Each request also includes: - The Quickwit version, and how the binary was built: the commit, the target platform, and whether it was built from a release tag. - The services the node runs, such as `indexer`, `searcher` or `metastore`. - The features in use: - - the metastore type, file-backed or PostgreSQL, if the node runs the metastore; + - the metastore type, file-backed or PostgreSQL, if the node runs the metastore or a metastore read replica; - where the default index root URI points: S3, Azure Blob Storage, Google Cloud Storage or the local file system; - whether the S3 storage config targets an S3-compatible store, through a custom endpoint or a flavor such as MinIO, Garage, DigitalOcean Spaces or Google Cloud Storage; - whether the OTLP endpoints are enabled on an indexer, and the Jaeger endpoint on a searcher; diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs index 63e41a012e2..31c3dd44675 100644 --- a/quickwit/quickwit-cli/src/telemetry.rs +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -73,9 +73,16 @@ fn telemetry_features(config: &NodeConfig) -> HashSet { { features.insert(QuickwitFeature::Otlp); } - // The metastore URI is only relevant if the metastore is enabled. - if config.is_service_enabled(QuickwitService::Metastore) { - let feature = if config.metastore_uri.protocol() == Protocol::PostgreSQL { + // The metastore URI is only relevant on the nodes that serve the metastore. + let served_metastore_uri_opt = if config.is_service_enabled(QuickwitService::Metastore) { + Some(&config.metastore_uri) + } else if config.is_service_enabled(QuickwitService::MetastoreReadReplica) { + config.metastore_read_replica_uri.as_ref() + } else { + None + }; + if let Some(served_metastore_uri) = served_metastore_uri_opt { + let feature = if served_metastore_uri.protocol() == Protocol::PostgreSQL { QuickwitFeature::PostgresqMetastore } else { QuickwitFeature::FileBackedMetastore @@ -152,6 +159,12 @@ mod tests { assert!(!features.contains(&QuickwitFeature::Otlp)); assert!(!features.contains(&QuickwitFeature::Jaeger)); + node_config.enabled_services = HashSet::from([QuickwitService::MetastoreReadReplica]); + node_config.metastore_read_replica_uri = + Some(Uri::for_test("postgresql://replica/metastore")); + let features = telemetry_features(&node_config); + assert!(features.contains(&QuickwitFeature::PostgresqMetastore)); + node_config.enabled_services = HashSet::from([ QuickwitService::Indexer, QuickwitService::Metastore, diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 76892d80e41..9b51564071b 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -52,7 +52,8 @@ pub fn start_telemetry_loop(quickwit_info: QuickwitTelemetryInfo) -> Option String { if let Some(at_index) = location[..authority_len].rfind('@') { location = &location[at_index + 1..]; } - let digest = md5::compute(format!("{cluster_id}\n{location}")); + let protocol = metastore_uri.protocol(); + let digest = md5::compute(format!("{cluster_id}\n{protocol}://{location}")); format!("{digest:x}") } @@ -249,10 +250,18 @@ mod tests { hashed_cluster_id("other-cluster", "postgresql://db:5432/metastore"), expected ); + assert_eq!( + hashed_cluster_id("my-cluster", "postgres://db:5432/metastore"), + expected + ); assert_eq!( hashed_cluster_id("my-cluster", "s3://bucket/metastore#polling_interval=30s"), hashed_cluster_id("my-cluster", "s3://bucket/metastore") ); + assert_ne!( + hashed_cluster_id("my-cluster", "gs://bucket/metastore"), + hashed_cluster_id("my-cluster", "s3://bucket/metastore") + ); } #[test] diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 82df3682e50..95f2e24d691 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -30,7 +30,7 @@ use crate::payload::{ }; use crate::sink::{HttpClient, Sink}; -/// At most 1 Request per minutes. +/// At most 1 request per minute, except for the last request on termination. const TELEMETRY_PUSH_COOLDOWN: Duration = Duration::from_secs(60); /// Interval at which to send telemetry `Running` event. @@ -109,7 +109,19 @@ impl EventsState { fn push_event(&mut self, mut event: TelemetryEvent) -> bool { if self.events.len() >= MAX_NUM_EVENTS_IN_QUEUE { self.num_dropped_events += 1; - return false; + // UI page loads are the only events that can fill the queue. Any other event, such as + // a summary carrying the API counts, replaces the latest one instead of being dropped. + if matches!(event, TelemetryEvent::UiIndexPageLoad) { + return false; + } + let Some(ui_page_load_idx) = self + .events + .iter() + .rposition(|queued| matches!(queued.event, TelemetryEvent::UiIndexPageLoad)) + else { + return false; + }; + self.events.remove(ui_page_load_idx); } event.add_usage(); let events_was_empty = self.events.is_empty(); @@ -353,6 +365,22 @@ mod tests { ); } + #[test] + fn test_push_event_replaces_ui_page_load_when_queue_is_full() { + let mut events_state = EventsState::default(); + for _ in 0..=MAX_NUM_EVENTS_IN_QUEUE { + events_state.push_event(TelemetryEvent::UiIndexPageLoad); + } + events_state.push_event(TelemetryEvent::end_command(0)); + + assert_eq!(events_state.events.len(), MAX_NUM_EVENTS_IN_QUEUE); + assert_eq!(events_state.num_dropped_events, 2); + assert!(matches!( + events_state.events.last().unwrap().event, + TelemetryEvent::EndCommand { .. } + )); + } + #[tokio::test] async fn test_telemetry_no_wait_for_first_event() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); From 8b0567b632376a1b0b15fd9137d6444641944612 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 29 Oct 2026 20:14:07 +0000 Subject: [PATCH 08/13] Space telemetry requests a cooldown apart and honor whitespace opt-outs - Wait for the cooldown after each request instead of ticking a periodic interval. The interval accumulated missed ticks while the loop waited for events, so after an idle period every new event was sent in its own request. - Treat a whitespace-only QW_DISABLE_TELEMETRY as a non-empty value that disables telemetry, as documented. --- quickwit/quickwit-telemetry/Cargo.toml | 1 + quickwit/quickwit-telemetry/src/lib.rs | 5 +-- quickwit/quickwit-telemetry/src/sender.rs | 45 ++++++++++++++++++----- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index ca8270977f4..1782702cc10 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -27,3 +27,4 @@ quickwit-common = { workspace = true } [dev-dependencies] serde_json = { workspace = true } tempfile = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 9b51564071b..c1fbc4672d7 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -93,7 +93,7 @@ fn is_telemetry_disabled_by_env_value(env_value_opt: Option<&OsStr>) -> bool { let Some(env_value_str) = env_value.to_str() else { return true; }; - if env_value_str.trim().is_empty() { + if env_value_str.is_empty() { return false; } quickwit_common::parse_bool_lenient(env_value_str) != Some(false) @@ -107,13 +107,12 @@ mod tests { fn test_is_telemetry_disabled_by_env_value() { assert!(!is_telemetry_disabled_by_env_value(None)); assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new("")))); - assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new(" ")))); for enabled_value in ["false", "FALSE", "no", "0"] { assert!(!is_telemetry_disabled_by_env_value(Some(OsStr::new( enabled_value )))); } - for disabled_value in ["1", "true", "yes", "on", "disabled", "anything"] { + for disabled_value in [" ", "1", "true", "yes", "on", "disabled", "anything"] { assert!(is_telemetry_disabled_by_env_value(Some(OsStr::new( disabled_value )))); diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 95f2e24d691..e1d935f5833 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -17,10 +17,11 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +#[cfg(test)] +use tokio::sync::Mutex; use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::{Mutex, RwLock, oneshot}; +use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinHandle; -use tokio::time::Interval; use tracing::info; use crate::is_telemetry_disabled; @@ -54,15 +55,23 @@ impl ClockButton { } enum Clock { - Periodical(Mutex), + /// The first tick is immediate. Every later tick waits for the cooldown from the moment it is + /// awaited. The telemetry loop awaits a tick right after each request, so requests are at + /// least a cooldown apart, however long the loop waited for events in between. + Cooldown { + cooldown: Duration, + is_first_tick: AtomicBool, + }, #[cfg(test)] Manual(Mutex>), } impl Clock { - pub fn periodical(period: Duration) -> Clock { - let interval = tokio::time::interval(period); - Clock::Periodical(Mutex::new(interval)) + pub fn cooldown(cooldown: Duration) -> Clock { + Clock::Cooldown { + cooldown, + is_first_tick: AtomicBool::new(true), + } } #[cfg(test)] @@ -75,8 +84,13 @@ impl Clock { async fn tick(&self) { match self { - Clock::Periodical(interval) => { - interval.lock().await.tick().await; + Clock::Cooldown { + cooldown, + is_first_tick, + } => { + if !is_first_tick.swap(false, Ordering::Relaxed) { + tokio::time::sleep(*cooldown).await; + } } #[cfg(test)] Clock::Manual(channel) => { @@ -242,7 +256,7 @@ impl TelemetrySender { TelemetrySender::new( quickwit_info, http_client, - Clock::periodical(TELEMETRY_PUSH_COOLDOWN), + Clock::cooldown(TELEMETRY_PUSH_COOLDOWN), ) } @@ -381,6 +395,19 @@ mod tests { )); } + #[tokio::test(start_paused = true)] + async fn test_cooldown_clock_waits_for_cooldown_after_idle_time() { + let clock = Clock::cooldown(TELEMETRY_PUSH_COOLDOWN); + let first_tick_start = tokio::time::Instant::now(); + clock.tick().await; + assert_eq!(first_tick_start.elapsed(), Duration::ZERO); + + tokio::time::sleep(10 * TELEMETRY_PUSH_COOLDOWN).await; + let tick_start = tokio::time::Instant::now(); + clock.tick().await; + assert!(tick_start.elapsed() >= TELEMETRY_PUSH_COOLDOWN); + } + #[tokio::test] async fn test_telemetry_no_wait_for_first_event() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); From 03c68dc9434cd0a21ee0b8478a3ebc3c287bc4dd Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Thu, 29 Oct 2026 20:30:35 +0000 Subject: [PATCH 09/13] Move the telemetry clock to its own module and fix the opt-out example - Move the cooldown clock out of sender.rs, which had grown past 500 lines. - Use `quickwit --help` in the QW_DISABLE_TELEMETRY example, since the CLI has no help subcommand. - State that a cluster has a single control plane where the cluster summary relies on it. --- docs/reference/cli.md | 2 +- quickwit/quickwit-telemetry/src/clock.rs | 97 +++++++++++++++++++++++ quickwit/quickwit-telemetry/src/lib.rs | 1 + quickwit/quickwit-telemetry/src/sender.rs | 72 +---------------- quickwit/quickwit-telemetry/src/usage.rs | 3 +- 5 files changed, 102 insertions(+), 73 deletions(-) create mode 100644 quickwit/quickwit-telemetry/src/clock.rs diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7cfdf0b81a7..82ddd236925 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -861,7 +861,7 @@ Disables [telemetry](../telemetry.md) when set to any non-empty value other than *Example* -`QW_DISABLE_TELEMETRY=1 quickwit help` +`QW_DISABLE_TELEMETRY=1 quickwit --help` ### QW_POSTGRES_SKIP_MIGRATIONS diff --git a/quickwit/quickwit-telemetry/src/clock.rs b/quickwit/quickwit-telemetry/src/clock.rs new file mode 100644 index 00000000000..0d776e6a16a --- /dev/null +++ b/quickwit/quickwit-telemetry/src/clock.rs @@ -0,0 +1,97 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +#[cfg(test)] +use tokio::sync::Mutex; +#[cfg(test)] +use tokio::sync::mpsc::{Receiver, Sender}; + +#[cfg(test)] +pub(crate) struct ClockButton(Sender<()>); + +#[cfg(test)] +impl ClockButton { + pub(crate) async fn tick(&self) { + let _ = self.0.send(()).await; + } +} + +pub(crate) enum Clock { + /// The first tick is immediate. Every later tick waits for the cooldown from the moment it is + /// awaited. The telemetry loop awaits a tick right after each request, so requests are at + /// least a cooldown apart, however long the loop waited for events in between. + Cooldown { + cooldown: Duration, + is_first_tick: AtomicBool, + }, + #[cfg(test)] + Manual(Mutex>), +} + +impl Clock { + pub(crate) fn cooldown(cooldown: Duration) -> Clock { + Clock::Cooldown { + cooldown, + is_first_tick: AtomicBool::new(true), + } + } + + #[cfg(test)] + pub(crate) async fn manual() -> (ClockButton, Clock) { + let (tx, rx) = tokio::sync::mpsc::channel(1); + let _ = tx.send(()).await; + let button = ClockButton(tx); + (button, Clock::Manual(Mutex::new(rx))) + } + + pub(crate) async fn tick(&self) { + match self { + Clock::Cooldown { + cooldown, + is_first_tick, + } => { + if !is_first_tick.swap(false, Ordering::Relaxed) { + tokio::time::sleep(*cooldown).await; + } + } + #[cfg(test)] + Clock::Manual(channel) => { + channel.lock().await.recv().await; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const COOLDOWN: Duration = Duration::from_secs(60); + + #[tokio::test(start_paused = true)] + async fn test_cooldown_clock_waits_for_cooldown_after_idle_time() { + let clock = Clock::cooldown(COOLDOWN); + let first_tick_start = tokio::time::Instant::now(); + clock.tick().await; + assert_eq!(first_tick_start.elapsed(), Duration::ZERO); + + tokio::time::sleep(10 * COOLDOWN).await; + let tick_start = tokio::time::Instant::now(); + clock.tick().await; + assert!(tick_start.elapsed() >= COOLDOWN); + } +} diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index c1fbc4672d7..aa99757bda6 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -15,6 +15,7 @@ #![allow(clippy::bool_assert_comparison)] #![deny(clippy::disallowed_methods)] +mod clock; mod install; mod machine; pub mod payload; diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index e1d935f5833..cff17a81743 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -17,13 +17,12 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -#[cfg(test)] -use tokio::sync::Mutex; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinHandle; use tracing::info; +use crate::clock::Clock; use crate::is_telemetry_disabled; use crate::payload::{ ClientInfo, EventWithTimestamp, QuickwitTelemetryInfo, TELEMETRY_SCHEMA_VERSION, @@ -44,62 +43,6 @@ const LAST_REQUEST_TIMEOUT: Duration = Duration::from_secs(1); const MAX_NUM_EVENTS_IN_QUEUE: usize = 10; -#[cfg(test)] -struct ClockButton(Sender<()>); - -#[cfg(test)] -impl ClockButton { - async fn tick(&self) { - let _ = self.0.send(()).await; - } -} - -enum Clock { - /// The first tick is immediate. Every later tick waits for the cooldown from the moment it is - /// awaited. The telemetry loop awaits a tick right after each request, so requests are at - /// least a cooldown apart, however long the loop waited for events in between. - Cooldown { - cooldown: Duration, - is_first_tick: AtomicBool, - }, - #[cfg(test)] - Manual(Mutex>), -} - -impl Clock { - pub fn cooldown(cooldown: Duration) -> Clock { - Clock::Cooldown { - cooldown, - is_first_tick: AtomicBool::new(true), - } - } - - #[cfg(test)] - pub async fn manual() -> (ClockButton, Clock) { - let (tx, rx) = tokio::sync::mpsc::channel(1); - let _ = tx.send(()).await; - let button = ClockButton(tx); - (button, Clock::Manual(Mutex::new(rx))) - } - - async fn tick(&self) { - match self { - Clock::Cooldown { - cooldown, - is_first_tick, - } => { - if !is_first_tick.swap(false, Ordering::Relaxed) { - tokio::time::sleep(*cooldown).await; - } - } - #[cfg(test)] - Clock::Manual(channel) => { - channel.lock().await.recv().await; - } - } - } -} - #[derive(Default)] struct EventsState { events: Vec, @@ -395,19 +338,6 @@ mod tests { )); } - #[tokio::test(start_paused = true)] - async fn test_cooldown_clock_waits_for_cooldown_after_idle_time() { - let clock = Clock::cooldown(TELEMETRY_PUSH_COOLDOWN); - let first_tick_start = tokio::time::Instant::now(); - clock.tick().await; - assert_eq!(first_tick_start.elapsed(), Duration::ZERO); - - tokio::time::sleep(10 * TELEMETRY_PUSH_COOLDOWN).await; - let tick_start = tokio::time::Instant::now(); - clock.tick().await; - assert!(tick_start.elapsed() >= TELEMETRY_PUSH_COOLDOWN); - } - #[tokio::test] async fn test_telemetry_no_wait_for_first_event() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs index ac118ac07e3..dab623903de 100644 --- a/quickwit/quickwit-telemetry/src/usage.rs +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -86,7 +86,8 @@ pub fn record_api_request(api_kind: ApiKind) { API_REQUEST_COUNTS.record(api_kind); } -/// Only the node running the control plane calls it, so each cluster reports its summary once. +/// Only the node running the control plane calls it. A cluster must have a single control plane +/// (see `docs/deployment/cluster-sizing.md`), so each cluster reports its summary once. pub fn set_cluster_summary(num_indexes: usize, source_types: BTreeSet) { let cluster_summary = ClusterSummary { num_indexes: index_count_bucket(num_indexes).to_string(), From 5609fb80827f797149360d393830a7ed10ae0999 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Fri, 30 Oct 2026 14:19:22 +0000 Subject: [PATCH 10/13] Fix telemetry issues found in review - Disable telemetry for everything cargo runs (tests, cargo run) and for the REST API tests, so local test runs no longer report to production. - Count only the exact shapes of the counted REST routes, so index and template management requests are no longer counted as API usage, and skip OTLP and Jaeger requests on nodes where those endpoints are disabled. - Don't wait for new events when telemetry terminates, so shutdown no longer waits out the 1s timeout. - Start the telemetry loop with a single atomic check instead of a check followed by an assert that could panic. - Log why telemetry is disabled when the HTTP client can't be built. - Report S3 custom endpoints set only through QW_S3_ENDPOINT. - Derive the API kinds with enum-iterator instead of a hand-kept list. - Document per-event timestamps, when install is sent, what the UI page-load event counts, restarts, and what older releases send, and add a CHANGELOG entry. --- .cargo/config.toml | 4 + CHANGELOG.md | 1 + docs/telemetry.md | 16 ++-- quickwit/Cargo.lock | 1 + quickwit/quickwit-cli/src/telemetry.rs | 32 ++++---- quickwit/quickwit-serve/src/api_usage.rs | 96 +++++++++++++++-------- quickwit/quickwit-serve/src/rest.rs | 12 ++- quickwit/quickwit-telemetry/Cargo.toml | 1 + quickwit/quickwit-telemetry/src/lib.rs | 6 +- quickwit/quickwit-telemetry/src/sender.rs | 85 +++++++++++++------- quickwit/quickwit-telemetry/src/sink.rs | 7 +- quickwit/quickwit-telemetry/src/usage.rs | 31 +++----- quickwit/rest-api-tests/run_tests.py | 2 +- 13 files changed, 181 insertions(+), 113 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 493077312e1..d637b80719e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,6 +2,10 @@ rustflags = ["--cfg", "tokio_unstable"] rustdocflags = ["--cfg", "tokio_unstable"] +# Keeps tests and `cargo run` from sending telemetry. A value already set in the environment wins. +[env] +QW_DISABLE_TELEMETRY = "1" + [target.x86_64-unknown-linux-gnu] # x86-64-v3 requires AVX2, BMI1/BMI2, FMA — available on Intel Haswell (2013)+ # and AMD Zen 1 (2017)+. Covers all current-gen AWS and GCP instance families. diff --git a/CHANGELOG.md b/CHANGELOG.md index d0675117ebc..b9f2b1cfcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Azure Blob Storage: support custom endpoints via `endpoint` and `endpoint_suffix` configuration options for sovereign clouds (#6624) +- Restore anonymous usage telemetry, removed in 0.9.0. It is on by default: set `QW_DISABLE_TELEMETRY=1` to turn it off. `docs/telemetry.md` lists what is sent and where it goes (#6927) ### Fixed - (Jaeger) Query resource attributes when Jaeger request carries tags diff --git a/docs/telemetry.md b/docs/telemetry.md index ef1392b5ffb..fe38cd0990f 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -31,15 +31,15 @@ Only `quickwit run` sends telemetry. Other CLI commands never do. ## What is sent -A node sends at most one request per minute, plus one last request with the pending events when it shuts down, with up to 10 events per request. +Each time a node starts, it sends a request right away. After that, it sends at most one request per minute, plus one last request with the pending events when it shuts down. Each request holds up to 10 events. -Events: +Events, each with the time it happened, in seconds since the Unix epoch: -- `install`: the node started with a data directory that it had never used, so this counts new installations. +- `install`: the node found no install ID in its data directory and created one. This happens with a new data directory, but also on the first start after upgrading from a release without install IDs, and on the first start after telemetry is turned on. - `run_command`: the node started. - `end_command`: the node stopped, with its exit code. - `running`: the node is still up. Sent every 12 hours. -- `ui_index_page_load`: someone opened the Quickwit UI. +- `ui_index_page_load`: the node served the Quickwit UI page. Browsers request it when someone opens or reloads the UI, and so does any other client that requests a UI URL, such as an uptime check. Each request also includes: @@ -59,7 +59,7 @@ Each request also includes: - The number of CPUs and the amount of memory available to the node, as ranges such as `5-8` CPUs or `8-16GiB`. - Whether the node runs on Kubernetes, based on whether `KUBERNETES_SERVICE_HOST` is set. - A session ID, generated at random each time the node starts. -- An install ID, generated at random the first time the node uses its data directory and stored there in the `telemetry_install_id` file. It is only created while telemetry is on. +- An install ID, generated at random when the node starts with telemetry on and finds none in its data directory, then stored there in the `telemetry_install_id` file. It is never created while telemetry is off. - An MD5 hash of the host name and the user name, used to count distinct installations from older releases, which have no install ID. The relay replaces it with a keyed hash before storage. See below. - An MD5 hash of the cluster ID and the metastore URI, without the user name, password, query parameters or fragment, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. - The number of events dropped because too many were queued. @@ -82,4 +82,8 @@ Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog ope For nodes running in the cloud, the country and city are usually those of the data center. -Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, without the API counts or cluster summary, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, and the metastore, OTLP and Jaeger features. +Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, each with its time, without the API counts or cluster summary, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, number of dropped events, and the metastore, OTLP and Jaeger features. Their AWS Lambda indexer and searcher send `run_command` with only the `aws_lambda` feature. + +Releases before 0.6.0 also send data to the same address, from every CLI command: an event when a command ends, with its exit code, and an event when an index is created, ingested into, deleted or garbage collected, or when services start, with their names (in 0.1.x, with whether peer addresses were given). Each event has its time, and each request includes the number of dropped events, the version, OS, architecture, Kubernetes flag, session ID and host hash. + +To turn telemetry off in releases before 0.9.0, set `QW_DISABLE_TELEMETRY` to any value, or `DISABLE_QUICKWIT_TELEMETRY` in 0.1.x. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 59ab56d8bcb..4c150fbd40b 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -8454,6 +8454,7 @@ name = "quickwit-telemetry" version = "0.9.3" dependencies = [ "async-trait", + "enum-iterator", "hostname", "md5", "quickwit-common", diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs index 31c3dd44675..d6f292a6e0a 100644 --- a/quickwit/quickwit-cli/src/telemetry.rs +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -99,20 +99,24 @@ fn telemetry_features(config: &NodeConfig) -> HashSet { if let Some(storage_feature) = storage_feature_opt { features.insert(storage_feature); } - if let Some(s3_storage_config) = config.storage_configs.find_s3() { - if s3_storage_config.endpoint().is_some() { - features.insert(QuickwitFeature::S3CustomEndpoint); - } - let flavor_feature_opt = match s3_storage_config.flavor { - Some(StorageBackendFlavor::DigitalOcean) => Some(QuickwitFeature::S3FlavorDigitalOcean), - Some(StorageBackendFlavor::Garage) => Some(QuickwitFeature::S3FlavorGarage), - Some(StorageBackendFlavor::Gcs) => Some(QuickwitFeature::S3FlavorGcs), - Some(StorageBackendFlavor::MinIO) => Some(QuickwitFeature::S3FlavorMinio), - None => None, - }; - if let Some(flavor_feature) = flavor_feature_opt { - features.insert(flavor_feature); - } + // Same fallback as the storage resolver, so that `QW_S3_ENDPOINT` alone is reported. + let s3_storage_config = config + .storage_configs + .find_s3() + .cloned() + .unwrap_or_default(); + if s3_storage_config.endpoint().is_some() { + features.insert(QuickwitFeature::S3CustomEndpoint); + } + let flavor_feature_opt = match s3_storage_config.flavor { + Some(StorageBackendFlavor::DigitalOcean) => Some(QuickwitFeature::S3FlavorDigitalOcean), + Some(StorageBackendFlavor::Garage) => Some(QuickwitFeature::S3FlavorGarage), + Some(StorageBackendFlavor::Gcs) => Some(QuickwitFeature::S3FlavorGcs), + Some(StorageBackendFlavor::MinIO) => Some(QuickwitFeature::S3FlavorMinio), + None => None, + }; + if let Some(flavor_feature) = flavor_feature_opt { + features.insert(flavor_feature); } if config.is_service_enabled(QuickwitService::Searcher) { if config.jaeger_config.enable_endpoint { diff --git a/quickwit/quickwit-serve/src/api_usage.rs b/quickwit/quickwit-serve/src/api_usage.rs index 2b2d2ff2d00..fcbf98ae270 100644 --- a/quickwit/quickwit-serve/src/api_usage.rs +++ b/quickwit/quickwit-serve/src/api_usage.rs @@ -13,37 +13,37 @@ // limitations under the License. use quickwit_telemetry::ApiKind; +use warp::http::Method; -pub(crate) fn classify_rest_request(path: &str) -> Option { +pub(crate) fn classify_rest_request(method: &Method, path: &str) -> Option { let api_path = path.strip_prefix("/api/v1/")?; - let segments: Vec<&str> = api_path.split('/').collect(); - if segments[0] == "_elastic" { - return Some(classify_elasticsearch_request(&segments[1..])); + let api_path = api_path.strip_suffix('/').unwrap_or(api_path); + let (first_segment, rest) = api_path.split_once('/').unwrap_or((api_path, "")); + if first_segment == "_elastic" { + return Some(classify_elasticsearch_request(rest)); } - // `/api/v1/otlp/v1/{logs,traces}`, before the index routes since an index may be named `otlp`. - if segments[0] == "otlp" && segments.get(1) == Some(&"v1") { - return Some(ApiKind::Otlp); - } - match segments.get(1) { - Some(&"search") => Some(ApiKind::NativeSearch), - Some(&"ingest") => Some(ApiKind::NativeIngest), - Some(&"otlp") => Some(ApiKind::Otlp), - Some(&"jaeger") => Some(ApiKind::Jaeger), + // Management routes such as `indexes/{index_id}/...` and `templates/{template_id}` put a + // user-chosen ID after the first segment, so only the exact shapes of the counted routes match. + match (method, rest) { + (&Method::GET | &Method::POST, "search") => Some(ApiKind::NativeSearch), + (&Method::POST, "ingest") => Some(ApiKind::NativeIngest), + (&Method::POST, "v1/logs" | "v1/traces") if first_segment == "otlp" => Some(ApiKind::Otlp), + (&Method::POST, "otlp/v1/logs" | "otlp/v1/traces") => Some(ApiKind::Otlp), + (&Method::GET, _) if rest.starts_with("jaeger/api/") => Some(ApiKind::Jaeger), _ => None, } } -fn classify_elasticsearch_request(segments: &[&str]) -> ApiKind { - if segments.contains(&"_bulk") { - return ApiKind::ElasticsearchIngest; - } - let is_search = segments - .iter() - .any(|segment| matches!(*segment, "_search" | "_msearch" | "_count")); - if is_search { - return ApiKind::ElasticsearchSearch; +fn classify_elasticsearch_request(elasticsearch_path: &str) -> ApiKind { + let mut api_kind = ApiKind::ElasticsearchOther; + for segment in elasticsearch_path.split('/') { + match segment { + "_bulk" => return ApiKind::ElasticsearchIngest, + "_search" | "_msearch" | "_count" => api_kind = ApiKind::ElasticsearchSearch, + _ => {} + } } - ApiKind::ElasticsearchOther + api_kind } pub(crate) fn count_otlp_grpc_request( @@ -68,27 +68,59 @@ mod tests { fn test_classify_rest_request() { let cases = [ ( + Method::POST, "/api/v1/_elastic/logs/_bulk", Some(ApiKind::ElasticsearchIngest), ), ( + Method::POST, "/api/v1/_elastic/_msearch", Some(ApiKind::ElasticsearchSearch), ), ( + Method::GET, "/api/v1/_elastic/_field_caps", Some(ApiKind::ElasticsearchOther), ), - ("/api/v1/logs/search", Some(ApiKind::NativeSearch)), - ("/api/v1/logs/ingest", Some(ApiKind::NativeIngest)), - ("/api/v1/otlp/v1/logs", Some(ApiKind::Otlp)), - ("/api/v1/otlp/search", Some(ApiKind::NativeSearch)), - ("/api/v1/traces/jaeger/api/services", Some(ApiKind::Jaeger)), - ("/api/v1/indexes", None), - ("/ui/search", None), + ( + Method::GET, + "/api/v1/logs/search", + Some(ApiKind::NativeSearch), + ), + ( + Method::POST, + "/api/v1/logs/ingest", + Some(ApiKind::NativeIngest), + ), + (Method::POST, "/api/v1/otlp/v1/logs", Some(ApiKind::Otlp)), + ( + Method::POST, + "/api/v1/logs/otlp/v1/traces", + Some(ApiKind::Otlp), + ), + ( + Method::GET, + "/api/v1/otlp/search", + Some(ApiKind::NativeSearch), + ), + ( + Method::GET, + "/api/v1/traces/jaeger/api/services", + Some(ApiKind::Jaeger), + ), + (Method::GET, "/api/v1/indexes", None), + (Method::GET, "/api/v1/indexes/otlp/splits", None), + (Method::DELETE, "/api/v1/indexes/jaeger", None), + (Method::PUT, "/api/v1/templates/ingest", None), + (Method::GET, "/api/v1/indexes/search/describe", None), + (Method::GET, "/ui/search", None), ]; - for (path, expected_api_kind) in cases { - assert_eq!(classify_rest_request(path), expected_api_kind, "{path}"); + for (method, path, expected_api_kind) in cases { + assert_eq!( + classify_rest_request(&method, path), + expected_api_kind, + "{method} {path}" + ); } } } diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index d1b51d046ab..93e4836fcc8 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -27,6 +27,7 @@ use quickwit_common::tower::BoxFutureInfaillible; use quickwit_config::{disable_ingest_v1, enable_ingest_v2}; use quickwit_metrics::{counter, histogram, labels}; use quickwit_search::SearchService; +use quickwit_telemetry::ApiKind; use tokio::net::{TcpListener, TcpStream}; use tokio::task::JoinSet; use tokio_rustls::TlsAcceptor; @@ -148,7 +149,9 @@ pub(crate) async fn start_rest_server( readiness_trigger: BoxFutureInfaillible<()>, shutdown_signal: BoxFutureInfaillible<()>, ) -> anyhow::Result<()> { - let request_counter = warp::log::custom(|info: Info| { + let is_otlp_enabled = quickwit_services.otlp_logs_service_opt.is_some(); + let is_jaeger_enabled = quickwit_services.jaeger_service_opt.is_some(); + let request_counter = warp::log::custom(move |info: Info| { let elapsed = info.elapsed(); let labels = labels!( "method" => info.method().as_str().to_string(), @@ -164,8 +167,11 @@ pub(crate) async fn start_rest_server( labels: [labels], ) .inc(); - if let Some(api_kind) = classify_rest_request(info.path()) { - quickwit_telemetry::record_api_request(api_kind); + match classify_rest_request(info.method(), info.path()) { + Some(ApiKind::Otlp) if !is_otlp_enabled => {} + Some(ApiKind::Jaeger) if !is_jaeger_enabled => {} + Some(api_kind) => quickwit_telemetry::record_api_request(api_kind), + None => {} } }); // Docs routes diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index 1782702cc10..ab0cd7b5074 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true [dependencies] async-trait = { workspace = true } +enum-iterator = { workspace = true } hostname = { workspace = true } md5 = { workspace = true } reqwest = { workspace = true } diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index aa99757bda6..3030a72615e 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -42,13 +42,13 @@ static TELEMETRY_SENDER: OnceLock = OnceLock::new(); pub fn start_telemetry_loop(quickwit_info: QuickwitTelemetryInfo) -> Option { let telemetry_sender = TELEMETRY_SENDER.get_or_init(|| TelemetrySender::from_quickwit_info(quickwit_info)); + let telemetry_loop_handle_opt = telemetry_sender.start_loop(); // This should not happen... unless telemetry is enabled and you are running tests in parallel // in the same process. - if telemetry_sender.loop_started() { + if telemetry_loop_handle_opt.is_none() { info!("telemetry loop already started. please disable telemetry during tests"); - return None; } - Some(telemetry_sender.start_loop()) + telemetry_loop_handle_opt } /// Sends a telemetry event to the telemetry relay via HTTP. diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index cff17a81743..52df85d38da 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -20,7 +20,7 @@ use std::time::Duration; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinHandle; -use tracing::info; +use tracing::{info, warn}; use crate::clock::Clock; use crate::is_telemetry_disabled; @@ -135,8 +135,7 @@ impl Inner { self.sink.is_none() } - async fn create_telemetry_payload(&self) -> TelemetryPayload { - let events_state = self.events.drain_events().await; + fn create_telemetry_payload(&self, events_state: EventsState) -> TelemetryPayload { TelemetryPayload { schema_version: TELEMETRY_SCHEMA_VERSION, client_info: self.client_info.clone(), @@ -152,11 +151,23 @@ impl Inner { /// If the requests fails, it fails silently. async fn send_pending_events(&self) { if let Some(sink) = self.sink.as_ref() { - let payload = self.create_telemetry_payload().await; + let events_state = self.events.drain_events().await; + let payload = self.create_telemetry_payload(events_state); sink.send_payload(payload).await; } } + /// Sends the queued events, if any, without waiting for new ones. + async fn send_remaining_events(&self) { + if let Some(sink) = self.sink.as_ref() { + let events_state = self.events.state.write().await.drain_events(); + if !events_state.events.is_empty() { + let payload = self.create_telemetry_payload(events_state); + sink.send_payload(payload).await; + } + } + } + async fn send(&self, event: TelemetryEvent) { if self.is_disabled() { return; @@ -225,43 +236,34 @@ impl TelemetrySender { } } - pub fn loop_started(&self) -> bool { - self.inner.is_started.load(Ordering::Relaxed) - } - - pub fn start_loop(&self) -> TelemetryLoopHandle { - let (terminate_command_tx, mut terminate_command_rx) = oneshot::channel(); + /// Returns `None` if the loop is already started. + pub fn start_loop(&self) -> Option { if self.inner.is_disabled() { - return TelemetryLoopHandle::NoLoop; + return Some(TelemetryLoopHandle::NoLoop); } - - assert!( - self.inner - .is_started - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(), - "The telemetry loop is already started." - ); - + if self.inner.is_started.swap(true, Ordering::SeqCst) { + return None; + } + let (terminate_command_tx, mut terminate_command_rx) = oneshot::channel(); let inner = self.inner.clone(); start_monitor_if_server_running_task(inner.clone()); let join_handle = tokio::task::spawn(async move { - // This channel is used to send the command to terminate telemetry. loop { let quit_loop = tokio::select! { _ = (&mut terminate_command_rx) => { true } _ = inner.clock.tick() => { false } }; - inner.send_pending_events().await; if quit_loop { + inner.send_remaining_events().await; break; } + inner.send_pending_events().await; } }); - TelemetryLoopHandle::WithLoop { + Some(TelemetryLoopHandle::WithLoop { join_handle, terminate_command_tx, - } + }) } pub async fn send(&self, event: TelemetryEvent) { @@ -286,7 +288,13 @@ fn create_http_client() -> Option { info!("telemetry is disabled"); return None; } - let client = HttpClient::try_new()?; + let client = match HttpClient::try_new() { + Ok(client) => client, + Err(error) => { + warn!(%error, "failed to create the telemetry HTTP client, telemetry is disabled"); + return None; + } + }; info!("telemetry to {} is enabled", client.endpoint()); Some(client) } @@ -344,7 +352,7 @@ mod tests { let (_clock_btn, clock) = Clock::manual().await; let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop(); + let loop_handler = telemetry_sender.start_loop().unwrap(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; let payload_opt = rx.recv().await; assert!(payload_opt.is_some()); @@ -359,7 +367,7 @@ mod tests { let (clock_btn, clock) = Clock::manual().await; let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop(); + let loop_handler = telemetry_sender.start_loop().unwrap(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -380,7 +388,7 @@ mod tests { let (clock_btn, clock) = Clock::manual().await; let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop(); + let loop_handler = telemetry_sender.start_loop().unwrap(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -405,7 +413,7 @@ mod tests { let (clock_btn, clock) = Clock::manual().await; let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop(); + let loop_handler = telemetry_sender.start_loop().unwrap(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -432,7 +440,7 @@ mod tests { let (_clock_btn, clock) = Clock::manual().await; let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop(); + let loop_handler = telemetry_sender.start_loop().unwrap(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; let payload = rx.recv().await.unwrap(); assert_eq!(payload.events.len(), 1); @@ -447,4 +455,21 @@ mod tests { &TelemetryEvent::EndCommand { .. } )); } + + #[tokio::test(start_paused = true)] + async fn test_terminate_telemetry_does_not_wait_without_pending_events() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let (_clock_btn, clock) = Clock::manual().await; + let telemetry_sender = + TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); + let loop_handler = telemetry_sender.start_loop().unwrap(); + telemetry_sender + .send(TelemetryEvent::end_command(0i32)) + .await; + rx.recv().await.unwrap(); + + let terminate_start = tokio::time::Instant::now(); + loop_handler.terminate_telemetry().await; + assert!(terminate_start.elapsed() < LAST_REQUEST_TIMEOUT); + } } diff --git a/quickwit/quickwit-telemetry/src/sink.rs b/quickwit/quickwit-telemetry/src/sink.rs index 9ca49ebd841..844b9b4e2ec 100644 --- a/quickwit/quickwit-telemetry/src/sink.rs +++ b/quickwit/quickwit-telemetry/src/sink.rs @@ -42,13 +42,12 @@ pub struct HttpClient { } impl HttpClient { - pub fn try_new() -> Option { + pub fn try_new() -> reqwest::Result { let client = Client::builder() .redirect(Policy::limited(3)) .timeout(Duration::from_secs(10)) - .build() - .ok()?; - Some(HttpClient { + .build()?; + Ok(HttpClient { client, endpoint: telemetry_ingest_api_url(), }) diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs index dab623903de..13254c620a4 100644 --- a/quickwit/quickwit-telemetry/src/usage.rs +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -16,11 +16,14 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; +use enum_iterator::{Sequence, all, cardinality}; use serde::{Deserialize, Serialize}; use crate::payload::TelemetryEvent; -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] +#[derive( + Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize, Sequence, +)] #[serde(rename_all = "snake_case")] pub enum ApiKind { ElasticsearchSearch, @@ -32,24 +35,12 @@ pub enum ApiKind { Jaeger, } -impl ApiKind { - /// In declaration order: `ApiRequestCounts` is indexed by `api_kind as usize`. - const ALL: [ApiKind; 7] = [ - ApiKind::ElasticsearchSearch, - ApiKind::ElasticsearchIngest, - ApiKind::ElasticsearchOther, - ApiKind::NativeSearch, - ApiKind::NativeIngest, - ApiKind::Otlp, - ApiKind::Jaeger, - ]; -} - -struct ApiRequestCounts([AtomicU64; ApiKind::ALL.len()]); +/// Indexed by `api_kind as usize`. +struct ApiRequestCounts([AtomicU64; cardinality::()]); impl ApiRequestCounts { const fn new() -> Self { - Self([const { AtomicU64::new(0) }; ApiKind::ALL.len()]) + Self([const { AtomicU64::new(0) }; cardinality::()]) } fn record(&self, api_kind: ApiKind) { @@ -59,7 +50,7 @@ impl ApiRequestCounts { /// Returns the bucketed counts of the APIs that received requests, and resets the counts. fn take(&self) -> Option> { let mut api_usage = BTreeMap::new(); - for api_kind in ApiKind::ALL { + for api_kind in all::() { let num_requests = self.0[api_kind as usize].swap(0, Ordering::Relaxed); if num_requests > 0 { api_usage.insert(api_kind, request_count_bucket(num_requests).to_string()); @@ -165,9 +156,9 @@ mod tests { use super::*; #[test] - fn test_api_kind_all_matches_discriminants() { - for (index, api_kind) in ApiKind::ALL.iter().enumerate() { - assert_eq!(*api_kind as usize, index); + fn test_api_kind_discriminants_follow_declaration_order() { + for (index, api_kind) in all::().enumerate() { + assert_eq!(api_kind as usize, index); } } diff --git a/quickwit/rest-api-tests/run_tests.py b/quickwit/rest-api-tests/run_tests.py index c1ad6c3b950..e2644163602 100755 --- a/quickwit/rest-api-tests/run_tests.py +++ b/quickwit/rest-api-tests/run_tests.py @@ -363,7 +363,7 @@ def __init__(self, quickwit_bin_path): mkdir(config) shutil.copy("../../config/quickwit.yaml", config) shutil.copy(quickwit_bin_path, self.quickwit_dir.name) - self.proc = subprocess.Popen(["./quickwit", "run"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.quickwit_dir.name) + self.proc = subprocess.Popen(["./quickwit", "run"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.quickwit_dir.name, env={"QW_DISABLE_TELEMETRY": "1", **os.environ}) for i in range(100): try: print("Checking on quickwit") From 04863506a2b28afedea1ebb416ecc531b25a7896 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Fri, 30 Oct 2026 14:35:14 +0000 Subject: [PATCH 11/13] Stop sending the cluster hash and the cluster summary The cluster hash merged unrelated default deployments and split clusters whose nodes don't all set the metastore URI. The cluster summary made the control plane walk every source of every index every 5s. Neither was sent before telemetry was removed, so drop both and leave the control plane as on main. --- docs/telemetry.md | 10 +-- quickwit/Cargo.lock | 1 - quickwit/quickwit-cli/src/telemetry.rs | 8 +- quickwit/quickwit-control-plane/Cargo.toml | 1 - .../src/control_plane.rs | 2 - quickwit/quickwit-control-plane/src/lib.rs | 1 - .../quickwit-control-plane/src/telemetry.rs | 85 ------------------- quickwit/quickwit-telemetry/src/lib.rs | 2 +- quickwit/quickwit-telemetry/src/payload.rs | 69 +-------------- quickwit/quickwit-telemetry/src/usage.rs | 62 ++------------ 10 files changed, 13 insertions(+), 228 deletions(-) delete mode 100644 quickwit/quickwit-control-plane/src/telemetry.rs diff --git a/docs/telemetry.md b/docs/telemetry.md index fe38cd0990f..ba611157b8b 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -61,13 +61,9 @@ Each request also includes: - A session ID, generated at random each time the node starts. - An install ID, generated at random when the node starts with telemetry on and finds none in its data directory, then stored there in the `telemetry_install_id` file. It is never created while telemetry is off. - An MD5 hash of the host name and the user name, used to count distinct installations from older releases, which have no install ID. The relay replaces it with a keyed hash before storage. See below. -- An MD5 hash of the cluster ID and the metastore URI, without the user name, password, query parameters or fragment, used to count clusters and the nodes in each. The relay replaces it with a keyed hash before storage. - The number of events dropped because too many were queued. -`running` and `end_command` events also include: - -- How many requests each API received since the previous report, as ranges such as `10-99`: search and bulk ingestion on the Elasticsearch-compatible API, other Elasticsearch-compatible requests, search and ingestion on the native API, OTLP, and Jaeger. Searches from the Quickwit UI count as native searches. -- On the node that runs the control plane only, so once per cluster: the number of indexes, as a range such as `2-10`, and the types of the sources created by users, such as `kafka` or `kinesis`. +`running` and `end_command` events also include how many requests each API received since the previous report, as ranges such as `10-99`: search and bulk ingestion on the Elasticsearch-compatible API, other Elasticsearch-compatible requests, search and ingestion on the native API, OTLP, and Jaeger. Searches from the Quickwit UI count as native searches. Quickwit never sends index or field names, documents, queries, configuration values or credentials. @@ -77,12 +73,12 @@ Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog ope - The request goes through Amazon CloudFront, which looks up the country and city of your IP address and passes them on. The IP address itself goes no further. - A small relay checks the request against the format above, rejects anything else, and keeps only the fields listed on this page. -- The relay replaces the host and cluster hashes with keyed hashes, computed with a secret that only the relay knows, so the stored values can't be matched against guessed host names, user names or cluster IDs. +- The relay replaces the host hash with a keyed hash, computed with a secret that only the relay knows, so the stored value can't be matched against guessed host names and user names. - The relay forwards the events, with the country and city, to Datadog, where they are stored as logs. Your IP address is not stored. For nodes running in the cloud, the country and city are usually those of the data center. -Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, each with its time, without the API counts or cluster summary, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, number of dropped events, and the metastore, OTLP and Jaeger features. Their AWS Lambda indexer and searcher send `run_command` with only the `aws_lambda` feature. +Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, each with its time, without the API counts, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, number of dropped events, and the metastore, OTLP and Jaeger features. Their AWS Lambda indexer and searcher send `run_command` with only the `aws_lambda` feature. Releases before 0.6.0 also send data to the same address, from every CLI command: an event when a command ends, with its exit code, and an event when an index is created, ingested into, deleted or garbage collected, or when services start, with their names (in 0.1.x, with whether peer addresses were given). Each event has its time, and each request includes the number of dropped events, the version, OS, architecture, Kubernetes flag, session ID and host hash. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 4c150fbd40b..79661d1852a 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -7714,7 +7714,6 @@ dependencies = [ "quickwit-metastore", "quickwit-metrics", "quickwit-proto", - "quickwit-telemetry", "rand 0.10.3", "serde", "serde_json", diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs index d6f292a6e0a..0b38511ccf3 100644 --- a/quickwit/quickwit-cli/src/telemetry.rs +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -19,9 +19,7 @@ use quickwit_common::uri::Protocol; use quickwit_config::service::QuickwitService; use quickwit_config::{NodeConfig, StorageBackendFlavor, disable_ingest_v1, enable_ingest_v2}; use quickwit_serve::BuildInfo; -use quickwit_telemetry::payload::{ - BuildTelemetryInfo, QuickwitFeature, QuickwitTelemetryInfo, hash_cluster_identity, -}; +use quickwit_telemetry::payload::{BuildTelemetryInfo, QuickwitFeature, QuickwitTelemetryInfo}; use tracing::warn; pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { @@ -37,10 +35,6 @@ pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryI target: build_info.build_target.to_string(), release: !build_info.version.ends_with("-nightly"), }, - hashed_cluster_id: Some(hash_cluster_identity( - &config.cluster_id, - &config.metastore_uri, - )), ..QuickwitTelemetryInfo::new(services, telemetry_features(config)) } } diff --git a/quickwit/quickwit-control-plane/Cargo.toml b/quickwit/quickwit-control-plane/Cargo.toml index 61b7848cfa7..e0c8e951e98 100644 --- a/quickwit/quickwit-control-plane/Cargo.toml +++ b/quickwit/quickwit-control-plane/Cargo.toml @@ -35,7 +35,6 @@ quickwit-config = { workspace = true } quickwit-ingest = { workspace = true } quickwit-metastore = { workspace = true } quickwit-proto = { workspace = true } -quickwit-telemetry = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs index bdc0896d211..66ede136770 100644 --- a/quickwit/quickwit-control-plane/src/control_plane.rs +++ b/quickwit/quickwit-control-plane/src/control_plane.rs @@ -62,7 +62,6 @@ use crate::ingest::IngestController; use crate::ingest::ingest_controller::{IngestControllerStats, RebalanceShardsCallback}; use crate::metrics::{METASTORE_ERROR_ABORTED, METASTORE_ERROR_MAYBE_EXECUTED, RESTART_TOTAL}; use crate::model::ControlPlaneModel; -use crate::telemetry::update_telemetry_cluster_summary; /// Interval between two controls (or checks) of the desired plan VS running plan. pub(crate) const CONTROL_PLAN_LOOP_INTERVAL: Duration = if cfg!(any(test, feature = "testsuite")) { @@ -507,7 +506,6 @@ impl Handler for ControlPlane { } } self.indexing_scheduler.control_running_plan(&self.model); - update_telemetry_cluster_summary(&self.model); ctx.schedule_self_msg(CONTROL_PLAN_LOOP_INTERVAL, ControlPlaneLoop); Ok(()) } diff --git a/quickwit/quickwit-control-plane/src/lib.rs b/quickwit/quickwit-control-plane/src/lib.rs index df8e3531296..45ee0275d21 100644 --- a/quickwit/quickwit-control-plane/src/lib.rs +++ b/quickwit/quickwit-control-plane/src/lib.rs @@ -18,7 +18,6 @@ pub mod indexing_scheduler; pub mod ingest; pub(crate) mod metrics; pub(crate) mod model; -mod telemetry; use quickwit_common::tower::Pool; use quickwit_proto::indexing::{CpuCapacity, IndexingServiceClient, IndexingTask}; diff --git a/quickwit/quickwit-control-plane/src/telemetry.rs b/quickwit/quickwit-control-plane/src/telemetry.rs deleted file mode 100644 index bf022366f4c..00000000000 --- a/quickwit/quickwit-control-plane/src/telemetry.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeSet; - -use quickwit_config::{CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID}; -use quickwit_proto::metastore::SourceType; - -use crate::model::ControlPlaneModel; - -const RESERVED_SOURCE_IDS: [&str; 3] = [CLI_SOURCE_ID, INGEST_API_SOURCE_ID, INGEST_V2_SOURCE_ID]; - -pub(crate) fn update_telemetry_cluster_summary(model: &ControlPlaneModel) { - let (num_indexes, source_types) = cluster_summary(model); - quickwit_telemetry::set_cluster_summary(num_indexes, source_types); -} - -fn cluster_summary(model: &ControlPlaneModel) -> (usize, BTreeSet) { - let source_types = model - .source_configs() - .filter(|(source_uid, _)| !RESERVED_SOURCE_IDS.contains(&source_uid.source_id.as_str())) - .filter_map(|(_, source_config)| source_type_name(source_config.source_type())) - .map(str::to_string) - .collect(); - (model.num_indexes(), source_types) -} - -fn source_type_name(source_type: SourceType) -> Option<&'static str> { - match source_type { - SourceType::Cli => Some("cli"), - SourceType::File => Some("file"), - SourceType::IngestV1 => Some("ingest_v1"), - SourceType::IngestV2 => Some("ingest_v2"), - SourceType::Kafka => Some("kafka"), - SourceType::Kinesis => Some("kinesis"), - SourceType::Nats => Some("nats"), - SourceType::PubSub => Some("pubsub"), - SourceType::Pulsar => Some("pulsar"), - SourceType::Stdin => Some("stdin"), - SourceType::Vec => Some("vec"), - SourceType::Void => Some("void"), - SourceType::Unspecified => None, - } -} - -#[cfg(test)] -mod tests { - use quickwit_config::{SourceConfig, SourceParams}; - use quickwit_metastore::IndexMetadata; - - use super::*; - - #[test] - fn test_cluster_summary_leaves_out_reserved_sources() { - let mut model = ControlPlaneModel::default(); - assert_eq!(cluster_summary(&model), (0, BTreeSet::new())); - - let mut index_metadata = IndexMetadata::for_test("test-index", "ram:///indexes"); - index_metadata.add_source(SourceConfig::cli()).unwrap(); - index_metadata - .add_source(SourceConfig::ingest_v2()) - .unwrap(); - index_metadata - .add_source(SourceConfig::for_test("my-source", SourceParams::void())) - .unwrap(); - model.add_index(index_metadata); - model.add_index(IndexMetadata::for_test("other-index", "ram:///indexes")); - - assert_eq!( - cluster_summary(&model), - (2, BTreeSet::from(["void".to_string()])) - ); - } -} diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 3030a72615e..88f2165903a 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -34,7 +34,7 @@ use tracing::info; pub use crate::install::load_or_create_install_id; use crate::payload::TelemetryEvent; use crate::sender::{TelemetryLoopHandle, TelemetrySender}; -pub use crate::usage::{ApiKind, record_api_request, set_cluster_summary}; +pub use crate::usage::{ApiKind, record_api_request}; static TELEMETRY_SENDER: OnceLock = OnceLock::new(); diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index c857b6e9849..2c25af8d599 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -16,12 +16,11 @@ use std::collections::{BTreeMap, HashSet}; use std::env; use std::time::UNIX_EPOCH; -use quickwit_common::uri::Uri; use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::machine::{available_memory_num_bytes, cpu_count_bucket, memory_bucket}; -use crate::usage::{ApiKind, ClusterSummary}; +use crate::usage::ApiKind; /// Version of the payload format. Payloads without one are version 1. pub(crate) const TELEMETRY_SCHEMA_VERSION: u32 = 2; @@ -79,15 +78,11 @@ pub enum TelemetryEvent { return_code: i32, #[serde(default, skip_serializing_if = "Option::is_none")] api_usage: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - cluster_summary: Option, }, /// Event sent every 12h to signal the server is running. Running { #[serde(default, skip_serializing_if = "Option::is_none")] api_usage: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - cluster_summary: Option, }, /// UI index.html was requested. UiIndexPageLoad, @@ -131,8 +126,6 @@ pub struct QuickwitTelemetryInfo { pub build: BuildTelemetryInfo, #[serde(skip_serializing_if = "Option::is_none")] pub install_uuid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub hashed_cluster_id: Option, } impl QuickwitTelemetryInfo { @@ -153,7 +146,6 @@ impl Default for QuickwitTelemetryInfo { services: HashSet::new(), build: BuildTelemetryInfo::default(), install_uuid: None, - hashed_cluster_id: None, } } } @@ -182,25 +174,6 @@ pub enum QuickwitFeature { Tls, } -/// Leaves out the user, the password, the query parameters and the fragment of the metastore URI, -/// which may differ between the nodes of a cluster. -pub fn hash_cluster_identity(cluster_id: &str, metastore_uri: &Uri) -> String { - let mut location = metastore_uri.as_str(); - if let Some((_, after_scheme)) = location.split_once("://") { - location = after_scheme; - } - if let Some(query_or_fragment_start) = location.find(['?', '#']) { - location = &location[..query_or_fragment_start]; - } - let authority_len = location.find('/').unwrap_or(location.len()); - if let Some(at_index) = location[..authority_len].rfind('@') { - location = &location[at_index + 1..]; - } - let protocol = metastore_uri.protocol(); - let digest = md5::compute(format!("{cluster_id}\n{protocol}://{location}")); - format!("{digest:x}") -} - fn hashed_host_username() -> String { let hostname = hostname::get() .map(|hostname| hostname.to_string_lossy().to_string()) @@ -227,43 +200,10 @@ impl Default for ClientInfo { #[cfg(test)] mod tests { - use std::str::FromStr; - use serde_json; use super::*; - #[test] - fn test_hash_cluster_identity_leaves_out_credentials() { - let hashed_cluster_id = |cluster_id, metastore_uri| { - hash_cluster_identity(cluster_id, &Uri::from_str(metastore_uri).unwrap()) - }; - let expected = hashed_cluster_id("my-cluster", "postgresql://db:5432/metastore"); - assert_eq!( - hashed_cluster_id( - "my-cluster", - "postgresql://node-1:secret@db:5432/metastore?sslmode=require" - ), - expected - ); - assert_ne!( - hashed_cluster_id("other-cluster", "postgresql://db:5432/metastore"), - expected - ); - assert_eq!( - hashed_cluster_id("my-cluster", "postgres://db:5432/metastore"), - expected - ); - assert_eq!( - hashed_cluster_id("my-cluster", "s3://bucket/metastore#polling_interval=30s"), - hashed_cluster_id("my-cluster", "s3://bucket/metastore") - ); - assert_ne!( - hashed_cluster_id("my-cluster", "gs://bucket/metastore"), - hashed_cluster_id("my-cluster", "s3://bucket/metastore") - ); - } - #[test] fn test_serialize_features_as_expected() { // These names are part of the telemetry payload documented in `docs/telemetry.md`. @@ -302,7 +242,6 @@ mod tests { event: TelemetryEvent::EndCommand { return_code: 0, api_usage: None, - cluster_summary: None, }, }; let json = serde_json::to_string(&event).unwrap(); @@ -318,16 +257,12 @@ mod tests { unixtime: 0, event: TelemetryEvent::Running { api_usage: Some(BTreeMap::from([(ApiKind::NativeSearch, "1-9".to_string())])), - cluster_summary: Some(ClusterSummary { - num_indexes: "2-10".to_string(), - source_types: ["kafka".to_string()].into(), - }), }, }; let json = serde_json::to_string(&event).unwrap(); assert_eq!( json, - r#"{"unixtime":0,"type":"running","api_usage":{"native_search":"1-9"},"cluster_summary":{"num_indexes":"2-10","source_types":["kafka"]}}"# + r#"{"unixtime":0,"type":"running","api_usage":{"native_search":"1-9"}}"# ); } } diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs index 13254c620a4..c4d27226919 100644 --- a/quickwit/quickwit-telemetry/src/usage.rs +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Mutex, PoisonError}; use enum_iterator::{Sequence, all, cardinality}; use serde::{Deserialize, Serialize}; @@ -65,64 +64,29 @@ impl ApiRequestCounts { static API_REQUEST_COUNTS: ApiRequestCounts = ApiRequestCounts::new(); -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct ClusterSummary { - pub num_indexes: String, - pub source_types: BTreeSet, -} - -static CLUSTER_SUMMARY: Mutex> = Mutex::new(None); - pub fn record_api_request(api_kind: ApiKind) { API_REQUEST_COUNTS.record(api_kind); } -/// Only the node running the control plane calls it. A cluster must have a single control plane -/// (see `docs/deployment/cluster-sizing.md`), so each cluster reports its summary once. -pub fn set_cluster_summary(num_indexes: usize, source_types: BTreeSet) { - let cluster_summary = ClusterSummary { - num_indexes: index_count_bucket(num_indexes).to_string(), - source_types, - }; - *CLUSTER_SUMMARY - .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(cluster_summary); -} - impl TelemetryEvent { pub fn running() -> Self { - TelemetryEvent::Running { - api_usage: None, - cluster_summary: None, - } + TelemetryEvent::Running { api_usage: None } } pub fn end_command(return_code: i32) -> Self { TelemetryEvent::EndCommand { return_code, api_usage: None, - cluster_summary: None, } } - /// Adds the API request counts since the previous report and the cluster summary. Only called - /// once the event is queued, so that a dropped event never loses the counts. + /// Adds the API request counts since the previous report. Only called once the event is + /// queued, so that a dropped event never loses the counts. pub(crate) fn add_usage(&mut self) { - if let TelemetryEvent::Running { - api_usage, - cluster_summary, - } - | TelemetryEvent::EndCommand { - api_usage, - cluster_summary, - .. - } = self + if let TelemetryEvent::Running { api_usage } + | TelemetryEvent::EndCommand { api_usage, .. } = self { *api_usage = API_REQUEST_COUNTS.take(); - *cluster_summary = CLUSTER_SUMMARY - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); } } } @@ -140,17 +104,6 @@ fn request_count_bucket(num_requests: u64) -> &'static str { } } -fn index_count_bucket(num_indexes: usize) -> &'static str { - match num_indexes { - 0 => "0", - 1 => "1", - 2..=10 => "2-10", - 11..=100 => "11-100", - 101..=1_000 => "101-1000", - _ => "1001+", - } -} - #[cfg(test)] mod tests { use super::*; @@ -185,8 +138,5 @@ mod tests { assert_eq!(request_count_bucket(9), "1-9"); assert_eq!(request_count_bucket(10), "10-99"); assert_eq!(request_count_bucket(1_000_000), "1000000+"); - assert_eq!(index_count_bucket(1), "1"); - assert_eq!(index_count_bucket(2), "2-10"); - assert_eq!(index_count_bucket(1_001), "1001+"); } } From 2a5c0ffd6be2a086a8765d3428968acb5290ad1c Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Fri, 30 Oct 2026 15:53:25 +0000 Subject: [PATCH 12/13] Skip API usage counting when telemetry is off The REST server no longer classifies requests, and the OTLP and Jaeger gRPC interceptors no longer count them, unless this process sends telemetry. --- quickwit/quickwit-serve/src/rest.rs | 4 ++++ quickwit/quickwit-telemetry/src/lib.rs | 8 ++++++++ quickwit/quickwit-telemetry/src/usage.rs | 4 +++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index 93e4836fcc8..a09c336ad3d 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -149,6 +149,7 @@ pub(crate) async fn start_rest_server( readiness_trigger: BoxFutureInfaillible<()>, shutdown_signal: BoxFutureInfaillible<()>, ) -> anyhow::Result<()> { + let is_telemetry_enabled = quickwit_telemetry::is_telemetry_enabled(); let is_otlp_enabled = quickwit_services.otlp_logs_service_opt.is_some(); let is_jaeger_enabled = quickwit_services.jaeger_service_opt.is_some(); let request_counter = warp::log::custom(move |info: Info| { @@ -167,6 +168,9 @@ pub(crate) async fn start_rest_server( labels: [labels], ) .inc(); + if !is_telemetry_enabled { + return; + } match classify_rest_request(info.method(), info.path()) { Some(ApiKind::Otlp) if !is_otlp_enabled => {} Some(ApiKind::Jaeger) if !is_jaeger_enabled => {} diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 88f2165903a..41cf21c8433 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -69,6 +69,14 @@ pub async fn send_telemetry_event(event: TelemetryEvent) { } } +/// Returns true once this process started a telemetry loop that sends events. +pub fn is_telemetry_enabled() -> bool { + let Some(telemetry_sender) = TELEMETRY_SENDER.get() else { + return false; + }; + !telemetry_sender.inner.is_disabled() +} + /// This environment variable can be set to disable sending telemetry events. pub const DISABLE_TELEMETRY_ENV_KEY: &str = "QW_DISABLE_TELEMETRY"; diff --git a/quickwit/quickwit-telemetry/src/usage.rs b/quickwit/quickwit-telemetry/src/usage.rs index c4d27226919..10852a5d67e 100644 --- a/quickwit/quickwit-telemetry/src/usage.rs +++ b/quickwit/quickwit-telemetry/src/usage.rs @@ -65,7 +65,9 @@ impl ApiRequestCounts { static API_REQUEST_COUNTS: ApiRequestCounts = ApiRequestCounts::new(); pub fn record_api_request(api_kind: ApiKind) { - API_REQUEST_COUNTS.record(api_kind); + if crate::is_telemetry_enabled() { + API_REQUEST_COUNTS.record(api_kind); + } } impl TelemetryEvent { From bfbe45f0d0b5e386658c74e1b9624681ca796019 Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Mon, 2 Nov 2026 17:57:38 +0000 Subject: [PATCH 13/13] Fix telemetry shutdown, API counting and docs issues found in review --- LICENSE-3rdparty.csv | 3 - docs/operating/data-directory.md | 5 + docs/reference/cli.md | 8 + docs/telemetry.md | 8 +- quickwit/Cargo.lock | 49 +--- quickwit/Cargo.toml | 2 +- quickwit/quickwit-cli/src/service.rs | 12 +- quickwit/quickwit-cli/src/telemetry.rs | 23 -- quickwit/quickwit-serve/src/api_usage.rs | 46 +++- quickwit/quickwit-serve/src/rest.rs | 3 +- quickwit/quickwit-serve/src/ui_handler.rs | 17 +- quickwit/quickwit-telemetry/Cargo.toml | 3 +- quickwit/quickwit-telemetry/src/clock.rs | 53 +--- quickwit/quickwit-telemetry/src/install.rs | 2 +- quickwit/quickwit-telemetry/src/lib.rs | 103 +++++--- quickwit/quickwit-telemetry/src/payload.rs | 12 +- quickwit/quickwit-telemetry/src/sender.rs | 272 ++++++--------------- quickwit/quickwit-telemetry/src/sink.rs | 18 +- 18 files changed, 252 insertions(+), 387 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index d09d4a0ddfb..4252326571f 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2,7 +2,6 @@ Component,Origin,License,Copyright RustyXML,https://github.com/Florob/RustyXML,MIT OR Apache-2.0,Florian Zeitz addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line Authors adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " -advapi32-sys,https://github.com/retep998/winapi-rs,MIT,Peter Atashian aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers aes,https://github.com/RustCrypto/block-ciphers,MIT OR Apache-2.0,RustCrypto Developers aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Developers @@ -849,7 +848,6 @@ unwrap-infallible,https://github.com/mzabaluev/unwrap-infallible,MIT OR Apache-2 ureq-proto,https://github.com/algesten/ureq-proto,MIT OR Apache-2.0,Martin Algesten url,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers urlencoding,https://github.com/kornelski/rust_urlencoding,MIT,"Kornel , Bertram Truong " -username,https://pijul.org/darcs/user,MIT OR Apache-2.0,Pierre-Étienne Meunier utf8-ranges,https://github.com/BurntSushi/utf8-ranges,Unlicense OR MIT,Andrew Gallant utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8-zero,https://github.com/algesten/utf8-zero,MIT OR Apache-2.0,"Simon Sapin , Martin Algesten " @@ -887,7 +885,6 @@ web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Aut webpki-root-certs,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-root-certs Authors webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors -winapi,https://github.com/retep998/winapi-rs,MIT,Peter Atashian winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant diff --git a/docs/operating/data-directory.md b/docs/operating/data-directory.md index aca5b4c8baa..b3bbaae8452 100644 --- a/docs/operating/data-directory.md +++ b/docs/operating/data-directory.md @@ -22,6 +22,7 @@ qwdata ├── indexing │ ├── wikipedia%01H13SVKDS03P%_ingest-api-source%RbaOAI │ └── wikipedia%01H13SVKDS03P%kafka-source%cNqQtI +├── telemetry_install_id ├── wal │ ├── wal-00000000000000000056 │ └── wal-00000000000000000057 @@ -58,6 +59,10 @@ You can [configure](../configuration/node-config#indexer-configuration) the numb This directory is used by searcher nodes to cache entire splits and reduce calls to the object store. It won't be created unless you set the `split_cache` fields in the [searcher configuration](../configuration/node-config.md#searcher-configuration). +### `/telemetry_install_id` file + +This file stores the install ID that [telemetry](../telemetry.md) reports, and is only created while telemetry is on. Nodes started from copies of the same data directory report the same install ID, and deleting the file makes the node report a new `install` event. + ## Setting the right splits cache limits diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 82ddd236925..cc212dfe17f 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -863,6 +863,14 @@ Disables [telemetry](../telemetry.md) when set to any non-empty value other than `QW_DISABLE_TELEMETRY=1 quickwit --help` +### TELEMETRY_INGEST_API + +Sends [telemetry](../telemetry.md) to this URL instead of `https://telemetry.quickwit.io/`. The URL is used as is, so the data is not encrypted with an `http://` URL. It does not turn telemetry off. + +*Example* + +`TELEMETRY_INGEST_API=http://127.0.0.1:8080/ quickwit run` + ### QW_POSTGRES_SKIP_MIGRATIONS Don't run database migrations (but verify that migrations were run successfully before, and no that unknown migration was run). diff --git a/docs/telemetry.md b/docs/telemetry.md index ba611157b8b..b6b2761461e 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -25,7 +25,7 @@ Sub-second search & analytics engine on cloud storage. Telemetry: enabled ``` -The line disappears when telemetry is off. +The line disappears when telemetry is off. If the node can't set up HTTPS, for example because no CA certificates are installed, `quickwit run` logs a warning and sends nothing even though the line is shown. Only `quickwit run` sends telemetry. Other CLI commands never do. @@ -69,15 +69,17 @@ Quickwit never sends index or field names, documents, queries, configuration val ## Where the data goes -Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog operates on AWS. +By default, Quickwit sends the data over HTTPS to `telemetry.quickwit.io`, which Datadog operates on AWS. -- The request goes through Amazon CloudFront, which looks up the country and city of your IP address and passes them on. The IP address itself goes no further. +- The request goes through Amazon CloudFront, which looks up the country and city of your IP address and passes them on to the relay. CloudFront also passes on the IP address itself, in the `X-Forwarded-For` header that it adds to every request, but the relay ignores it. - A small relay checks the request against the format above, rejects anything else, and keeps only the fields listed on this page. - The relay replaces the host hash with a keyed hash, computed with a secret that only the relay knows, so the stored value can't be matched against guessed host names and user names. - The relay forwards the events, with the country and city, to Datadog, where they are stored as logs. Your IP address is not stored. For nodes running in the cloud, the country and city are usually those of the data center. +To send the data somewhere else instead, for example to a local HTTP server to see exactly what is sent, set `TELEMETRY_INGEST_API` to a URL before starting Quickwit. Nothing then goes to `telemetry.quickwit.io`, and the relay is bypassed: the receiver gets the requests as Quickwit sends them, unencrypted with an `http://` URL. This does not turn telemetry off. + Releases 0.6.0 to 0.8.x send an older version of this data to the same address: the events other than `install`, each with its time, without the API counts, plus the version, services, OS, architecture, Kubernetes flag, session ID, host hash, number of dropped events, and the metastore, OTLP and Jaeger features. Their AWS Lambda indexer and searcher send `run_command` with only the `aws_lambda` feature. Releases before 0.6.0 also send data to the same address, from every CLI command: an event when a command ends, with its exit code, and an event when an index is created, ingested into, deleted or garbage collected, or when services start, with their names (in 0.1.x, with whether peer addresses were given). Each event has its time, and each request includes the number of dropped events, the version, OS, architecture, Kubernetes flag, session ID and host hash. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 79661d1852a..d9646f52a00 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -23,16 +23,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "advapi32-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "aead" version = "0.5.2" @@ -3364,7 +3354,7 @@ dependencies = [ "cc", "lazy_static", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3511,7 +3501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -5563,7 +5553,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6280,7 +6270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6721,7 +6711,7 @@ dependencies = [ "libc", "pnet_base", "pnet_sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6764,7 +6754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d4643d3d4db6b08741050c2f3afa9a892c4244c085a72fcda93c9c2c9a00f4b" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -7356,7 +7346,7 @@ dependencies = [ "raw-cpuid", "wasi 0.11.1+wasi-snapshot-preview1", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -8464,7 +8454,6 @@ dependencies = [ "tempfile", "tokio", "tracing", - "username", "uuid", ] @@ -10809,7 +10798,7 @@ dependencies = [ "typetag", "unwrap-infallible", "uuid", - "winapi 0.3.9", + "winapi", "zstd 0.14.3", ] @@ -11862,16 +11851,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "username" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e425df6527f7bc1adc7eb3b829ecaec746fbbc0b05e42133ff84afef3b1a09" -dependencies = [ - "advapi32-sys", - "winapi 0.2.8", -] - [[package]] name = "utf8-ranges" version = "1.0.5" @@ -12372,12 +12351,6 @@ dependencies = [ "wasite", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -12388,12 +12361,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 6f96ae73c7f..275449f3f39 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -234,7 +234,7 @@ sqlx = { version = "0.8", features = [ ] } syn = { version = "3.0", features = ["extra-traits", "full", "parsing"] } sync_wrapper = "1" -sysinfo = { version = "0.39", default-features = false, features = ["disk", "system"] } +sysinfo = { version = "0.39", default-features = false, features = ["disk"] } tabled = { version = "0.22", features = ["ansi"] } tempfile = "3" thiserror = "2" diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index c0af34f61c2..52e06edbca8 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -30,7 +30,7 @@ use tokio::signal; use tracing::{debug, info}; use crate::checklist::{BLUE_COLOR, RED_COLOR}; -use crate::telemetry::{quickwit_telemetry_info, set_install_uuid}; +use crate::telemetry::quickwit_telemetry_info; use crate::{config_cli_arg, get_resolvers, load_node_config, start_actor_runtimes}; pub fn build_run_command() -> Command { @@ -126,12 +126,10 @@ impl RunCliCommand { let (storage_resolver, metastore_resolver) = get_resolvers(&node_config.storage_configs, &node_config.metastore_configs); crate::busy_detector::set_enabled(true); - let mut telemetry_info = quickwit_telemetry_info(&node_config); - let is_new_install = set_install_uuid(&mut telemetry_info, &node_config.data_dir_path); - let telemetry_handle_opt = quickwit_telemetry::start_telemetry_loop(telemetry_info); - if is_new_install { - quickwit_telemetry::send_telemetry_event(TelemetryEvent::Install).await; - } + let telemetry_info = quickwit_telemetry_info(&node_config); + let telemetry_handle_opt = + quickwit_telemetry::start_telemetry_loop(telemetry_info, &node_config.data_dir_path) + .await; quickwit_telemetry::send_telemetry_event(TelemetryEvent::RunCommand).await; // TODO move in serve quickwit? let runtimes_config = RuntimesConfig::default(); diff --git a/quickwit/quickwit-cli/src/telemetry.rs b/quickwit/quickwit-cli/src/telemetry.rs index 0b38511ccf3..a60c55cf0c6 100644 --- a/quickwit/quickwit-cli/src/telemetry.rs +++ b/quickwit/quickwit-cli/src/telemetry.rs @@ -13,14 +13,12 @@ // limitations under the License. use std::collections::HashSet; -use std::path::Path; use quickwit_common::uri::Protocol; use quickwit_config::service::QuickwitService; use quickwit_config::{NodeConfig, StorageBackendFlavor, disable_ingest_v1, enable_ingest_v2}; use quickwit_serve::BuildInfo; use quickwit_telemetry::payload::{BuildTelemetryInfo, QuickwitFeature, QuickwitTelemetryInfo}; -use tracing::warn; pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { let services = config @@ -39,27 +37,6 @@ pub(crate) fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryI } } -/// Returns `true` when the install ID was just created. Does nothing when telemetry is disabled, so -/// that a node with telemetry off never writes the ID. -pub(crate) fn set_install_uuid( - telemetry_info: &mut QuickwitTelemetryInfo, - data_dir: &Path, -) -> bool { - if quickwit_telemetry::is_telemetry_disabled() { - return false; - } - match quickwit_telemetry::load_or_create_install_id(data_dir) { - Ok((install_uuid, created)) => { - telemetry_info.install_uuid = Some(install_uuid); - created - } - Err(error) => { - warn!(%error, "failed to load or create the telemetry install ID"); - false - } - } -} - fn telemetry_features(config: &NodeConfig) -> HashSet { let mut features = HashSet::new(); if config.is_service_enabled(QuickwitService::Indexer) diff --git a/quickwit/quickwit-serve/src/api_usage.rs b/quickwit/quickwit-serve/src/api_usage.rs index fcbf98ae270..4574d945b4c 100644 --- a/quickwit/quickwit-serve/src/api_usage.rs +++ b/quickwit/quickwit-serve/src/api_usage.rs @@ -20,7 +20,7 @@ pub(crate) fn classify_rest_request(method: &Method, path: &str) -> Option Option ApiKind { - let mut api_kind = ApiKind::ElasticsearchOther; - for segment in elasticsearch_path.split('/') { - match segment { - "_bulk" => return ApiKind::ElasticsearchIngest, - "_search" | "_msearch" | "_count" => api_kind = ApiKind::ElasticsearchSearch, - _ => {} - } +/// Matches the shapes and methods of the search and bulk routes in `elasticsearch_api/filter.rs`. +/// Any other request, including `DELETE _search/scroll` and requests rejected for their method, +/// counts as other. +fn classify_elasticsearch_request(method: &Method, elasticsearch_path: &str) -> ApiKind { + let (first_segment, rest_opt) = match elasticsearch_path.split_once('/') { + Some((first_segment, rest)) => (first_segment, Some(rest)), + None => (elasticsearch_path, None), + }; + match (method, first_segment, rest_opt) { + (&Method::POST | &Method::PUT, "_bulk", None) + | (&Method::POST | &Method::PUT, _, Some("_bulk")) => ApiKind::ElasticsearchIngest, + (&Method::GET | &Method::POST, "_search", None | Some("scroll")) + | (&Method::GET | &Method::POST, _, Some("_search" | "_count")) + | (&Method::POST, "_msearch", None) => ApiKind::ElasticsearchSearch, + _ => ApiKind::ElasticsearchOther, } - api_kind } pub(crate) fn count_otlp_grpc_request( @@ -82,6 +88,26 @@ mod tests { "/api/v1/_elastic/_field_caps", Some(ApiKind::ElasticsearchOther), ), + ( + Method::GET, + "/api/v1/_elastic/_bulk", + Some(ApiKind::ElasticsearchOther), + ), + ( + Method::GET, + "/api/v1/_elastic/_search/scroll", + Some(ApiKind::ElasticsearchSearch), + ), + ( + Method::DELETE, + "/api/v1/_elastic/_search/scroll", + Some(ApiKind::ElasticsearchOther), + ), + ( + Method::POST, + "/api/v1/_elastic/logs/_msearch", + Some(ApiKind::ElasticsearchOther), + ), ( Method::GET, "/api/v1/logs/search", diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index a09c336ad3d..e9976c8ff66 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -149,7 +149,6 @@ pub(crate) async fn start_rest_server( readiness_trigger: BoxFutureInfaillible<()>, shutdown_signal: BoxFutureInfaillible<()>, ) -> anyhow::Result<()> { - let is_telemetry_enabled = quickwit_telemetry::is_telemetry_enabled(); let is_otlp_enabled = quickwit_services.otlp_logs_service_opt.is_some(); let is_jaeger_enabled = quickwit_services.jaeger_service_opt.is_some(); let request_counter = warp::log::custom(move |info: Info| { @@ -168,7 +167,7 @@ pub(crate) async fn start_rest_server( labels: [labels], ) .inc(); - if !is_telemetry_enabled { + if !quickwit_telemetry::is_telemetry_enabled() { return; } match classify_rest_request(info.method(), info.path()) { diff --git a/quickwit/quickwit-serve/src/ui_handler.rs b/quickwit/quickwit-serve/src/ui_handler.rs index 2cdecec50a6..0c9ded6b887 100644 --- a/quickwit/quickwit-serve/src/ui_handler.rs +++ b/quickwit/quickwit-serve/src/ui_handler.rs @@ -48,17 +48,20 @@ async fn serve_file(path: Tail) -> Result { async fn serve_impl(path: &str) -> Result, Rejection> { static PATH_PTN: LazyLock = LazyLock::new(|| Regex::new(PATH_PATTERN).unwrap()); - let path_to_file = if PATH_PTN.is_match(path) { - path + // Quickwit UI is a single page application. + // Any path request that is not an asset should serve the `index.html` file. + let is_index_page = !PATH_PTN.is_match(path); + let path_to_file = if is_index_page { + UI_INDEX_FILE_NAME } else { - // Quickwit UI is a single page application. - // Any path request that is not an asset should serve the `index.html` file. + path + }; + let asset = Asset::get(path_to_file).ok_or_else(warp::reject::not_found)?; + if is_index_page { // The client (browser) usually request `index.html` once unless the user refreshes the // page. quickwit_telemetry::send_telemetry_event(TelemetryEvent::UiIndexPageLoad).await; - UI_INDEX_FILE_NAME - }; - let asset = Asset::get(path_to_file).ok_or_else(warp::reject::not_found)?; + } let mime = mime_guess::from_path(path_to_file).first_or_octet_stream(); let mut res = Response::new(asset.data.into_owned().into()); diff --git a/quickwit/quickwit-telemetry/Cargo.toml b/quickwit/quickwit-telemetry/Cargo.toml index ab0cd7b5074..96788f76ad5 100644 --- a/quickwit/quickwit-telemetry/Cargo.toml +++ b/quickwit/quickwit-telemetry/Cargo.toml @@ -17,10 +17,9 @@ hostname = { workspace = true } md5 = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } -sysinfo = { workspace = true } +sysinfo = { workspace = true, features = ["system"] } tokio = { workspace = true } tracing = { workspace = true } -username = { workspace = true } uuid = { workspace = true } quickwit-common = { workspace = true } diff --git a/quickwit/quickwit-telemetry/src/clock.rs b/quickwit/quickwit-telemetry/src/clock.rs index 0d776e6a16a..7ce60618eb5 100644 --- a/quickwit/quickwit-telemetry/src/clock.rs +++ b/quickwit/quickwit-telemetry/src/clock.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; #[cfg(test)] @@ -31,43 +30,23 @@ impl ClockButton { } pub(crate) enum Clock { - /// The first tick is immediate. Every later tick waits for the cooldown from the moment it is - /// awaited. The telemetry loop awaits a tick right after each request, so requests are at - /// least a cooldown apart, however long the loop waited for events in between. - Cooldown { - cooldown: Duration, - is_first_tick: AtomicBool, - }, + /// Each tick waits for the cooldown. The telemetry loop ticks right after each request, so + /// requests are at least a cooldown apart. + Cooldown(Duration), #[cfg(test)] Manual(Mutex>), } impl Clock { - pub(crate) fn cooldown(cooldown: Duration) -> Clock { - Clock::Cooldown { - cooldown, - is_first_tick: AtomicBool::new(true), - } - } - #[cfg(test)] - pub(crate) async fn manual() -> (ClockButton, Clock) { + pub(crate) fn manual() -> (ClockButton, Clock) { let (tx, rx) = tokio::sync::mpsc::channel(1); - let _ = tx.send(()).await; - let button = ClockButton(tx); - (button, Clock::Manual(Mutex::new(rx))) + (ClockButton(tx), Clock::Manual(Mutex::new(rx))) } pub(crate) async fn tick(&self) { match self { - Clock::Cooldown { - cooldown, - is_first_tick, - } => { - if !is_first_tick.swap(false, Ordering::Relaxed) { - tokio::time::sleep(*cooldown).await; - } - } + Clock::Cooldown(cooldown) => tokio::time::sleep(*cooldown).await, #[cfg(test)] Clock::Manual(channel) => { channel.lock().await.recv().await; @@ -75,23 +54,3 @@ impl Clock { } } } - -#[cfg(test)] -mod tests { - use super::*; - - const COOLDOWN: Duration = Duration::from_secs(60); - - #[tokio::test(start_paused = true)] - async fn test_cooldown_clock_waits_for_cooldown_after_idle_time() { - let clock = Clock::cooldown(COOLDOWN); - let first_tick_start = tokio::time::Instant::now(); - clock.tick().await; - assert_eq!(first_tick_start.elapsed(), Duration::ZERO); - - tokio::time::sleep(10 * COOLDOWN).await; - let tick_start = tokio::time::Instant::now(); - clock.tick().await; - assert!(tick_start.elapsed() >= COOLDOWN); - } -} diff --git a/quickwit/quickwit-telemetry/src/install.rs b/quickwit/quickwit-telemetry/src/install.rs index 8bf7c867a67..460199b5a99 100644 --- a/quickwit/quickwit-telemetry/src/install.rs +++ b/quickwit/quickwit-telemetry/src/install.rs @@ -23,7 +23,7 @@ const INSTALL_ID_TEMP_FILE_NAME: &str = "telemetry_install_id.tmp"; /// Returns the install ID stored in `data_dir`, creating it if needed, and whether it was just /// created. Returns an error if the stored install ID is invalid. -pub fn load_or_create_install_id(data_dir: &Path) -> io::Result<(Uuid, bool)> { +pub(crate) fn load_or_create_install_id(data_dir: &Path) -> io::Result<(Uuid, bool)> { let install_id_path = data_dir.join(INSTALL_ID_FILE_NAME); match std::fs::read_to_string(&install_id_path) { Ok(install_id_str) => { diff --git a/quickwit/quickwit-telemetry/src/lib.rs b/quickwit/quickwit-telemetry/src/lib.rs index 41cf21c8433..dd0d3421529 100644 --- a/quickwit/quickwit-telemetry/src/lib.rs +++ b/quickwit/quickwit-telemetry/src/lib.rs @@ -12,83 +12,114 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(clippy::bool_assert_comparison)] +//! This crate contains the code responsible for sending anonymous usage data to the telemetry +//! relay. See `docs/telemetry.md` for the data that is sent. + #![deny(clippy::disallowed_methods)] mod clock; mod install; mod machine; pub mod payload; -/// This crate contains the code responsible for sending anonymous usage data to the telemetry -/// relay. See `docs/telemetry.md` for the data that is sent. mod sender; pub(crate) mod sink; mod usage; use std::ffi::OsStr; +use std::path::Path; use std::sync::OnceLock; use payload::QuickwitTelemetryInfo; -use tracing::info; +use tracing::{info, warn}; -pub use crate::install::load_or_create_install_id; +use crate::install::load_or_create_install_id; use crate::payload::TelemetryEvent; use crate::sender::{TelemetryLoopHandle, TelemetrySender}; +use crate::sink::HttpClient; pub use crate::usage::{ApiKind, record_api_request}; static TELEMETRY_SENDER: OnceLock = OnceLock::new(); -/// Returns a `TelemetryLoopHandle` if the telemetry loop is not yet started. -pub fn start_telemetry_loop(quickwit_info: QuickwitTelemetryInfo) -> Option { - let telemetry_sender = - TELEMETRY_SENDER.get_or_init(|| TelemetrySender::from_quickwit_info(quickwit_info)); - let telemetry_loop_handle_opt = telemetry_sender.start_loop(); - // This should not happen... unless telemetry is enabled and you are running tests in parallel - // in the same process. - if telemetry_loop_handle_opt.is_none() { +/// Starts the telemetry loop, and queues the `install` event if the install ID stored in +/// `data_dir` was just created. Returns `None` if telemetry is disabled, if the HTTP client can't +/// be created, or if the loop is already started. +pub async fn start_telemetry_loop( + mut quickwit_info: QuickwitTelemetryInfo, + data_dir: &Path, +) -> Option { + if is_telemetry_disabled() { + info!("telemetry is disabled"); + return None; + } + let http_client = match HttpClient::try_new() { + Ok(http_client) => http_client, + Err(error) => { + warn!( + ?error, + "failed to create the telemetry HTTP client, telemetry is disabled" + ); + return None; + } + }; + let mut is_new_install_opt = None; + let telemetry_sender = TELEMETRY_SENDER.get_or_init(|| { + // Only created once telemetry can be sent, so that a node with telemetry off never writes + // the install ID. + let is_new_install = match load_or_create_install_id(data_dir) { + Ok((install_uuid, is_new_install)) => { + quickwit_info.install_uuid = Some(install_uuid); + is_new_install + } + Err(error) => { + warn!(%error, "failed to load or create the telemetry install ID"); + false + } + }; + is_new_install_opt = Some(is_new_install); + info!("telemetry to {} is enabled", http_client.endpoint()); + TelemetrySender::with_http_client(quickwit_info, http_client) + }); + let Some(is_new_install) = is_new_install_opt else { + // This should not happen... unless telemetry is enabled and you are running tests in + // parallel in the same process. info!("telemetry loop already started. please disable telemetry during tests"); + return None; + }; + let telemetry_loop_handle = telemetry_sender.start_loop(); + if is_new_install { + telemetry_sender.send(TelemetryEvent::Install).await; } - telemetry_loop_handle_opt + Some(telemetry_loop_handle) } -/// Sends a telemetry event to the telemetry relay via HTTP. -/// -/// Telemetry guarantees to send at most 1 request per minute, except for one last request with -/// the pending events when the node shuts down. -/// Each requests can ship at most 10 messages. +/// Queues a telemetry event for the telemetry loop to send to the telemetry relay. Does nothing +/// unless `start_telemetry_loop` started the loop. /// -/// If this methods is called too often, some events will be dropped. +/// Telemetry sends at most 1 request per minute, except for one last request with the pending +/// events when the node shuts down. Each request holds at most 10 events, so some events are +/// dropped if this function is called too often. /// -/// If the http requests fail, the error will be silent. +/// If a request fails, the error is only logged at the debug level. /// -/// We voluntarily use an enum here to make it easier for reader -/// to audit the type of information that is send home. +/// The event is an enum so that readers can audit what information is sent. pub async fn send_telemetry_event(event: TelemetryEvent) { if let Some(telemetry_sender) = TELEMETRY_SENDER.get() { telemetry_sender.send(event).await; } } -/// Returns true once this process started a telemetry loop that sends events. +/// Returns true once `start_telemetry_loop` has started the telemetry loop. It is false before +/// that, so check it on each use instead of caching it. pub fn is_telemetry_enabled() -> bool { - let Some(telemetry_sender) = TELEMETRY_SENDER.get() else { - return false; - }; - !telemetry_sender.inner.is_disabled() + TELEMETRY_SENDER.get().is_some() } /// This environment variable can be set to disable sending telemetry events. pub const DISABLE_TELEMETRY_ENV_KEY: &str = "QW_DISABLE_TELEMETRY"; -/// telemetry is disabled in tests. -#[cfg(test)] -pub fn is_telemetry_disabled() -> bool { - true -} -/// Check to see if telemetry is enabled. -#[cfg(not(test))] +/// Returns true if `QW_DISABLE_TELEMETRY` disables telemetry. pub fn is_telemetry_disabled() -> bool { - let env_value_opt = std::env::var_os(crate::DISABLE_TELEMETRY_ENV_KEY); + let env_value_opt = std::env::var_os(DISABLE_TELEMETRY_ENV_KEY); is_telemetry_disabled_by_env_value(env_value_opt.as_deref()) } diff --git a/quickwit/quickwit-telemetry/src/payload.rs b/quickwit/quickwit-telemetry/src/payload.rs index 2c25af8d599..551c00ad122 100644 --- a/quickwit/quickwit-telemetry/src/payload.rs +++ b/quickwit/quickwit-telemetry/src/payload.rs @@ -29,13 +29,10 @@ pub(crate) const TELEMETRY_SCHEMA_VERSION: u32 = 2; #[derive(Debug, Serialize, Deserialize)] pub struct TelemetryPayload { pub schema_version: u32, - /// Client information. See details in `[ClientInformation]`. pub client_info: ClientInfo, - /// Quickwit information. See details in `[QuickwitInfo]`. pub quickwit_info: QuickwitTelemetryInfo, pub events: Vec, - /// Represents the number of events that where drops due to the - /// combination of the `TELEMETRY_PUSH_COOLDOWN` and `MAX_EVENT_IN_QUEUE`. + /// Number of events dropped since the previous request because the queue was full. pub num_dropped_events: usize, } @@ -43,7 +40,6 @@ pub struct TelemetryPayload { pub struct EventWithTimestamp { /// Unix time in seconds. pub unixtime: u64, - /// Telemetry event. #[serde(flatten)] pub event: TelemetryEvent, } @@ -73,7 +69,7 @@ impl From for EventWithTimestamp { #[serde(rename_all = "snake_case")] pub enum TelemetryEvent { RunCommand, - /// EndCommand (with the return code). + /// Event sent when the node stops, with its exit code. EndCommand { return_code: i32, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -178,7 +174,9 @@ fn hashed_host_username() -> String { let hostname = hostname::get() .map(|hostname| hostname.to_string_lossy().to_string()) .unwrap_or_default(); - let username = username::get_user_name().unwrap_or_default(); + // What the `username` crate used by releases 0.2 to 0.8 returns on Linux and macOS, so that the + // hash stays the same for nodes that upgrade. + let username = env::var("LOGNAME").unwrap_or_default(); let hashed_value = format!("{hostname}:{username}"); let digest = md5::compute(hashed_value.as_bytes()); format!("{digest:x}") diff --git a/quickwit/quickwit-telemetry/src/sender.rs b/quickwit/quickwit-telemetry/src/sender.rs index 52df85d38da..ae1a33a2599 100644 --- a/quickwit/quickwit-telemetry/src/sender.rs +++ b/quickwit/quickwit-telemetry/src/sender.rs @@ -14,16 +14,13 @@ use std::mem; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinHandle; -use tracing::{info, warn}; use crate::clock::Clock; -use crate::is_telemetry_disabled; use crate::payload::{ ClientInfo, EventWithTimestamp, QuickwitTelemetryInfo, TELEMETRY_SCHEMA_VERSION, TelemetryEvent, TelemetryPayload, @@ -35,7 +32,7 @@ const TELEMETRY_PUSH_COOLDOWN: Duration = Duration::from_secs(60); /// Interval at which to send telemetry `Running` event. const TELEMETRY_RUNNING_EVENT_INTERVAL: Duration = - Duration::from_secs(if cfg!(test) { 3 } else { 60 * 60 * 12 }); // 12h + Duration::from_secs(if cfg!(test) { 3 } else { 60 * 60 * 12 }); /// Upon termination of the program, we send one last telemetry request with pending events. /// This duration is the amount of time we wait for at most to send that last telemetry request. @@ -50,19 +47,7 @@ struct EventsState { } impl EventsState { - fn drain_events(&mut self) -> EventsState { - mem::replace( - self, - EventsState { - events: Vec::new(), - num_dropped_events: 0, - }, - ) - } - - /// Adds an event. - /// If the queue is already saturated, (ie. it has reached the len `MAX_NUM_EVENTS_IN_QUEUE`) - // Returns true iff it was the first event in the queue. + /// Returns true if the event was queued into an empty queue. fn push_event(&mut self, mut event: TelemetryEvent) -> bool { if self.events.len() >= MAX_NUM_EVENTS_IN_QUEUE { self.num_dropped_events += 1; @@ -89,6 +74,7 @@ impl EventsState { struct Events { state: RwLock, + /// Wakes the telemetry loop up when an event is queued into an empty queue. items_available_tx: Sender<()>, items_available_rx: RwLock>, } @@ -105,231 +91,143 @@ impl Default for Events { } impl Events { - /// Wait for events to be available (if there are pending events, then do not wait) - /// and then send them to the ingest API server. - async fn drain_events(&self) -> EventsState { + async fn wait_for_events(&self) { self.items_available_rx.write().await.recv().await; - self.state.write().await.drain_events() + } + + async fn take_events(&self) -> EventsState { + mem::take(&mut *self.state.write().await) } async fn push_event(&self, event: TelemetryEvent) { let is_first_event = self.state.write().await.push_event(event); if is_first_event { - let _ = self.items_available_tx.send(()).await; + // A full channel means that a wake-up is already pending. + let _ = self.items_available_tx.try_send(()); } } } -pub(crate) struct Inner { - sink: Option>, +struct Inner { + sink: Box, client_info: ClientInfo, quickwit_info: QuickwitTelemetryInfo, - /// This channel is just used to signal there are new items available. events: Events, clock: Clock, - is_started: AtomicBool, } impl Inner { - pub fn is_disabled(&self) -> bool { - self.sink.is_none() - } - - fn create_telemetry_payload(&self, events_state: EventsState) -> TelemetryPayload { - TelemetryPayload { + /// Sends the queued events, if any. + async fn send_queued_events(&self) { + let events_state = self.events.take_events().await; + if events_state.events.is_empty() { + return; + } + let payload = TelemetryPayload { schema_version: TELEMETRY_SCHEMA_VERSION, client_info: self.client_info.clone(), quickwit_info: self.quickwit_info.clone(), events: events_state.events, num_dropped_events: events_state.num_dropped_events, - } - } - - /// Wait for events to be available (if there are pending events, then do not wait) - /// and then send them to the ingest API server. - /// - /// If the requests fails, it fails silently. - async fn send_pending_events(&self) { - if let Some(sink) = self.sink.as_ref() { - let events_state = self.events.drain_events().await; - let payload = self.create_telemetry_payload(events_state); - sink.send_payload(payload).await; - } - } - - /// Sends the queued events, if any, without waiting for new ones. - async fn send_remaining_events(&self) { - if let Some(sink) = self.sink.as_ref() { - let events_state = self.events.state.write().await.drain_events(); - if !events_state.events.is_empty() { - let payload = self.create_telemetry_payload(events_state); - sink.send_payload(payload).await; - } - } - } - - async fn send(&self, event: TelemetryEvent) { - if self.is_disabled() { - return; - } - self.events.push_event(event).await; + }; + self.sink.send_payload(payload).await; } } pub struct TelemetrySender { - pub(crate) inner: Arc, + inner: Arc, } -pub enum TelemetryLoopHandle { - NoLoop, - WithLoop { - join_handle: JoinHandle<()>, - terminate_command_tx: oneshot::Sender<()>, - }, +pub struct TelemetryLoopHandle { + join_handle: JoinHandle<()>, + terminate_command_tx: oneshot::Sender<()>, } impl TelemetryLoopHandle { - /// Terminate telemetry will exit the telemetry loop - /// and possibly send the last request, possibly ignoring the - /// telemetry cooldown. + /// Stops the telemetry loop, which sends the queued events without waiting for the cooldown, + /// and waits for it at most `LAST_REQUEST_TIMEOUT`. pub async fn terminate_telemetry(self) { - if let Self::WithLoop { - join_handle, - terminate_command_tx, - } = self - { - let _ = terminate_command_tx.send(()); - let _ = tokio::time::timeout(LAST_REQUEST_TIMEOUT, join_handle).await; - } + let _ = self.terminate_command_tx.send(()); + let _ = tokio::time::timeout(LAST_REQUEST_TIMEOUT, self.join_handle).await; } } impl TelemetrySender { - pub fn from_quickwit_info(quickwit_info: QuickwitTelemetryInfo) -> Self { - let http_client = create_http_client(); - TelemetrySender::new( + pub(crate) fn with_http_client( + quickwit_info: QuickwitTelemetryInfo, + http_client: HttpClient, + ) -> Self { + Self::new( quickwit_info, http_client, - Clock::cooldown(TELEMETRY_PUSH_COOLDOWN), + Clock::Cooldown(TELEMETRY_PUSH_COOLDOWN), ) } - fn new( - quickwit_info: QuickwitTelemetryInfo, - sink_opt: Option, - clock: Clock, - ) -> Self { - let sink_opt: Option> = if let Some(sink) = sink_opt { - Some(Box::new(sink)) - } else { - None - }; + fn new(quickwit_info: QuickwitTelemetryInfo, sink: impl Sink, clock: Clock) -> Self { Self { inner: Arc::new(Inner { - sink: sink_opt, + sink: Box::new(sink), client_info: ClientInfo::default(), quickwit_info, events: Events::default(), clock, - is_started: AtomicBool::new(false), }), } } - /// Returns `None` if the loop is already started. - pub fn start_loop(&self) -> Option { - if self.inner.is_disabled() { - return Some(TelemetryLoopHandle::NoLoop); - } - if self.inner.is_started.swap(true, Ordering::SeqCst) { - return None; - } + /// Starts the loop that sends the queued events. Must be called at most once. + pub(crate) fn start_loop(&self) -> TelemetryLoopHandle { let (terminate_command_tx, mut terminate_command_rx) = oneshot::channel(); let inner = self.inner.clone(); start_monitor_if_server_running_task(inner.clone()); - let join_handle = tokio::task::spawn(async move { + let join_handle = tokio::spawn(async move { loop { let quit_loop = tokio::select! { - _ = (&mut terminate_command_rx) => { true } - _ = inner.clock.tick() => { false } + _ = &mut terminate_command_rx => true, + _ = inner.events.wait_for_events() => false, }; + inner.send_queued_events().await; if quit_loop { - inner.send_remaining_events().await; break; } - inner.send_pending_events().await; + let quit_loop = tokio::select! { + _ = &mut terminate_command_rx => true, + _ = inner.clock.tick() => false, + }; + if quit_loop { + inner.send_queued_events().await; + break; + } } }); - Some(TelemetryLoopHandle::WithLoop { + TelemetryLoopHandle { join_handle, terminate_command_tx, - }) + } } pub async fn send(&self, event: TelemetryEvent) { - self.inner.send(event).await; + self.inner.events.push_event(event).await; } } -fn start_monitor_if_server_running_task(telemetry_sender: Arc) { +fn start_monitor_if_server_running_task(inner: Arc) { let mut clock = tokio::time::interval(TELEMETRY_RUNNING_EVENT_INTERVAL); tokio::spawn(async move { // Drop the first immediate tick. clock.tick().await; loop { clock.tick().await; - telemetry_sender.send(TelemetryEvent::running()).await; + inner.events.push_event(TelemetryEvent::running()).await; } }); } -fn create_http_client() -> Option { - if is_telemetry_disabled() { - info!("telemetry is disabled"); - return None; - } - let client = match HttpClient::try_new() { - Ok(client) => client, - Err(error) => { - warn!(%error, "failed to create the telemetry HTTP client, telemetry is disabled"); - return None; - } - }; - info!("telemetry to {} is enabled", client.endpoint()); - Some(client) -} - #[cfg(test)] mod tests { - - use std::env; - use super::*; - #[ignore] - #[tokio::test] - async fn test_enabling_and_disabling_telemetry() { - // SAFETY: this test may not be entirely sound if not run with nextest or --test-threads=1 - // as this is only a test, and it would be extremly inconvenient to run it in a different - // way, we are keeping it that way - - // We group the two in a single test to ensure it happens on the same thread. - unsafe { env::set_var(crate::DISABLE_TELEMETRY_ENV_KEY, "") }; - assert_eq!( - TelemetrySender::from_quickwit_info(QuickwitTelemetryInfo::default()) - .inner - .is_disabled(), - true - ); - unsafe { env::remove_var(crate::DISABLE_TELEMETRY_ENV_KEY) }; - assert_eq!( - TelemetrySender::from_quickwit_info(QuickwitTelemetryInfo::default()) - .inner - .is_disabled(), - false - ); - } - #[test] fn test_push_event_replaces_ui_page_load_when_queue_is_full() { let mut events_state = EventsState::default(); @@ -346,28 +244,12 @@ mod tests { )); } - #[tokio::test] - async fn test_telemetry_no_wait_for_first_event() { - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (_clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); - telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; - let payload_opt = rx.recv().await; - assert!(payload_opt.is_some()); - let payload = payload_opt.unwrap(); - assert_eq!(payload.events.len(), 1); - loop_handler.terminate_telemetry().await; - } - #[tokio::test] async fn test_telemetry_two_events() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); + let (clock_btn, clock) = Clock::manual(); + let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), tx, clock); + let loop_handler = telemetry_sender.start_loop(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -385,10 +267,9 @@ mod tests { #[tokio::test] async fn test_telemetry_uptime_events() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); + let (clock_btn, clock) = Clock::manual(); + let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), tx, clock); + let loop_handler = telemetry_sender.start_loop(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -410,10 +291,9 @@ mod tests { #[tokio::test] async fn test_telemetry_cooldown_observed() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); + let (clock_btn, clock) = Clock::manual(); + let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), tx, clock); + let loop_handler = telemetry_sender.start_loop(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; { let payload = rx.recv().await.unwrap(); @@ -437,10 +317,9 @@ mod tests { #[tokio::test] async fn test_terminate_telemetry_sends_pending_events() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (_clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); + let (_clock_btn, clock) = Clock::manual(); + let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), tx, clock); + let loop_handler = telemetry_sender.start_loop(); telemetry_sender.send(TelemetryEvent::UiIndexPageLoad).await; let payload = rx.recv().await.unwrap(); assert_eq!(payload.events.len(), 1); @@ -454,19 +333,26 @@ mod tests { &payload.events[0].event, &TelemetryEvent::EndCommand { .. } )); + tokio::time::timeout( + Duration::from_secs(1), + telemetry_sender.send(TelemetryEvent::UiIndexPageLoad), + ) + .await + .unwrap(); } #[tokio::test(start_paused = true)] async fn test_terminate_telemetry_does_not_wait_without_pending_events() { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let (_clock_btn, clock) = Clock::manual().await; - let telemetry_sender = - TelemetrySender::new(QuickwitTelemetryInfo::default(), Some(tx), clock); - let loop_handler = telemetry_sender.start_loop().unwrap(); + let (clock_btn, clock) = Clock::manual(); + let telemetry_sender = TelemetrySender::new(QuickwitTelemetryInfo::default(), tx, clock); + let loop_handler = telemetry_sender.start_loop(); telemetry_sender .send(TelemetryEvent::end_command(0i32)) .await; rx.recv().await.unwrap(); + clock_btn.tick().await; + tokio::task::yield_now().await; let terminate_start = tokio::time::Instant::now(); loop_handler.terminate_telemetry().await; diff --git a/quickwit/quickwit-telemetry/src/sink.rs b/quickwit/quickwit-telemetry/src/sink.rs index 844b9b4e2ec..efaabbe2457 100644 --- a/quickwit/quickwit-telemetry/src/sink.rs +++ b/quickwit/quickwit-telemetry/src/sink.rs @@ -15,13 +15,13 @@ use std::time::Duration; use async_trait::async_trait; -use reqwest::Client; use reqwest::redirect::Policy; +use reqwest::{Client, Response}; use tokio::sync::mpsc::UnboundedSender; +use tracing::debug; use crate::payload::TelemetryPayload; -/// Telemetry ingest API URL const DEFAULT_TELEMETRY_INGEST_API_URL: &str = "https://telemetry.quickwit.io/"; fn telemetry_ingest_api_url() -> String { @@ -68,7 +68,17 @@ impl Sink for UnboundedSender { #[async_trait] impl Sink for HttpClient { async fn send_payload(&self, payload: TelemetryPayload) { - // Note that we swallow the error if any - let _ = self.client.post(&self.endpoint).json(&payload).send().await; + // Telemetry is best effort, so failures, including payloads that the relay rejects, are + // only logged at the debug level. + let send_result = self + .client + .post(&self.endpoint) + .json(&payload) + .send() + .await + .and_then(Response::error_for_status); + if let Err(error) = send_result { + debug!(?error, "failed to send telemetry request"); + } } }