Cold-start improvements — integration (combined testing)

bottlecap/Cargo.toml

@@ -22,13 +22,13 @@ lazy_static = { version = "1.5", default-features = false }
 log = { version = "0.4", default-features = false }
 mime = { version = "0.3", default-features = false }
 multipart = { version = "0.18", default-features = false, features = ["server"] }
-nix = { version = "0.26", default-features = false, features = ["feature", "fs"] }
+nix = { version = "0.29", default-features = false, features = ["feature", "fs"] }
 ordered_hash_map = { version = "0.4", default-features = false }
 regex = { version = "1.10", default-features = false }
 reqwest = { version = "0.12.11", features = ["json", "http2"], default-features = false }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
-thiserror = { version = "1.0", default-features = false }
+thiserror = { version = "2.0", default-features = false }
 # Transitive dependency (pulled in via cookie). Pinned to >=0.3.47 so cargo audit / CI passes (RUSTSEC-2026-0009).
 time = { version = "0.3.47", default-features = false }
 tokio = { version = "1.47", default-features = false, features = ["macros", "rt-multi-thread", "time"] }
@@ -46,7 +46,7 @@ rustls-webpki = { version = "0.103.13", default-features = false }
 rustls-pemfile = { version = "2.0", default-features = false, features = ["std"] }
 rustls-pki-types = { version = "1.0", default-features = false }
 hyper-rustls = { version = "0.27.7", default-features = false }
-rand = { version = "0.8", default-features = false }
+rand = { version = "0.9", default-features = false }
 prost = { version = "0.14", default-features = false }
 tonic = { version = "0.14", features = ["transport", "codegen", "server", "channel", "router"], default-features = false }
 tonic-types = { version = "0.14", default-features = false }
@@ -55,7 +55,7 @@ futures = { version = "0.3.31", default-features = false }
 serde-aux = { version = "4.7", default-features = false }
 serde_html_form = { version = "0.2", default-features = false }
 opentelemetry-proto = { version = "0.31.0", features = ["trace", "with-serde", "gen-tonic"] }
-opentelemetry-semantic-conventions = { version = "0.30", features = ["semconv_experimental"] }
+opentelemetry-semantic-conventions = { version = "0.31", features = ["semconv_experimental"] }
 # Pinned to <0.8.3: version 0.8.3 upgraded to openssl-probe 0.2.x which scans all cert
 # directories and parses ~200 individual cert files on Lambda instead of loading a single
 # bundle file, adding ~45ms to each reqwest::Client::build() call.
@@ -129,7 +129,19 @@ tikv-jemallocator = "0.5"
 
 [features]
 default = [
-    "reqwest/rustls-tls-native-roots",
+    # Use webpki-roots (compiled-in Mozilla CA bundle) rather than
+    # rustls-tls-native-roots for the non-FIPS reqwest clients. native-roots
+    # calls rustls_native_certs::load_native_certs() — a filesystem cert-bundle
+    # scan/parse — on *every* reqwest::Client::build(), and bottlecap builds two
+    # reqwest clients during cold-start init (the register client in
+    # bin/bottlecap/main.rs and the shared flush client in src/http.rs).
+    # webpki-roots removes that per-build scan. Trade-off: webpki trusts only the
+    # public Mozilla roots, so private/internal OS-installed CAs are no longer
+    # picked up implicitly; users relying on those must point DD_PROXY_HTTPS at a
+    # trusted intercept or supply the CA via tls_cert_file (which still layers on
+    # top via add_root_certificate in src/http.rs). FIPS builds keep native-roots
+    # (see the `fips` feature) and are unaffected.
+    "reqwest/rustls-tls-webpki-roots",
     "datadog-fips/default",
     "datadog-agent-config/https",
     "libdd-common/https",

bottlecap/src/appsec/mod.rs

@@ -1,9 +1,37 @@
 use std::env;
+use std::sync::Arc;
 
+use tokio::sync::{Mutex, OnceCell};
+use tracing::error;
+
+use crate::appsec::processor::{Error as AppSecError, Processor};
 use crate::config::Config;
 
 pub mod processor;
 
+/// A [`Processor`] shared across the trace agent and the runtime API proxy.
+///
+/// Wrapped in a [`Mutex`] (and an [`Arc`] so the value stays [`Send`]) because
+/// the WAF context buffer is mutated from multiple asynchronous tasks.
+pub type SharedProcessor = Arc<Mutex<Processor>>;
+
+/// A handle to a [`SharedProcessor`] whose construction is deferred off the
+/// initialization critical path.
+///
+/// Building the WAF (zstd-decompressing and JSON-parsing the ruleset, then
+/// compiling it through `libddwaf`) costs tens of milliseconds and is only
+/// needed once the first request payload is evaluated — which is strictly after
+/// the first `/next`. Rather than block init on that work, consumers hold this
+/// awaitable handle and resolve it (via [`resolve`]) at the point where they
+/// actually need the WAF.
+///
+/// The outer [`Option`] (see [`defer_processor`]) distinguishes the
+/// feature-disabled case (no handle at all) from the enabled case. The inner
+/// [`Option`] distinguishes a successfully built processor (`Some`) from a build
+/// that failed (`None`), in which case the feature is treated as a no-op exactly
+/// as it would have been with the previous eager construction.
+pub type DeferredProcessor = Arc<OnceCell<Option<SharedProcessor>>>;
+
 /// Determines whether the Serverless App & API Protection features are enabled.
 #[must_use]
 pub const fn is_enabled(cfg: &Config) -> bool {
@@ -16,3 +44,72 @@ pub const fn is_enabled(cfg: &Config) -> bool {
 pub fn is_standalone() -> bool {
     env::var("DD_APM_TRACING_ENABLED").is_ok_and(|s| s.to_lowercase() == "true")
 }
+
+/// Prepares a [`DeferredProcessor`] for the App & API Protection feature without
+/// blocking the caller on the (CPU-bound) WAF build.
+///
+/// Returns [`None`] immediately when the feature is disabled, preserving the
+/// cheap, synchronous disabled-by-default path. When enabled, a background task
+/// is spawned to build the processor off the critical path, and a handle that
+/// resolves to it is returned right away. Consumers call [`resolve`] where they
+/// use the WAF; if a request arrives before the background build has finished,
+/// that call simply awaits the in-flight build (or kicks it off itself).
+#[must_use]
+pub fn defer_processor(cfg: &Arc<Config>) -> Option<DeferredProcessor> {
+    if !is_enabled(cfg) {
+        // Feature disabled: nothing to build, and nothing to await.
+        return None;
+    }
+
+    let cell: DeferredProcessor = Arc::new(OnceCell::new());
+
+    // Kick the build off the synchronous init path. The first consumer to call
+    // `resolve` will await whatever this task produces (or, if it somehow races
+    // ahead, run an equivalent build itself via `get_or_init`).
+    let background_cell = Arc::clone(&cell);
+    let background_cfg = Arc::clone(cfg);
+    tokio::spawn(async move {
+        let _ = background_cell
+            .get_or_init(|| build_processor(background_cfg))
+            .await;
+    });
+
+    Some(cell)
+}
+
+/// Resolves a [`DeferredProcessor`] to the underlying [`SharedProcessor`], if the
+/// WAF was built successfully.
+///
+/// Awaits the background build started by [`defer_processor`] when it is still in
+/// flight; if no build is running yet, it starts one (off-thread, CPU-bound work
+/// runs on the blocking pool). Subsequent calls return immediately.
+pub async fn resolve(handle: &DeferredProcessor, cfg: &Arc<Config>) -> Option<SharedProcessor> {
+    handle
+        .get_or_init(|| build_processor(Arc::clone(cfg)))
+        .await
+        .clone()
+}
+
+/// Builds the App & API Protection [`Processor`] on the blocking thread pool.
+///
+/// The WAF build is CPU-bound (ruleset decompression, JSON parsing, and WAF
+/// compilation), so it is offloaded with [`tokio::task::spawn_blocking`] to keep
+/// it off the async worker threads. Returns [`None`] (logging at the appropriate
+/// level) when the feature is disabled or the build fails, matching the previous
+/// "feature is silently a no-op" behaviour.
+async fn build_processor(cfg: Arc<Config>) -> Option<SharedProcessor> {
+    match tokio::task::spawn_blocking(move || Processor::new(&cfg)).await {
+        Ok(Ok(processor)) => Some(Arc::new(Mutex::new(processor))),
+        Ok(Err(AppSecError::FeatureDisabled)) => None,
+        Ok(Err(e)) => {
+            error!(
+                "AAP | error creating App & API Protection processor, the feature will be disabled: {e}"
+            );
+            None
+        }
+        Err(e) => {
+            error!("AAP | App & API Protection processor build task failed to join: {e}");
+            None
+        }
+    }
+}