diff --git a/.github/workflows/gateway-ci.yml b/.github/workflows/gateway-ci.yml index b43f7656ef..c43f14f1ce 100644 --- a/.github/workflows/gateway-ci.yml +++ b/.github/workflows/gateway-ci.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Gateway-only CI: gateway changes never build the fluss-rust workspace and vice versa. +# Gateway CI also follows the native Rust client sources used by its configuration adapter. # `uses:` step inputs are relative to the repository root. name: Gateway CI @@ -25,12 +25,16 @@ on: - main paths: - 'fluss-gateway/**' + - 'fluss-rust/crates/fluss/**' + - 'fluss-rpc/src/main/proto/**' - '.github/workflows/gateway-ci.yml' pull_request: branches: - main paths: - 'fluss-gateway/**' + - 'fluss-rust/crates/fluss/**' + - 'fluss-rpc/src/main/proto/**' - '.github/workflows/gateway-ci.yml' workflow_dispatch: @@ -55,6 +59,14 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Install protobuf compiler + run: brew install protobuf + if: runner.os == 'macOS' + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + if: runner.os == 'Linux' + - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -91,6 +103,9 @@ jobs: - name: Install the MSRV toolchain run: rustup toolchain install 1.88.0 --profile minimal + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -115,6 +130,9 @@ jobs: with: tool: cargo-deny@0.14.22 + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Check dependency licenses (Apache-compatible) run: cargo deny --locked check licenses @@ -124,7 +142,7 @@ jobs: # The inventory ships with the source release, so a stale one is worse than none. - name: Dependency inventory drift check run: | - cargo deny --locked list -f tsv -t 0.6 > DEPENDENCIES.rust.tsv + cargo deny --locked list -f tsv -t 0.6 | sed 's/[[:space:]]*$//' > DEPENDENCIES.rust.tsv git diff --exit-code DEPENDENCIES.rust.tsv - name: Rust Cache diff --git a/fluss-gateway/Cargo.lock b/fluss-gateway/Cargo.lock index 0e57278e5e..4c08f394a8 100644 --- a/fluss-gateway/Cargo.lock +++ b/fluss-gateway/Cargo.lock @@ -2,6 +2,20 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +25,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -47,7 +70,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -58,7 +81,244 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -67,6 +327,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.8.9" @@ -113,36 +379,140 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "clap" version = "4.6.6" @@ -190,118 +560,389 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "crossbeam-epoch" -version = "0.9.20" +name = "const-random" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ - "crossbeam-utils", + "const-random-macro", ] [[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "displaydoc" -version = "0.2.7" +name = "const-random-macro" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "errno" -version = "0.3.14" +name = "cpufeatures" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", - "windows-sys", ] [[package]] -name = "fastrand" -version = "2.5.0" +name = "crc32c" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] [[package]] -name = "fluss-gateway" -version = "1.0.0" +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ - "axum", - "clap", - "futures-util", - "http-body-util", - "hyper", - "hyper-util", - "libc", - "log", - "metrics", - "metrics-exporter-prometheus", - "reqwest", - "serde", - "serde_json", - "serde_path_to_error", - "serde_yaml_ng", - "tempfile", - "tokio", - "tokio-util", - "tower", - "utoipa", - "utoipa-axum", - "uuid", + "crossbeam-utils", ] [[package]] -name = "foldhash" -version = "0.2.0" +name = "crossbeam-utils" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "crunchy" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "percent-encoding", + "generic-array", + "typenum", ] [[package]] -name = "futures-channel" -version = "0.3.34" +name = "csv" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ - "futures-core", + "csv-core", + "itoa", + "ryu", + "serde_core", ] [[package]] -name = "futures-core" -version = "0.3.34" +name = "csv-core" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] [[package]] -name = "futures-macro" -version = "0.3.34" +name = "dashmap" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ - "proc-macro2", + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.1", + "rustc_version", +] + +[[package]] +name = "fluss-gateway" +version = "1.0.0" +dependencies = [ + "axum", + "clap", + "fluss-rs", + "futures-util", + "http-body-util", + "hyper", + "hyper-util", + "libc", + "log", + "metrics", + "metrics-exporter-prometheus", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "serde_yaml_ng", + "tempfile", + "tokio", + "tokio-util", + "tower", + "utoipa", + "utoipa-axum", + "uuid", +] + +[[package]] +name = "fluss-rs" +version = "1.0.0" +dependencies = [ + "arrow", + "arrow-schema", + "bigdecimal", + "bitvec", + "byteorder", + "bytes", + "clap", + "crc32c", + "dashmap", + "delegate", + "futures", + "jiff", + "linked-hash-map", + "log", + "metrics", + "opendal", + "ordered-float", + "parking_lot", + "parse-display", + "prost", + "prost-build", + "rand 0.9.5", + "scopeguard", + "serde", + "serde_json", + "snafu", + "strum", + "strum_macros", + "tempfile", + "thiserror 1.0.69", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", "quote", "syn 3.0.3", ] @@ -324,13 +965,40 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -350,8 +1018,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", ] [[package]] @@ -360,7 +1070,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -441,13 +1151,29 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -464,6 +1190,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -546,65 +1296,196 @@ dependencies = [ ] [[package]] -name = "idna" -version = "1.1.0" +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "getrandom 0.4.3", + "libc", ] [[package]] -name = "idna_adapter" -version = "1.2.1" +name = "js-sys" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ - "icu_normalizer", - "icu_properties", + "cfg-if", + "futures-util", + "wasm-bindgen", ] [[package]] -name = "indexmap" -version = "2.14.0" +name = "lexical-core" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", ] [[package]] -name = "ipnet" -version = "2.12.1" +name = "lexical-parse-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "lexical-parse-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] [[package]] -name = "itoa" -version = "1.0.18" +name = "lexical-util" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] -name = "js-sys" -version = "0.3.104" +name = "lexical-write-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", ] [[package]] @@ -613,6 +1494,18 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -625,11 +1518,38 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +dependencies = [ + "value-bag", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] [[package]] name = "matchit" @@ -637,6 +1557,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -659,12 +1589,12 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" dependencies = [ - "base64", + "base64 0.22.1", "indexmap", "metrics", "metrics-util", "quanta", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -678,7 +1608,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand", + "rand 0.9.5", "rand_xoshiro", "rapidhash", "sketches-ddsketch", @@ -698,7 +1628,61 @@ checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", ] [[package]] @@ -713,6 +1697,92 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opendal" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +dependencies = [ + "anyhow", + "backon", + "base64 0.22.1", + "bytes", + "futures", + "getrandom 0.2.17", + "http", + "http-body", + "jiff", + "log", + "md-5", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", + "rand 0.8.7", + "serde", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parse-display" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287d8d3ebdce117b8539f59411e4ed9ec226e0a4153c7f55495c6070d68e6f72" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc048687be30d79502dea2f623d052f3a074012c6eac41726b7ab17213616b1" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.119", +] + [[package]] name = "paste" version = "1.0.15" @@ -725,18 +1795,44 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "portable-atomic" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -756,27 +1852,154 @@ dependencies = [ ] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ - "unicode-ident", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", ] [[package]] -name = "quanta" -version = "0.12.6" +name = "quinn-udp" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ - "crossbeam-utils", + "cfg_aliases", "libc", "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", + "socket2", + "tracing", + "windows-sys 0.61.2", ] [[package]] @@ -800,6 +2023,22 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_core 0.6.4", + "serde", +] + [[package]] name = "rand" version = "0.9.5" @@ -807,7 +2046,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -817,7 +2067,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "serde", ] [[package]] @@ -829,13 +2088,28 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" dependencies = [ - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -853,7 +2127,16 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.13.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", ] [[package]] @@ -891,30 +2174,68 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", ] [[package]] @@ -923,11 +2244,46 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -942,6 +2298,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -973,103 +2341,272 @@ dependencies = [ ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "serde_fmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e497af288b3b95d067a23a4f749f2861121ffcb2f6d8379310dcda040c345ed" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.119", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "sval" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "ec4a2a7d92fa86fcc6222e4c3845f8486cff899d9db32480b26c91a5dbf2e22d" + +[[package]] +name = "sval_buffer" +version = "2.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4324db9ac500c609d659b752edf9c8abbf2233f8afd61a503fd6f88ed625032" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "sval", + "sval_ref", + "zerocopy", ] [[package]] -name = "serde_path_to_error" -version = "0.1.20" +name = "sval_dynamic" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +checksum = "4046add0eecf55e680b9e207edf5fc7737b18a1d950db363d97e7f1b2d7c629c" dependencies = [ - "itoa", - "serde", - "serde_core", + "sval", ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "sval_fmt" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "911a3486b5984a0a4f25edefcf2c2dba23654c29f63e75493b671d338bf24243" dependencies = [ - "form_urlencoded", "itoa", "ryu", - "serde", + "sval", ] [[package]] -name = "serde_yaml_ng" -version = "0.10.0" +name = "sval_json" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +checksum = "da53aae7c737b5b5f1be4bcb0ff20e057bf6b2ee4e9d025560075c5830d09f95" dependencies = [ - "indexmap", "itoa", "ryu", - "serde", - "unsafe-libyaml", + "sval", ] [[package]] -name = "signal-hook-registry" -version = "1.4.8" +name = "sval_nested" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "df24df43cbdc4bb8c9f5ed19d0d57dc8f60a1a4259cdce52d597fe774ad3a71f" dependencies = [ - "errno", - "libc", + "sval", + "sval_buffer", + "sval_ref", ] [[package]] -name = "sketches-ddsketch" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" +name = "sval_ref" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "2bebc17f0f1fad060e57b778728d41ef87627e9111a6365d7463472cb58fc1b3" dependencies = [ - "libc", - "windows-sys", + "sval", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strsim" -version = "0.11.1" +name = "sval_serde" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "9f26fe3f6a68b40e6c8d654ea48c00e4316272fddf68c80493714c1b034ae70b" +dependencies = [ + "serde_core", + "sval", + "sval_nested", +] [[package]] name = "syn" @@ -1113,6 +2650,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1123,7 +2666,16 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", ] [[package]] @@ -1132,7 +2684,18 @@ version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1146,6 +2709,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1156,6 +2728,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -1165,11 +2752,12 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1183,6 +2771,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -1217,7 +2815,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -1266,6 +2864,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1278,6 +2894,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1347,9 +2969,52 @@ checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] +[[package]] +name = "value-bag" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068e763e8279de7ab94b6afebded2cb701678af094feb1c12ccb061b4783c1be" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417d6197dd0ee696783d6be4276ac6ea74b985e00024c85ccfb37aff4f2bed82" +dependencies = [ + "erased-serde", + "serde_core", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f7251ecde2c9ed431bbe0659853e7991753447447bbf1ae59d8b31c578d4e" +dependencies = [ + "sval", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" @@ -1429,6 +3094,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.104" @@ -1439,6 +3117,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1461,12 +3158,74 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1476,6 +3235,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -1488,6 +3311,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yoke" version = "0.8.3" @@ -1552,6 +3384,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" @@ -1590,3 +3428,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/fluss-gateway/Cargo.toml b/fluss-gateway/Cargo.toml index b2df3ba10b..f5457f0003 100644 --- a/fluss-gateway/Cargo.toml +++ b/fluss-gateway/Cargo.toml @@ -51,6 +51,8 @@ axum = { version = "0.8", default-features = false, features = ["http1", "matche clap = { version = "4.5.37", features = ["derive"] } # Only FutureExt/join_all are used; avoid the larger futures facade crate. futures-util = "0.3" +# Reuse the native client configuration defaults and validation rather than copying them here. +fluss = { package = "fluss-rs", path = "../fluss-rust/crates/fluss" } # The listeners follow axum's official serve-with-hyper example: axum::serve does not expose the # hyper builder that the header read timeout lives on. The plain http1 builder (not auto) has no # HTTP/2 preface sniffing, so the header timer covers a connection from its first byte. diff --git a/fluss-gateway/DEPENDENCIES.rust.tsv b/fluss-gateway/DEPENDENCIES.rust.tsv index 5d3a4da26b..ec9bc96c26 100644 --- a/fluss-gateway/DEPENDENCIES.rust.tsv +++ b/fluss-gateway/DEPENDENCIES.rust.tsv @@ -1,130 +1,310 @@ -crate Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -aho-corasick@1.1.5 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -atomic-waker@1.1.2 X X -axum@0.8.9 X -axum-core@0.5.6 X -base64@0.22.1 X X -bitflags@2.13.1 X X -bumpalo@3.20.3 X X -bytes@1.12.1 X -cfg-if@1.0.4 X X -clap@4.6.6 X X -clap_builder@4.6.6 X X -clap_derive@4.6.4 X X -clap_lex@1.1.0 X X -colorchoice@1.0.5 X X -crossbeam-epoch@0.9.20 X X -crossbeam-utils@0.8.22 X X -equivalent@1.0.2 X X -errno@0.3.14 X X -fluss-gateway@1.0.0 X -foldhash@0.2.0 X -futures-channel@0.3.34 X X -futures-core@0.3.34 X X -futures-macro@0.3.34 X X -futures-sink@0.3.34 X X -futures-task@0.3.34 X X -futures-util@0.3.34 X X -getrandom@0.3.4 X X -getrandom@0.4.3 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.5.0 X X -http-body@1.1.0 X -http-body-util@0.1.5 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.11.0 X -hyper-util@0.1.20 X -indexmap@2.14.0 X X -ipnet@2.12.1 X X -is_terminal_polyfill@1.70.2 X X -itoa@1.0.18 X X -js-sys@0.3.104 X X -libc@0.2.189 X X -log@0.4.33 X X -matchit@0.8.4 X X -memchr@2.8.3 X X -metrics@0.24.6 X -metrics-exporter-prometheus@0.17.2 X -metrics-util@0.20.4 X -mime@0.3.17 X X -mio@1.2.2 X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -portable-atomic@1.15.0 X X -ppv-lite86@0.2.21 X X -proc-macro2@1.0.107 X X -quanta@0.12.6 X -quote@1.0.47 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.5 X X -rand_chacha@0.9.0 X X -rand_core@0.9.5 X X -rand_xoshiro@0.7.0 X X -rapidhash@4.5.1 X X -raw-cpuid@11.6.0 X -regex@1.13.1 X X -regex-automata@0.4.18 X X -regex-syntax@0.8.11 X X -rustversion@1.0.23 X X -ryu@1.0.23 X X -serde@1.0.229 X X -serde_core@1.0.229 X X -serde_derive@1.0.229 X X -serde_json@1.0.151 X X -serde_path_to_error@0.1.20 X X -serde_yaml_ng@0.10.0 X -signal-hook-registry@1.4.8 X X -sketches-ddsketch@0.3.1 X -slab@0.4.12 X -smallvec@1.15.2 X X -socket2@0.6.5 X X -strsim@0.11.1 X -syn@2.0.119 X X -syn@3.0.3 X X -sync_wrapper@1.0.2 X -thiserror@2.0.20 X X -thiserror-impl@2.0.20 X X -tokio@1.53.1 X -tokio-macros@2.7.2 X -tokio-util@0.7.19 X -tower@0.5.3 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-core@0.1.36 X -try-lock@0.2.5 X -unicode-ident@1.0.24 X X X -unsafe-libyaml@0.2.11 X -utf8parse@0.2.2 X X -utoipa@5.5.0 X X -utoipa-axum@0.2.0 X X -utoipa-gen@5.5.0 X X -uuid@1.24.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.1+wasi-0.2.4 X X X -wasm-bindgen@0.2.127 X X -wasm-bindgen-macro@0.2.127 X X -wasm-bindgen-macro-support@0.2.127 X X -wasm-bindgen-shared@0.2.127 X X -web-sys@0.3.104 X X -winapi@0.3.9 X X -winapi-i686-pc-windows-gnu@0.4.0 X X -winapi-x86_64-pc-windows-gnu@0.4.0 X X -windows-link@0.2.1 X X -windows-sys@0.61.2 X X -wit-bindgen@0.46.0 X X X -zerocopy@0.8.56 X X X -zmij@1.0.23 X +crate Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib +ahash@0.8.12 X X +aho-corasick@1.1.5 X X +android_system_properties@0.1.6 X X +anstream@1.0.0 X X +anstyle@1.0.14 X X +anstyle-parse@1.0.0 X X +anstyle-query@1.1.5 X X +anstyle-wincon@3.0.11 X X +anyhow@1.0.104 X X +arrow@59.2.0 X +arrow-arith@59.2.0 X +arrow-array@59.2.0 X X +arrow-buffer@59.2.0 X +arrow-cast@59.2.0 X +arrow-csv@59.2.0 X +arrow-data@59.2.0 X +arrow-ipc@59.2.0 X +arrow-json@59.2.0 X +arrow-ord@59.2.0 X +arrow-row@59.2.0 X +arrow-schema@59.2.0 X +arrow-select@59.2.0 X +arrow-string@59.2.0 X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +axum@0.8.9 X +axum-core@0.5.6 X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.1 X X +bigdecimal@0.4.10 X X +bitflags@2.13.1 X X +bitvec@1.1.1 X +block-buffer@0.10.4 X X +bumpalo@3.20.3 X X +byteorder@1.5.0 X X +bytes@1.12.1 X +cc@1.4.3 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +clap@4.6.6 X X +clap_builder@4.6.6 X X +clap_derive@4.6.4 X X +clap_lex@1.1.0 X X +colorchoice@1.0.5 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +crc32c@0.6.8 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +csv@1.4.0 X X +csv-core@0.1.13 X X +dashmap@6.2.1 X +delegate@0.13.5 X X +digest@0.10.7 X X +displaydoc@0.2.7 X X +either@1.17.0 X X +equivalent@1.0.2 X X +errno@0.3.14 X X +fastrand@2.5.0 X X +find-msvc-tools@0.1.11 X X +fixedbitset@0.5.7 X X +flatbuffers@25.12.19 X +fluss-gateway@1.0.0 X +fluss-rs@1.0.0 X +foldhash@0.1.5 X +foldhash@0.2.0 X +form_urlencoded@1.2.2 X X +funty@2.0.0 X +futures@0.3.34 X X +futures-channel@0.3.34 X X +futures-core@0.3.34 X X +futures-executor@0.3.34 X X +futures-io@0.3.34 X X +futures-macro@0.3.34 X X +futures-sink@0.3.34 X X +futures-task@0.3.34 X X +futures-util@0.3.34 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +gloo-timers@0.3.0 X X +half@2.7.1 X X +hashbrown@0.14.5 X X +hashbrown@0.15.5 X X +hashbrown@0.16.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +http@1.5.0 X X +http-body@1.1.0 X +http-body-util@0.1.5 X +httparse@1.10.1 X X +httpdate@1.0.3 X X +hyper@1.11.0 X +hyper-rustls@0.27.9 X X X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +icu_collections@2.1.1 X +icu_locale_core@2.1.1 X +icu_normalizer@2.1.1 X +icu_normalizer_data@2.1.1 X +icu_properties@2.1.2 X +icu_properties_data@2.1.2 X +icu_provider@2.1.1 X +idna@1.1.0 X X +idna_adapter@1.2.1 X X +indexmap@2.14.0 X X +ipnet@2.12.1 X X +is_terminal_polyfill@1.70.2 X X +itertools@0.14.0 X X +itoa@1.0.18 X X +jiff@0.2.35 X X +jiff-core@0.1.0 X X +jiff-tzdb@0.1.8 X X +jiff-tzdb-platform@0.1.3 X X +jobserver@0.1.35 X X +js-sys@0.3.104 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.189 X X +libm@0.2.16 X +linked-hash-map@0.5.6 X X +linux-raw-sys@0.12.1 X X X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +matchit@0.8.4 X X +md-5@0.10.6 X X +memchr@2.8.3 X X +metrics@0.24.6 X +metrics-exporter-prometheus@0.17.2 X +metrics-util@0.20.4 X +mime@0.3.17 X X +mio@1.2.2 X +multimap@0.10.1 X X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-integer@0.1.47 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +once_cell_polyfill@1.70.2 X X +opendal@0.55.0 X +ordered-float@5.3.0 X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parse-display@0.10.0 X X +parse-display-derive@0.10.0 X X +paste@1.0.15 X X +percent-encoding@2.3.2 X X +petgraph@0.8.3 X X +pin-project-lite@0.2.17 X X +pkg-config@0.3.34 X X +portable-atomic@1.15.0 X X +portable-atomic-util@0.2.7 X X +potential_utf@0.1.5 X +ppv-lite86@0.2.21 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.107 X X +prost@0.14.4 X +prost-build@0.14.4 X +prost-derive@0.14.4 X +prost-types@0.14.4 X +quanta@0.12.6 X +quick-xml@0.38.4 X +quote@1.0.47 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +radium@0.7.0 X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.9.5 X X +rand_xoshiro@0.7.0 X X +rapidhash@4.5.1 X X +raw-cpuid@11.6.0 X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.18 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +rustc_version@0.4.1 X X +rustix@1.1.4 X X X +rustls@0.23.43 X X X +rustls-pki-types@1.15.1 X X +rustls-webpki@0.103.14 X +rustversion@1.0.23 X X +ryu@1.0.23 X X +scopeguard@1.2.0 X X +semver@1.0.28 X X +serde@1.0.229 X X +serde_core@1.0.229 X X +serde_derive@1.0.229 X X +serde_json@1.0.151 X X +serde_path_to_error@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_yaml_ng@0.10.0 X +shlex@2.0.1 X X +signal-hook-registry@1.4.8 X X +simdutf8@0.1.5 X X +sketches-ddsketch@0.3.1 X +slab@0.4.12 X +smallvec@1.15.2 X X +snafu@0.8.9 X X +snafu-derive@0.8.9 X X +socket2@0.6.5 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +structmeta@0.3.0 X X +structmeta-derive@0.3.0 X X +strum@0.26.3 X +strum_macros@0.26.4 X +subtle@2.6.1 X +syn@2.0.119 X X +syn@3.0.3 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tap@1.0.1 X +tempfile@3.27.0 X X +thiserror@1.0.69 X X +thiserror@2.0.20 X X +thiserror-impl@1.0.69 X X +thiserror-impl@2.0.20 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.2 X +tokio-rustls@0.26.4 X X +tokio-util@0.7.19 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-core@0.1.36 X +try-lock@0.2.5 X +twox-hash@2.1.3 X +typenum@1.20.1 X X +unicode-ident@1.0.24 X X X +unsafe-libyaml@0.2.11 X +untrusted@0.9.0 X +url@2.5.8 X X +utf8_iter@1.0.4 X X +utf8parse@0.2.2 X X +utoipa@5.5.0 X X +utoipa-axum@0.2.0 X X +utoipa-gen@5.5.0 X X +uuid@1.24.0 X X +value-bag@1.13.2 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.1+wasi-0.2.4 X X X +wasm-bindgen@0.2.127 X X +wasm-bindgen-futures@0.4.77 X X +wasm-bindgen-macro@0.2.127 X X +wasm-bindgen-macro-support@0.2.127 X X +wasm-bindgen-shared@0.2.127 X X +wasm-streams@0.4.2 X X +web-sys@0.3.104 X X +webpki-roots@1.0.9 X +winapi@0.3.9 X X +winapi-i686-pc-windows-gnu@0.4.0 X X +winapi-x86_64-pc-windows-gnu@0.4.0 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.46.0 X X X +writeable@0.6.3 X +wyz@0.5.1 X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.56 X X X +zerocopy-derive@0.8.56 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zmij@1.0.23 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/fluss-gateway/justfile b/fluss-gateway/justfile index 968b59b363..9172a6c1c4 100644 --- a/fluss-gateway/justfile +++ b/fluss-gateway/justfile @@ -63,4 +63,4 @@ licenses: # Regenerate the checked-in dependency license inventory. deps: - cargo deny --locked list -f tsv -t 0.6 > DEPENDENCIES.rust.tsv + cargo deny --locked list -f tsv -t 0.6 | sed 's/[[:space:]]*$//' > DEPENDENCIES.rust.tsv diff --git a/fluss-gateway/src/config.rs b/fluss-gateway/src/config.rs index e1d66ff415..0f7023a3a5 100644 --- a/fluss-gateway/src/config.rs +++ b/fluss-gateway/src/config.rs @@ -22,11 +22,24 @@ //! ```yaml //! gateway.rest.listen: 0.0.0.0:8080 //! gateway.rest.write.max-request-bytes: 32MiB +//! gateway.clusters: default +//! gateway.cluster.default.bootstrap.servers: 127.0.0.1:9123 +//! gateway.cluster.default.connection.service.account: gateway_svc +//! gateway.cluster.default.client.writer.batch-size: 2MiB //! ``` //! //! Environment variable names are derived from these public keys; for example, -//! `gateway.rest.listen` becomes `FLUSS_GATEWAY__REST__LISTEN`. +//! `gateway.rest.listen` becomes `FLUSS_GATEWAY__REST__LISTEN`, and +//! `gateway.cluster.default.client.writer.batch-size` becomes +//! `FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE`. +//! +//! `gateway.clusters` is authoritative (with an implicit single `default` cluster), and +//! `gateway.cluster..client.*` accepts only validated options that preserve Gateway guarantees. +//! +//! Credentials are redacted in diagnostics: typed fields carry them as [`Secret`], and the open +//! `client.*` namespace declares per option which values are sensitive. +use fluss::config::Config as NativeClientConfig; use serde::Deserialize; use serde::de::{self, Deserializer}; use serde_yaml_ng::{Mapping, Value}; @@ -39,6 +52,38 @@ use std::time::Duration; /// Environment variable prefix for overrides. pub const ENV_PREFIX: &str = "FLUSS_GATEWAY__"; +/// Stand-in printed wherever a credential would otherwise be rendered. +/// +/// Spelled the same as `Password.HIDDEN_CONTENT` on the Java side, so one deployment's server and gateway +/// logs redact identically. +const REDACTED: &str = "******"; + +/// A configuration value that must never reach a log line, an error, or a debug dump. +/// +/// The wrapper is the whole mechanism: [`Debug`] is the only way diagnostics render configuration, so +/// redacting it here covers every present and future diagnostic without each call site remembering to. +#[derive(Clone, PartialEq, Eq, Deserialize)] +#[serde(transparent)] +pub struct Secret(String); + +impl Secret { + /// Wraps a credential that reached the gateway as plain text, so it redacts from here on. + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + /// Hands the credential to the component that authenticates with it. + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(REDACTED) + } +} + /// A duration written as ``. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ConfigDuration(Duration); @@ -200,6 +245,13 @@ const REST_MAX_REQUEST_BYTES_KEY: &str = "gateway.rest.write.max-request-bytes"; const METRICS_ENABLED_KEY: &str = "gateway.metrics.enabled"; const METRICS_LISTEN_KEY: &str = "gateway.metrics.exporter.prometheus.listen"; const SHUTDOWN_DRAIN_TIMEOUT_KEY: &str = "gateway.shutdown.drain-timeout"; +const CLUSTERS_KEY: &str = "gateway.clusters"; +const CLUSTER_KEY_PREFIX: &str = "gateway.cluster."; +const CLIENT_OPTION_PREFIX: &str = "client."; +const SECURITY_AUTHENTICATION_KEY: &str = "gateway.security.authentication"; +const SECURITY_USERS_KEY: &str = "gateway.security.users"; +const SECURITY_TOKENS_KEY: &str = "gateway.security.tokens"; +const SECURITY_TRUSTED_HEADER_NAME_KEY: &str = "gateway.security.trusted-header.name"; const DEFAULT_REST_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); const DEFAULT_REST_HEADER_READ_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(10); @@ -209,44 +261,180 @@ const DEFAULT_METRICS_ENABLED: bool = true; const DEFAULT_METRICS_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9095); const DEFAULT_SHUTDOWN_DRAIN_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(30); +/// The default cluster ID, so a single-cluster deployment configures no cluster list at all. +const DEFAULT_CLUSTER_ID: &str = "default"; +const DEFAULT_BOOTSTRAP_SERVERS: &str = "127.0.0.1:9123"; +const DEFAULT_TRUSTED_HEADER_NAME: &str = "x-forwarded-user"; + +/// How a source value is converted before it reaches Serde. +/// +/// Most options are strings whose typed parsing lives in a `Deserialize` impl, but a bool or an integer +/// has to arrive as the matching YAML scalar, and a server list accepts either a sequence or one +/// comma-separated string. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + /// Parsed downstream by the field's own `Deserialize` impl (address, duration, enum, secret, text). + Text, + Bool, + /// A non-negative integer, which Serde cannot read from a string. + Integer, + /// An integer count of bytes or a byte-size string such as `32MiB`. + Bytes, + /// A YAML sequence or one comma-separated string. + ServerList, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ConfigEntry { key: &'static str, internal_path: &'static str, + kind: ValueKind, } const CONFIG_ENTRIES: &[ConfigEntry] = &[ ConfigEntry { key: INSTANCE_ID_KEY, internal_path: "server.instance_id", + kind: ValueKind::Text, }, ConfigEntry { key: REST_LISTEN_KEY, internal_path: "server.rest.bind_address", + kind: ValueKind::Text, }, ConfigEntry { key: REST_HEADER_READ_TIMEOUT_KEY, internal_path: "server.rest.header_read_timeout", + kind: ValueKind::Text, }, ConfigEntry { key: REST_REQUEST_TIMEOUT_KEY, internal_path: "server.rest.request_timeout", + kind: ValueKind::Text, }, ConfigEntry { key: REST_MAX_REQUEST_BYTES_KEY, internal_path: "server.rest.max_body_bytes", + kind: ValueKind::Bytes, + }, + ConfigEntry { + key: "gateway.rest.write.max-rows", + internal_path: "request_limits.write_max_rows", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.write.max-concurrent-requests", + internal_path: "request_limits.write_max_concurrent_requests", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-keys", + internal_path: "request_limits.lookup_max_keys", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-key-bytes", + internal_path: "request_limits.lookup_max_key_bytes", + kind: ValueKind::Bytes, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-concurrent-requests", + internal_path: "request_limits.lookup_max_concurrent_requests", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-prefixes", + internal_path: "request_limits.prefix_lookup_max_prefixes", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-rows-per-prefix", + internal_path: "request_limits.prefix_lookup_max_rows_per_prefix", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-concurrent-requests", + internal_path: "request_limits.prefix_lookup_max_concurrent_requests", + kind: ValueKind::Integer, }, ConfigEntry { key: METRICS_ENABLED_KEY, internal_path: "server.metrics.enabled", + kind: ValueKind::Bool, }, ConfigEntry { key: METRICS_LISTEN_KEY, internal_path: "server.metrics.bind_address", + kind: ValueKind::Text, }, ConfigEntry { key: SHUTDOWN_DRAIN_TIMEOUT_KEY, internal_path: "shutdown.drain_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_AUTHENTICATION_KEY, + internal_path: "security.authentication", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_USERS_KEY, + internal_path: "security.users", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_TOKENS_KEY, + internal_path: "security.tokens", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_TRUSTED_HEADER_NAME_KEY, + internal_path: "security.trusted_header_name", + kind: ValueKind::Text, + }, +]; + +/// The per-cluster options under `gateway.cluster..`, excluding the open `client.*` namespace. +const CLUSTER_ENTRIES: &[ConfigEntry] = &[ + ConfigEntry { + key: "bootstrap.servers", + internal_path: "bootstrap_servers", + kind: ValueKind::ServerList, + }, + ConfigEntry { + key: "connect-timeout", + internal_path: "connect_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "request-timeout", + internal_path: "request_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.service.account", + internal_path: "service_account", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.service.secret", + internal_path: "service_secret", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.identity-mode", + internal_path: "identity_mode", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.max", + internal_path: "connection_max", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "connection.idle-timeout", + internal_path: "connection_idle_timeout", + kind: ValueKind::Text, }, ]; @@ -327,6 +515,553 @@ impl Default for MetricsServerConfig { } } +/// Deserializes a bootstrap list from either a YAML sequence or one comma-separated string. +fn deserialize_server_list<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum ListOrCsv { + List(Vec), + Csv(String), + } + + Ok(match ListOrCsv::deserialize(deserializer)? { + ListOrCsv::List(list) => list, + ListOrCsv::Csv(csv) => csv + .split(',') + .map(|entry| entry.trim().to_string()) + .collect(), + }) +} + +/// How the gateway derives the effective Fluss principal for one cluster. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum IdentityMode { + /// One shared connection authenticated as the configured service account. + #[default] + Service, + /// Authenticate as the service account and carry the request's principal as the authorization ID. + User, +} + +/// Connection and native-client settings for one Fluss cluster. +/// +/// The connections themselves arrive with the native backend; this task owns the typed, validated, and +/// redacted contract they will read. +#[derive(Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ClusterConfig { + #[serde(deserialize_with = "deserialize_server_list")] + pub bootstrap_servers: Vec, + pub connect_timeout: ConfigDuration, + pub request_timeout: ConfigDuration, + /// Canonical account the gateway authenticates to Fluss with. An identity, not a credential, so it is + /// rendered in diagnostics: Fluss treats secrets and tokens as sensitive, not principal names. + pub service_account: Option, + /// Canonical credential for [`Self::service_account`]. + pub service_secret: Option, + pub identity_mode: IdentityMode, + /// Cap on pooled per-user connections. User identity mode only. + pub connection_max: Option, + /// Idle reclamation for pooled per-user connections. User identity mode only. + pub connection_idle_timeout: Option, + /// Validated `gateway.cluster..client.*` options, keyed without the `client.` prefix. + pub client_options: BTreeMap, +} + +impl Default for ClusterConfig { + fn default() -> Self { + Self { + bootstrap_servers: vec![DEFAULT_BOOTSTRAP_SERVERS.to_string()], + connect_timeout: ConfigDuration::from_secs(10), + request_timeout: ConfigDuration::from_secs(30), + service_account: None, + service_secret: None, + identity_mode: IdentityMode::Service, + connection_max: None, + connection_idle_timeout: None, + client_options: BTreeMap::new(), + } + } +} + +impl ClusterConfig { + /// Returns the account actually used; the deprecated alias takes precedence when present. + pub fn effective_service_account(&self) -> Option<&str> { + self.client_option(LEGACY_SERVICE_ACCOUNT_OPTION) + .or(self.service_account.as_deref()) + } + + /// Returns the credential actually used; the deprecated alias takes precedence when present. + pub fn effective_service_secret(&self) -> Option<&str> { + self.client_option(LEGACY_SERVICE_SECRET_OPTION) + .or_else(|| self.service_secret.as_ref().map(Secret::expose)) + } + + /// Reads one configured native-client option without its `client.` prefix. + pub fn client_option(&self, option: &str) -> Option<&str> { + self.client_options.get(option).map(String::as_str) + } +} + +impl fmt::Debug for ClusterConfig { + /// Renders the client options with each sensitive value replaced, rather than hiding all of them: the + /// configured tuning is what an operator needs to see, and only the credentials have to disappear. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let client_options: BTreeMap<&str, &str> = self + .client_options + .iter() + .map(|(option, value)| { + let rendered = if client_option_is_sensitive(option) { + REDACTED + } else { + value.as_str() + }; + (option.as_str(), rendered) + }) + .collect(); + f.debug_struct("ClusterConfig") + .field("bootstrap_servers", &self.bootstrap_servers) + .field("connect_timeout", &self.connect_timeout) + .field("request_timeout", &self.request_timeout) + .field("service_account", &self.service_account) + .field("service_secret", &self.service_secret) + .field("identity_mode", &self.identity_mode) + .field("connection_max", &self.connection_max) + .field("connection_idle_timeout", &self.connection_idle_timeout) + .field("client_options", &client_options) + .finish() + } +} + +/// A validated native-client option. The variant records the conversion the native client needs. +/// +/// A credential stays wrapped in [`Secret`] rather than becoming a `Text`, so the parsed value redacts +/// exactly like the configured one did and no caller can print it back out by accident. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ClientOptionValue { + Text(String), + Secret(Secret), + Boolean(bool), + Integer(u64), + Millis(u64), + Bytes(u64), +} + +impl ClientOptionValue { + fn text(&self) -> &str { + match self { + Self::Text(value) => value, + _ => unreachable!("option kind produces text"), + } + } + + fn secret(&self) -> &str { + match self { + Self::Secret(value) => value.expose(), + _ => unreachable!("option kind produces a secret"), + } + } + + fn boolean(&self) -> bool { + match self { + Self::Boolean(value) => *value, + _ => unreachable!("option kind produces a boolean"), + } + } + + fn integer(&self) -> u64 { + match self { + Self::Integer(value) => *value, + _ => unreachable!("option kind produces an integer"), + } + } + + fn millis(&self) -> u64 { + match self { + Self::Millis(value) => *value, + _ => unreachable!("option kind produces milliseconds"), + } + } + + fn bytes(&self) -> u64 { + match self { + Self::Bytes(value) => *value, + _ => unreachable!("option kind produces bytes"), + } + } +} + +/// Superseded by `connection.service.account`; still honoured, with a warning. +const LEGACY_SERVICE_ACCOUNT_OPTION: &str = "security.sasl.username"; +/// Superseded by `connection.service.secret`; still honoured, with a warning. +const LEGACY_SERVICE_SECRET_OPTION: &str = "security.sasl.password"; + +/// The native field type targeted by one public `client.*` option. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ClientOptionKind { + Text, + Secret, + Boolean, + I32Count { min: u64 }, + UsizeCount { min: u64 }, + I32Bytes, + UsizeBytes, + I64Millis, + U64Millis, +} + +impl ClientOptionKind { + fn is_sensitive(self) -> bool { + matches!(self, Self::Secret) + } +} + +type ApplyClientOption = fn(&mut NativeClientConfig, &ClientOptionValue); + +#[derive(Clone, Copy)] +struct ClientOptionSpec { + option: &'static str, + kind: ClientOptionKind, + apply: ApplyClientOption, +} + +/// Public FIP names mapped to fields on [`NativeClientConfig`]. Native defaults and cross-field +/// validation come from that type rather than being copied into the gateway. +const CLIENT_OPTIONS: &[ClientOptionSpec] = &[ + ClientOptionSpec { + option: "security.protocol", + kind: ClientOptionKind::Text, + apply: |config, value| config.security_protocol = value.text().to_string(), + }, + ClientOptionSpec { + option: "security.sasl.mechanism", + kind: ClientOptionKind::Text, + apply: |config, value| config.security_sasl_mechanism = value.text().to_string(), + }, + ClientOptionSpec { + option: LEGACY_SERVICE_ACCOUNT_OPTION, + kind: ClientOptionKind::Text, + apply: |config, value| config.security_sasl_username = value.text().to_string(), + }, + ClientOptionSpec { + option: LEGACY_SERVICE_SECRET_OPTION, + kind: ClientOptionKind::Secret, + apply: |config, value| config.security_sasl_password = value.secret().to_string(), + }, + ClientOptionSpec { + option: "connect-timeout", + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.connect_timeout_ms = value.millis(), + }, + ClientOptionSpec { + option: "writer.batch-size", + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_batch_size = value.bytes() as i32, + }, + ClientOptionSpec { + option: "writer.request-max-size", + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_request_max_size = value.bytes() as i32, + }, + ClientOptionSpec { + option: "writer.buffer.memory-size", + kind: ClientOptionKind::UsizeBytes, + apply: |config, value| config.writer_buffer_memory_size = value.bytes() as usize, + }, + ClientOptionSpec { + option: "writer.buffer.wait-timeout", + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.writer_buffer_wait_timeout_ms = value.millis(), + }, + ClientOptionSpec { + option: "writer.batch-timeout", + kind: ClientOptionKind::I64Millis, + apply: |config, value| config.writer_batch_timeout_ms = value.millis() as i64, + }, + ClientOptionSpec { + option: "writer.dynamic-batch-size.enabled", + kind: ClientOptionKind::Boolean, + apply: |config, value| config.writer_dynamic_batch_size_enabled = value.boolean(), + }, + ClientOptionSpec { + option: "writer.dynamic-batch-size.min", + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_dynamic_batch_size_min = value.bytes() as i32, + }, + ClientOptionSpec { + option: "writer.kv-backpressure.max-throttle", + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.writer_kv_backpressure_max_throttle_ms = value.millis(), + }, + ClientOptionSpec { + option: "lookup.queue-size", + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_queue_size = value.integer() as usize, + }, + ClientOptionSpec { + option: "lookup.max-batch-size", + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_max_batch_size = value.integer() as usize, + }, + ClientOptionSpec { + option: "lookup.max-inflight-requests", + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_max_inflight_requests = value.integer() as usize, + }, + ClientOptionSpec { + option: "lookup.max-retries", + kind: ClientOptionKind::I32Count { min: 0 }, + apply: |config, value| config.lookup_max_retries = value.integer() as i32, + }, + ClientOptionSpec { + option: "lookup.batch-timeout", + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.lookup_batch_timeout_ms = value.millis(), + }, +]; + +/// The `client.*` options the gateway refuses outright, each with the reason an operator sees. +/// +/// The first three would silently weaken the write guarantees the gateway advertises. The fourth is never +/// static: user identity mode sets the authorization ID per connection from the authenticated principal. +const RESERVED_CLIENT_OPTIONS: &[(&str, &str)] = &[ + ( + "writer.acks", + "is owned by the Gateway and cannot be overridden", + ), + ( + "writer.retries", + "is owned by the Gateway and cannot be overridden", + ), + ( + "writer.enable-idempotence", + "is owned by the Gateway and cannot be overridden", + ), + ( + "security.sasl.authorization-id", + "is supplied per connection in user identity mode and cannot be configured statically", + ), +]; + +/// True when the option's value must not appear in diagnostics. Unknown options are assumed sensitive, so +/// a value that failed validation cannot leak through a debug dump. +fn client_option_is_sensitive(option: &str) -> bool { + client_option_spec(option).is_none_or(|spec| spec.kind.is_sensitive()) +} + +fn client_option_spec(option: &str) -> Option<&'static ClientOptionSpec> { + CLIENT_OPTIONS.iter().find(|spec| spec.option == option) +} + +/// Parses one option using the type and range of its destination in [`NativeClientConfig`]. +pub fn parse_client_option(option: &str, raw: &str) -> Result { + if let Some((_, reason)) = RESERVED_CLIENT_OPTIONS + .iter() + .find(|(reserved, _)| *reserved == option) + { + return Err(format!("client.{option} {reason}")); + } + let Some(spec) = client_option_spec(option) else { + return Err(format!( + "client.{option} is not a supported native-client option" + )); + }; + // No message echoes the value: this namespace also carries the legacy SASL credentials. + let malformed = + |expected: &str, reason: String| format!("client.{option}: expected {expected}: {reason}"); + match spec.kind { + ClientOptionKind::Text => Ok(ClientOptionValue::Text(raw.to_string())), + ClientOptionKind::Secret => Ok(ClientOptionValue::Secret(Secret::new(raw))), + ClientOptionKind::Boolean => raw.parse().map(ClientOptionValue::Boolean).map_err( + |error: std::str::ParseBoolError| malformed("true or false", error.to_string()), + ), + ClientOptionKind::I32Count { min } => { + let parsed = raw + .parse::() + .map_err(|error| malformed("a non-negative integer", error.to_string()))?; + if parsed < min as i32 { + return Err(format!("client.{option}: must be at least {min}")); + } + Ok(ClientOptionValue::Integer(parsed as u64)) + } + ClientOptionKind::UsizeCount { min } => { + let parsed = raw + .parse::() + .map_err(|error| malformed("a non-negative integer", error.to_string()))?; + if parsed < min as usize { + return Err(format!("client.{option}: must be at least {min}")); + } + Ok(ClientOptionValue::Integer(parsed as u64)) + } + ClientOptionKind::I32Bytes => { + let bytes = ByteSize::parse(raw) + .map_err(|error| malformed("a byte size", error))? + .bytes(); + i32::try_from(bytes) + .map(|_| ClientOptionValue::Bytes(bytes)) + .map_err(|_| format!("client.{option}: must not exceed {} bytes", i32::MAX)) + } + ClientOptionKind::UsizeBytes => { + let bytes = ByteSize::parse(raw) + .map_err(|error| malformed("a byte size", error))? + .bytes(); + usize::try_from(bytes) + .map(|_| ClientOptionValue::Bytes(bytes)) + .map_err(|_| format!("client.{option}: does not fit the native field")) + } + ClientOptionKind::I64Millis => parse_client_millis(option, raw, true) + .and_then(|millis| { + i64::try_from(millis) + .map_err(|_| format!("client.{option}: does not fit the native field")) + }) + .map(|millis| ClientOptionValue::Millis(millis as u64)), + ClientOptionKind::U64Millis => { + parse_client_millis(option, raw, true).map(ClientOptionValue::Millis) + } + } +} + +fn parse_client_millis(option: &str, raw: &str, allow_zero: bool) -> Result { + let (digits, unit) = split_number_and_unit(raw); + let value = digits + .parse::() + .map_err(|error| format!("client.{option}: expected a duration: {error}"))?; + let multiplier = match unit { + "ms" => 1, + "s" => 1_000, + "m" => 60_000, + "h" => 3_600_000, + _ => { + return Err(format!( + "client.{option}: expected a duration with unit ms, s, m, or h" + )); + } + }; + let millis = value + .checked_mul(multiplier) + .ok_or_else(|| format!("client.{option}: duration is too large"))?; + if !allow_zero && millis == 0 { + return Err(format!("client.{option}: must be greater than zero")); + } + Ok(millis) +} + +/// How the gateway authenticates its own HTTP callers. +/// +/// The authenticators arrive with the REST authentication task; this task owns the typed contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum AuthenticationMode { + /// Every caller is accepted and reported as an anonymous principal. + #[default] + Trust, + Password, + Token, + TrustedHeader, +} + +/// Client-to-gateway authentication settings. Every credential-bearing field is a [`Secret`]. +#[derive(Clone, PartialEq, Deserialize, Default)] +#[serde(deny_unknown_fields, default)] +pub struct SecurityConfig { + pub authentication: AuthenticationMode, + /// Password-mode user table; the entries embed password material. + pub users: Option, + /// Token-mode table; the entries are bearer tokens. + pub tokens: Option, + /// Trusted-header-mode header name, defaulting to `x-forwarded-user`. + pub trusted_header_name: Option, +} + +impl SecurityConfig { + /// Returns the header the trusted-header mode reads the principal from. + pub fn trusted_header_name(&self) -> &str { + self.trusted_header_name + .as_deref() + .unwrap_or(DEFAULT_TRUSTED_HEADER_NAME) + } +} + +impl fmt::Debug for SecurityConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecurityConfig") + .field("authentication", &self.authentication) + .field("users", &self.users) + .field("tokens", &self.tokens) + .field("trusted_header_name", &self.trusted_header_name) + .finish() + } +} + +/// Admission limits for the data-plane APIs, whose handlers arrive in later tasks. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RequestLimitsConfig { + pub write_max_rows: u32, + pub write_max_concurrent_requests: u32, + pub lookup_max_keys: u32, + pub lookup_max_key_bytes: ByteSize, + pub lookup_max_concurrent_requests: u32, + pub prefix_lookup_max_prefixes: u32, + pub prefix_lookup_max_rows_per_prefix: u32, + pub prefix_lookup_max_concurrent_requests: u32, +} + +impl Default for RequestLimitsConfig { + fn default() -> Self { + Self { + write_max_rows: 10_000, + write_max_concurrent_requests: 64, + lookup_max_keys: 128, + lookup_max_key_bytes: ByteSize::new(1024 * 1024), + lookup_max_concurrent_requests: 64, + prefix_lookup_max_prefixes: 16, + prefix_lookup_max_rows_per_prefix: 1000, + prefix_lookup_max_concurrent_requests: 32, + } + } +} + +impl RequestLimitsConfig { + fn validate(&self, problems: &mut Vec) { + for (key, value) in [ + ("gateway.rest.write.max-rows", self.write_max_rows), + ( + "gateway.rest.write.max-concurrent-requests", + self.write_max_concurrent_requests, + ), + ("gateway.rest.lookup.max-keys", self.lookup_max_keys), + ( + "gateway.rest.lookup.max-concurrent-requests", + self.lookup_max_concurrent_requests, + ), + ( + "gateway.rest.prefix-lookup.max-prefixes", + self.prefix_lookup_max_prefixes, + ), + ( + "gateway.rest.prefix-lookup.max-rows-per-prefix", + self.prefix_lookup_max_rows_per_prefix, + ), + ( + "gateway.rest.prefix-lookup.max-concurrent-requests", + self.prefix_lookup_max_concurrent_requests, + ), + ] { + if value == 0 { + problems.push(format!("{key} must be greater than zero")); + } + } + if self.lookup_max_key_bytes.bytes() == 0 { + problems + .push("gateway.rest.lookup.max-key-bytes must be greater than zero".to_string()); + } + } +} + /// Graceful-shutdown configuration. #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields, default)] @@ -355,13 +1090,30 @@ impl ShutdownConfig { } /// The validated gateway configuration: everything the process needs before it binds a listener. -#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct GatewayConfig { pub server: ServerConfig, + /// Every declared Fluss cluster, keyed by the ID used in REST paths. + pub clusters: BTreeMap, + pub security: SecurityConfig, + pub request_limits: RequestLimitsConfig, pub shutdown: ShutdownConfig, } +impl Default for GatewayConfig { + /// Defaults to the single `default` cluster, so a local deployment needs no cluster list. + fn default() -> Self { + Self { + server: ServerConfig::default(), + clusters: BTreeMap::from([(DEFAULT_CLUSTER_ID.to_string(), ClusterConfig::default())]), + security: SecurityConfig::default(), + request_limits: RequestLimitsConfig::default(), + shutdown: ShutdownConfig::default(), + } + } +} + impl GatewayConfig { /// Checks invariants, including values supplied programmatically. pub fn validate(&self) -> Result<(), ConfigError> { @@ -369,6 +1121,9 @@ impl GatewayConfig { self.server.rest.validate(&mut problems); self.shutdown.validate(&mut problems); self.validate_identity(&mut problems); + self.validate_clusters(&mut problems); + self.validate_security(&mut problems); + self.request_limits.validate(&mut problems); if problems.is_empty() { Ok(()) } else { @@ -376,6 +1131,181 @@ impl GatewayConfig { } } + /// Rejects any cluster the gateway could not connect with, before a listener is bound. + fn validate_clusters(&self, problems: &mut Vec) { + if self.clusters.is_empty() { + problems.push(format!("{CLUSTERS_KEY} must declare at least one cluster")); + } + for (id, cluster) in &self.clusters { + if !valid_cluster_id(id) { + problems.push(format!( + "cluster ID {id:?} must start with a lowercase letter and contain only \ + lowercase letters, digits, or underscores" + )); + } + if cluster.bootstrap_servers.is_empty() + || cluster + .bootstrap_servers + .iter() + .any(|server| server.trim().is_empty()) + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.bootstrap.servers must list at least one non-blank server" + )); + } + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.connect-timeout"), + cluster.connect_timeout.get(), + problems, + ); + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.request-timeout"), + cluster.request_timeout.get(), + problems, + ); + if let Some(idle_timeout) = cluster.connection_idle_timeout { + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.connection.idle-timeout"), + idle_timeout.get(), + problems, + ); + } + if cluster.connection_max == Some(0) { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.max must be greater than zero" + )); + } + let mut native = NativeClientConfig::default(); + for (option, raw) in &cluster.client_options { + match parse_client_option(option, raw) { + Ok(value) => (client_option_spec(option) + .expect("parsed option has a registered native destination") + .apply)(&mut native, &value), + Err(problem) => problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.{problem}")), + } + } + if let Some(account) = cluster.effective_service_account() { + native.security_sasl_username = account.to_string(); + } + if let Some(secret) = cluster.effective_service_secret() { + native.security_sasl_password = secret.to_string(); + } + if let Err(problem) = native.validate_writer() { + problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.client: {problem}")); + } + if let Err(problem) = native.validate_security() { + problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.client: {problem}")); + } + self.validate_credentials(id, cluster, problems); + } + } + + /// Requires a service identity the gateway can actually authenticate with, wherever one is needed. + fn validate_credentials(&self, id: &str, cluster: &ClusterConfig, problems: &mut Vec) { + let account = cluster.effective_service_account(); + let secret = cluster.effective_service_secret(); + for (setting, value) in [("account", account), ("secret", secret)] { + if value.is_some_and(|value| value.trim().is_empty()) { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.service.{setting} must not be blank" + )); + } + } + // A blank value is not a credential, so "complete" means usable rather than merely present. + let usable = |value: Option<&str>| value.is_some_and(|value| !value.trim().is_empty()); + let credentials_usable = usable(account) && usable(secret); + if account.is_some() != secret.is_some() { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.service.account and connection.service.secret \ + must be set together" + )); + } + + let protocol = cluster.client_option("security.protocol"); + let sasl = protocol.is_some_and(|protocol| protocol.eq_ignore_ascii_case("sasl")); + if let Some(protocol) = protocol + && !sasl + && !protocol.eq_ignore_ascii_case("plaintext") + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.protocol must be plaintext or sasl" + )); + } + if sasl && !credentials_usable { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.protocol sasl requires \ + connection.service.account and connection.service.secret" + )); + } + if let Some(mechanism) = cluster.client_option("security.sasl.mechanism") + && !mechanism.eq_ignore_ascii_case("plain") + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.sasl.mechanism must be PLAIN" + )); + } + + if cluster.identity_mode == IdentityMode::User { + if self.security.authentication == AuthenticationMode::Trust { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires verified client identities; set {SECURITY_AUTHENTICATION_KEY} to password, token, or trusted-header" + )); + } + if !credentials_usable { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires \ + connection.service.account and connection.service.secret" + )); + } + // Act-as travels as the SASL/PLAIN authorization ID, so without SASL there is nowhere to put + // the request's principal: Fluss would authorize the gateway's own identity for every caller. + // Requiring it to be declared rather than inferring it keeps a security-relevant choice visible. + if !sasl { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires \ + client.security.protocol sasl, which carries the request principal as the SASL \ + authorization ID" + )); + } + } + } + + /// Requires the credential table the selected authentication mode reads. + fn validate_security(&self, problems: &mut Vec) { + let configured = |secret: &Option| { + secret + .as_ref() + .is_some_and(|value| !value.expose().trim().is_empty()) + }; + match self.security.authentication { + AuthenticationMode::Password if !configured(&self.security.users) => { + problems.push(format!( + "{SECURITY_USERS_KEY} must configure at least one user when \ + {SECURITY_AUTHENTICATION_KEY} is password" + )); + } + AuthenticationMode::Token if !configured(&self.security.tokens) => { + problems.push(format!( + "{SECURITY_TOKENS_KEY} must configure at least one token when \ + {SECURITY_AUTHENTICATION_KEY} is token" + )); + } + AuthenticationMode::TrustedHeader => { + let name = self.security.trusted_header_name(); + if name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + problems.push(format!( + "{SECURITY_TRUSTED_HEADER_NAME_KEY} must be a legal HTTP header name" + )); + } + } + _ => {} + } + } + /// Rejects an unusable instance identity or a port clash between the two listeners. /// /// A non-loopback listener does **not** require an instance ID. Nothing the gateway returns is scoped to an @@ -414,8 +1344,47 @@ impl GatewayConfig { REST_LISTEN_KEY, self.server.rest.bind_address )); } + for (id, cluster) in &self.clusters { + let legacy: Vec<&str> = [LEGACY_SERVICE_ACCOUNT_OPTION, LEGACY_SERVICE_SECRET_OPTION] + .into_iter() + .filter(|option| cluster.client_options.contains_key(*option)) + .collect(); + // One warning per cluster, not per option: an operator migrates the pair together. + if !legacy.is_empty() { + warnings.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.{{{}}} is deprecated; use \ + connection.service.account and connection.service.secret. The legacy values \ + take precedence over the canonical fields", + legacy.join(",") + )); + } + if cluster.identity_mode == IdentityMode::Service + && (cluster.connection_max.is_some() || cluster.connection_idle_timeout.is_some()) + { + warnings.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.max and connection.idle-timeout are \ + ignored because connection.identity-mode is service" + )); + } + } warnings } + + /// Renders the whole configuration for diagnostics with every credential replaced. + /// + /// Redaction is a property of the types, so this is the plain [`Debug`] rendering; the method exists to + /// give callers something they can print without having to know that. + pub fn redacted_debug(&self) -> String { + format!("{self:?}") + } +} + +/// Cluster IDs appear in REST paths and in environment variable names, so they stay in the character set +/// that survives both: no hyphen, because the environment mapping already spends `_` on it. +fn valid_cluster_id(id: &str) -> bool { + let mut bytes = id.bytes(); + bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') } /// True when two listeners cannot both bind: the addresses are equal, or either is a wildcard @@ -503,10 +1472,7 @@ fn insert_path(table: &mut Mapping, path: &str, value: Value) { } /// Attributes a typed error to the override that supplied the failing option. -fn attribute( - message: String, - overrides: &[(&'static str, &'static str, String, Value)], -) -> ConfigError { +fn attribute(message: String, overrides: &[(String, String, String, Value)]) -> ConfigError { for (_, key, origin, _) in overrides.iter().rev() { if message.starts_with(key) { return ConfigError::Parse(format!("{origin}: {message}")); @@ -528,6 +1494,16 @@ fn publicize_error_path(message: String) -> String { return format!("{}{reason}", entry.key); } } + // A cluster path carries the ID, so the public key is rebuilt rather than looked up. + if let Some(rest) = message.strip_prefix("clusters.") + && let Some((id, rest)) = rest.split_once('.') + { + for entry in CLUSTER_ENTRIES { + if let Some(reason) = rest.strip_prefix(entry.internal_path) { + return format!("{CLUSTER_KEY_PREFIX}{id}.{}{reason}", entry.key); + } + } + } message } @@ -535,15 +1511,32 @@ fn config_entry(key: &str) -> Option<&'static ConfigEntry> { CONFIG_ENTRIES.iter().find(|entry| entry.key == key) } +/// Derives the environment variable name from a public key: drop the `gateway.` prefix, uppercase each +/// dotted segment with `-` folded to `_`, and join the segments with `__`. fn environment_variable(key: &str) -> String { - let suffix = key - .strip_prefix("gateway.") - .expect("configuration keys use the gateway prefix") + let suffix = environment_suffix( + key.strip_prefix("gateway.") + .expect("configuration keys use the gateway prefix"), + ); + format!("{ENV_PREFIX}{suffix}") +} + +fn environment_suffix(dotted: &str) -> String { + dotted .split('.') .map(|segment| segment.replace('-', "_").to_ascii_uppercase()) .collect::>() - .join("__"); - format!("{ENV_PREFIX}{suffix}") + .join("__") +} + +/// Inverts [`environment_suffix`] for the open `client.*` namespace, whose option names are not a fixed +/// list. The mapping round-trips because option names use `-` inside a segment and `.` between segments. +fn environment_suffix_to_option(suffix: &str) -> String { + suffix + .split("__") + .map(|segment| segment.to_ascii_lowercase().replace('_', "-")) + .collect::>() + .join(".") } fn environment_entry(variable: &str) -> Option<&'static ConfigEntry> { @@ -553,24 +1546,220 @@ fn environment_entry(variable: &str) -> Option<&'static ConfigEntry> { } fn convert_environment_value(entry: &ConfigEntry, raw: &str) -> Result { - match entry.key { - METRICS_ENABLED_KEY => raw + match entry.kind { + ValueKind::Bool => raw .parse::() .map(Value::Bool) .map_err(|_| "expected true or false".to_string()), - _ => Ok(Value::String(raw.to_string())), + ValueKind::Integer => raw + .parse::() + .map(|number| Value::Number(number.into())) + .map_err(|_| "expected a non-negative integer".to_string()), + ValueKind::Text | ValueKind::Bytes | ValueKind::ServerList => { + Ok(Value::String(raw.to_string())) + } } } fn convert_file_value(entry: &ConfigEntry, value: &Value) -> Result { - match entry.key { - METRICS_ENABLED_KEY => scalar(value).cloned(), - REST_MAX_REQUEST_BYTES_KEY => match scalar(value)? { + // A quoted scalar is accepted wherever a bare one is, so an operator can move the same value between + // the file and the environment, where everything arrives as a string. + match entry.kind { + ValueKind::Bool => match scalar(value)? { + Value::Bool(_) => Ok(value.clone()), + Value::String(text) => text + .parse::() + .map(Value::Bool) + .map_err(|_| "expected true or false".to_string()), + _ => Err("expected true or false".to_string()), + }, + ValueKind::Integer => match scalar(value)? { + Value::Number(_) => Ok(value.clone()), + Value::String(text) => text + .parse::() + .map(|number| Value::Number(number.into())) + .map_err(|_| "expected a non-negative integer".to_string()), + _ => Err("expected a non-negative integer".to_string()), + }, + ValueKind::Bytes => match scalar(value)? { Value::Number(_) | Value::String(_) => Ok(value.clone()), _ => Err("expected an integer or byte-size string".to_string()), }, - _ => scalar_text(value).map(Value::String), + // A sequence is the natural YAML spelling; one comma-separated string stays accepted so the + // same value can come from the environment. + ValueKind::ServerList => match value { + Value::Sequence(_) => Ok(value.clone()), + _ => scalar_text(value).map(Value::String), + }, + ValueKind::Text => scalar_text(value).map(Value::String), + } +} + +/// One recognised flat configuration key. +enum ResolvedKey { + /// `gateway.clusters`, the authoritative list of configurable cluster IDs. + ClusterDeclaration, + /// A fixed key, resolved to its dotted internal path. + Fixed(&'static ConfigEntry), + /// A per-cluster key under `gateway.cluster..`. + Cluster { + id: String, + entry: &'static ConfigEntry, + }, + /// A native-client option under `gateway.cluster..client.`. + ClientOption { id: String, option: String }, +} + +/// Resolves one public key, naming the key exactly as the operator wrote it when it is not recognised. +fn resolve_key(key: &str) -> Result { + let unknown = || ConfigError::Parse(format!("unknown configuration key: {key}")); + if key == CLUSTERS_KEY { + return Ok(ResolvedKey::ClusterDeclaration); + } + if let Some(entry) = config_entry(key) { + return Ok(ResolvedKey::Fixed(entry)); + } + let Some((id, suffix)) = key + .strip_prefix(CLUSTER_KEY_PREFIX) + .and_then(|rest| rest.split_once('.')) + else { + return Err(unknown()); + }; + if !valid_cluster_id(id) { + return Err(ConfigError::Parse(format!( + "invalid cluster ID in configuration key: {key}" + ))); + } + if let Some(option) = suffix.strip_prefix(CLIENT_OPTION_PREFIX) { + if option.is_empty() { + return Err(unknown()); + } + return Ok(ResolvedKey::ClientOption { + id: id.to_string(), + option: option.to_string(), + }); + } + CLUSTER_ENTRIES + .iter() + .find(|entry| entry.key == suffix) + .map(|entry| ResolvedKey::Cluster { + id: id.to_string(), + entry, + }) + .ok_or_else(unknown) +} + +/// Resolves one `FLUSS_GATEWAY__*` variable, which uses the same vocabulary as the file keys. +fn resolve_environment_variable(variable: &str) -> Result { + let unknown = || ConfigError::UnknownEnvKey(variable.to_string()); + let Some(suffix) = variable.strip_prefix(ENV_PREFIX) else { + return Err(unknown()); + }; + if suffix == environment_suffix("clusters") { + return Ok(ResolvedKey::ClusterDeclaration); + } + if let Some(entry) = environment_entry(variable) { + return Ok(ResolvedKey::Fixed(entry)); + } + let Some((id, rest)) = suffix + .strip_prefix("CLUSTER__") + .and_then(|rest| rest.split_once("__")) + else { + return Err(unknown()); + }; + let id = id.to_ascii_lowercase(); + if !valid_cluster_id(&id) { + return Err(unknown()); + } + if let Some(raw_option) = rest.strip_prefix("CLIENT__") { + let option = environment_suffix_to_option(raw_option); + let recognized = client_option_spec(&option).is_some() + || RESERVED_CLIENT_OPTIONS + .iter() + .any(|(reserved, _)| *reserved == option); + if option.is_empty() || environment_suffix(&option) != raw_option || !recognized { + return Err(unknown()); + } + return Ok(ResolvedKey::ClientOption { id, option }); + } + CLUSTER_ENTRIES + .iter() + .find(|entry| environment_suffix(entry.key) == rest) + .map(|entry| ResolvedKey::Cluster { id, entry }) + .ok_or_else(unknown) +} + +/// Reads the cluster IDs the file may configure, from a comma-separated string or a YAML sequence. +fn declared_cluster_ids(value: &Value) -> Result, ConfigError> { + let ids: Vec = match value { + Value::String(csv) => csv.split(',').map(|id| id.trim().to_string()).collect(), + Value::Sequence(items) => items + .iter() + .map(|item| { + item.as_str().map(str::to_string).ok_or_else(|| { + ConfigError::Parse(format!("{CLUSTERS_KEY}: entries must be strings")) + }) + }) + .collect::>()?, + _ => { + return Err(ConfigError::Parse(format!( + "{CLUSTERS_KEY}: expected a comma-separated string or a list" + ))); + } + }; + let mut unique = std::collections::BTreeSet::new(); + for id in &ids { + if !valid_cluster_id(id) { + return Err(ConfigError::Parse(format!( + "invalid cluster ID in {CLUSTERS_KEY}: {id:?}" + ))); + } + if !unique.insert(id) { + return Err(ConfigError::Parse(format!( + "duplicate cluster ID in {CLUSTERS_KEY}: {id:?}" + ))); + } } + Ok(ids) +} + +/// Returns the mapping holding one cluster's fields, creating it when the cluster is first mentioned. +fn cluster_mapping<'a>(table: &'a mut Mapping, id: &str) -> &'a mut Mapping { + let clusters = table + .entry(Value::String("clusters".to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !clusters.is_mapping() { + *clusters = Value::Mapping(Mapping::new()); + } + let clusters = clusters.as_mapping_mut().expect("mapping inserted above"); + let cluster = clusters + .entry(Value::String(id.to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !cluster.is_mapping() { + *cluster = Value::Mapping(Mapping::new()); + } + cluster.as_mapping_mut().expect("mapping inserted above") +} + +/// Writes one native-client option. The option name keeps its dots, so it cannot go through +/// [`insert_path`], which would read them as nesting. +fn insert_client_option(table: &mut Mapping, id: &str, option: &str, raw: String) { + let cluster = cluster_mapping(table, id); + let options = cluster + .entry(Value::String("client_options".to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !options.is_mapping() { + *options = Value::Mapping(Mapping::new()); + } + options + .as_mapping_mut() + .expect("mapping inserted above") + .insert(Value::String(option.to_string()), Value::String(raw)); +} + +/// Writes one per-cluster field. +fn insert_cluster_value(table: &mut Mapping, id: &str, field: &str, value: Value) { + cluster_mapping(table, id).insert(Value::String(field.to_string()), value); } fn scalar(value: &Value) -> Result<&Value, String> { @@ -592,13 +1781,14 @@ fn scalar_text(value: &Value) -> Result { } } -/// Parses the flat-key YAML file into the nested mapping deserialized by [`GatewayConfig`]. -fn read_config_file(contents: &str) -> Result { +/// Parses the flat-key YAML file into the nested mapping deserialized by [`GatewayConfig`], along with +/// the `gateway.clusters` declaration when the file carries one. +fn read_config_file(contents: &str) -> Result<(Mapping, Option>), ConfigError> { let document: Value = serde_yaml_ng::from_str(contents).map_err(|e| ConfigError::Parse(e.to_string()))?; let mut table = Mapping::new(); if document.is_null() { - return Ok(table); + return Ok((table, None)); } let mapping = document.as_mapping().ok_or_else(|| { ConfigError::Parse( @@ -606,17 +1796,68 @@ fn read_config_file(contents: &str) -> Result { ) })?; + let mut declared = None; for (key, value) in mapping { let key = key .as_str() .ok_or_else(|| ConfigError::Parse("configuration keys must be strings".to_string()))?; - let entry = config_entry(key) - .ok_or_else(|| ConfigError::Parse(format!("unknown configuration key: {key}")))?; - let value = convert_file_value(entry, value) - .map_err(|reason| ConfigError::Parse(format!("{key}: {reason}")))?; - insert_path(&mut table, entry.internal_path, value); + let reason = |reason: String| ConfigError::Parse(format!("{key}: {reason}")); + match resolve_key(key)? { + ResolvedKey::ClusterDeclaration => declared = Some(declared_cluster_ids(value)?), + ResolvedKey::Fixed(entry) => { + let value = convert_file_value(entry, value).map_err(reason)?; + insert_path(&mut table, entry.internal_path, value); + } + ResolvedKey::Cluster { id, entry } => { + let value = convert_file_value(entry, value).map_err(reason)?; + insert_cluster_value(&mut table, &id, entry.internal_path, value); + } + ResolvedKey::ClientOption { id, option } => { + let raw = scalar_text(value).map_err(reason)?; + if matches!( + option.as_str(), + LEGACY_SERVICE_ACCOUNT_OPTION | LEGACY_SERVICE_SECRET_OPTION + ) && raw.trim().is_empty() + { + return Err(ConfigError::Parse(format!("{key}: must not be blank"))); + } + insert_client_option(&mut table, &id, &option, raw); + } + } } - Ok(table) + Ok((table, declared)) +} + +/// Makes every declared cluster exist and rejects any cluster configured without being declared. +/// +/// The declaration is authoritative whether or not it was written: an absent `gateway.clusters` means the +/// single implicit `default` cluster, so a mistyped ID fails startup instead of quietly creating a second +/// cluster that nothing routes to. +fn reconcile_declared_clusters( + table: &mut Mapping, + declared: Option<&[String]>, +) -> Result<(), ConfigError> { + let implicit = [DEFAULT_CLUSTER_ID.to_string()]; + let declared = declared.unwrap_or(&implicit); + for id in declared { + cluster_mapping(table, id); + } + let configured: Vec = table + .get(Value::String("clusters".to_string())) + .and_then(Value::as_mapping) + .into_iter() + .flat_map(Mapping::keys) + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + for id in configured { + if !declared.contains(&id) { + return Err(ConfigError::Parse(format!( + "{CLUSTER_KEY_PREFIX}{id}.* is configured but {id} is not declared in {CLUSTERS_KEY}" + ))); + } + } + Ok(()) } /// Loads configuration from all sources with precedence CLI > env > file > defaults. @@ -628,34 +1869,68 @@ pub fn load( cli: &CliOverrides, ) -> Result { let mut table = Mapping::new(); + let mut declared = None; if let Some(path) = path { let contents = std::fs::read_to_string(path) .map_err(|e| ConfigError::Io(format!("{}: {e}", path.display())))?; - table = read_config_file(&contents)?; + (table, declared) = read_config_file(&contents)?; } // Each override is kept with the source that wrote it, so a failure names what the operator wrote. - let mut overrides: Vec<(&'static str, &'static str, String, Value)> = Vec::new(); - for (key, raw) in env { - if !key.starts_with(ENV_PREFIX) { + let mut overrides: Vec<(String, String, String, Value)> = Vec::new(); + for (variable, raw) in env { + if !variable.starts_with(ENV_PREFIX) { continue; } - let entry = - environment_entry(key).ok_or_else(|| ConfigError::UnknownEnvKey(key.clone()))?; - overrides.push(( - entry.internal_path, - entry.key, - key.clone(), - convert_environment_value(entry, raw) - .map_err(|reason| ConfigError::Parse(format!("{key}: {}: {reason}", entry.key)))?, - )); + match resolve_environment_variable(variable)? { + ResolvedKey::ClusterDeclaration => { + declared = Some(declared_cluster_ids(&Value::String(raw.clone()))?); + } + ResolvedKey::Fixed(entry) => overrides.push(( + entry.internal_path.to_string(), + entry.key.to_string(), + variable.clone(), + convert_environment_value(entry, raw).map_err(|reason| { + ConfigError::Parse(format!("{variable}: {}: {reason}", entry.key)) + })?, + )), + ResolvedKey::Cluster { id, entry } => { + let public_key = format!("{CLUSTER_KEY_PREFIX}{id}.{}", entry.key); + let value = convert_environment_value(entry, raw).map_err(|reason| { + ConfigError::Parse(format!("{variable}: {public_key}: {reason}")) + })?; + overrides.push(( + format!("clusters.{id}.{}", entry.internal_path), + public_key, + variable.clone(), + value, + )); + } + ResolvedKey::ClientOption { id, option } => { + parse_client_option(&option, raw).map_err(|reason| { + ConfigError::Parse(format!( + "{variable}: {CLUSTER_KEY_PREFIX}{id}.client.{option}: {reason}" + )) + })?; + if matches!( + option.as_str(), + LEGACY_SERVICE_ACCOUNT_OPTION | LEGACY_SERVICE_SECRET_OPTION + ) && raw.trim().is_empty() + { + return Err(ConfigError::Parse(format!( + "{variable}: {CLUSTER_KEY_PREFIX}{id}.client.{option} must not be blank" + ))); + } + insert_client_option(&mut table, &id, &option, raw.clone()); + } + } } if let Some(value) = &cli.bind_address { let entry = config_entry(REST_LISTEN_KEY).expect("REST listen option is registered"); overrides.push(( - entry.internal_path, - entry.key, + entry.internal_path.to_string(), + entry.key.to_string(), "--bind-address".to_string(), Value::String(value.clone()), )); @@ -665,6 +1940,8 @@ pub fn load( insert_path(&mut table, path, value.clone()); } + reconcile_declared_clusters(&mut table, declared.as_deref())?; + let config = deserialize_config(Value::Mapping(table)).map_err(|error| { let ConfigError::Parse(message) = error else { unreachable!("deserialization only creates parse errors") @@ -720,7 +1997,18 @@ mod tests { config.server.metrics.bind_address, "127.0.0.1:9095".parse().unwrap() ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); + assert_eq!(config.clusters.len(), 1); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, + IdentityMode::Service + ); + assert_eq!(config.security.authentication, AuthenticationMode::Trust); + assert_eq!(config.request_limits, RequestLimitsConfig::default()); assert!(config.warnings().is_empty()); } @@ -764,12 +2052,24 @@ mod tests { "gateway.rest.lookup.max-keyz: 5\n", "gateway.scan.cursor-ttl: 1m\n", "gateway.tls.cert: /etc/tls.pem\n", + "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", + "gateway.cluster.default.connection.identity: user\n", + "gateway.cluster.default.client.: 1\n", + "gateway.cluster.default: fluss:9123\n", ] { let error = load_file(contents).unwrap_err(); assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); let key = contents.split(':').next().unwrap(); assert!(error.to_string().contains(key), "{key}: {error}"); } + + let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); + assert!( + error + .to_string() + .contains("gateway.cluster.default.request-timeout"), + "got: {error}" + ); } #[test] @@ -839,6 +2139,10 @@ mod tests { "FLUSS_GATEWAY__REST__LISTENN", "FLUSS_GATEWAY__QUERY__ENABLED", "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", + "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", + "FLUSS_GATEWAY__CLUSTER__DEFAULT", + "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER_BATCH_SIZE", ] { let mut env = no_env(); env.insert(key.to_string(), "value".to_string()); @@ -899,6 +2203,10 @@ mod tests { "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), "10s".to_string(), ), + ( + "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), + "16".to_string(), + ), ]); let config = load(None, &env, &CliOverrides::default()).unwrap(); @@ -918,6 +2226,7 @@ mod tests { "127.0.0.1:19095".parse().unwrap() ); assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); + assert_eq!(config.request_limits.lookup_max_keys, 16); } #[test] @@ -934,6 +2243,35 @@ mod tests { .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), "got: {error}" ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), + "soon".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), + "{rendered}" + ); + assert!( + rendered.contains("gateway.cluster.default.connect-timeout"), + "{rendered}" + ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE".to_string(), + "many".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE"), + "{rendered}" + ); + assert!(rendered.contains("writer.batch-size"), "{rendered}"); } #[test] @@ -1138,6 +2476,838 @@ mod tests { assert!(ByteSize::parse("0").is_err()); } + fn cluster<'a>(config: &'a GatewayConfig, id: &str) -> &'a ClusterConfig { + config.clusters.get(id).expect("configured cluster") + } + + #[test] + fn typed_cluster_security_and_request_limit_options_are_loaded() { + let config = load_file( + "gateway.clusters: default, analytics\n\ + gateway.cluster.default.bootstrap.servers: [fluss-1:9123, fluss-2:9123]\n\ + gateway.cluster.default.connection.identity-mode: user\n\ + gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n\ + gateway.cluster.default.client.security.protocol: sasl\n\ + gateway.cluster.default.connection.max: 512\n\ + gateway.cluster.default.connection.idle-timeout: 10m\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123,analytics-2:9123\n\ + gateway.cluster.analytics.connect-timeout: 5s\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n\ + gateway.rest.write.max-rows: 500\n\ + gateway.rest.lookup.max-keys: 32\n\ + gateway.rest.lookup.max-key-bytes: 2MiB\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.bootstrap_servers, ["fluss-1:9123", "fluss-2:9123"]); + assert_eq!(default.identity_mode, IdentityMode::User); + assert_eq!(default.effective_service_account(), Some("gateway_svc")); + assert_eq!(default.effective_service_secret(), Some("gw-pass")); + assert_eq!(default.connection_max, Some(512)); + assert_eq!( + default.connection_idle_timeout.map(ConfigDuration::get), + Some(Duration::from_secs(600)) + ); + // A comma-separated string is accepted so the same value can arrive from the environment. + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + ["analytics:9123", "analytics-2:9123"] + ); + assert_eq!( + cluster(&config, "analytics").connect_timeout.get(), + Duration::from_secs(5) + ); + assert_eq!(config.security.authentication, AuthenticationMode::Password); + assert_eq!(config.request_limits.write_max_rows, 500); + assert_eq!(config.request_limits.lookup_max_keys, 32); + assert_eq!( + config.request_limits.lookup_max_key_bytes.bytes(), + 2 * 1024 * 1024 + ); + } + + /// The declaration bounds which clusters may be configured, and it does so whether or not it was + /// written: with no `gateway.clusters`, the only configurable cluster is the implicit `default`. That is + /// what turns a mistyped cluster ID into a startup failure instead of an unreachable second cluster. + #[test] + fn cluster_declarations_are_authoritative_and_validated() { + for contents in [ + "gateway.clusters: default\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + // No declaration at all: `analytics` is still not one of the allowed clusters. + "gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + "gateway.cluster.analytics.client.writer.batch-size: 2MiB\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("not declared"), + "{contents}: {error}" + ); + } + + // Declaring a cluster is enough to configure it; the rest of its settings default. + let config = load_file("gateway.clusters: default,analytics\n").unwrap(); + assert_eq!(config.clusters.len(), 2); + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + for declaration in [ + "gateway.clusters: default,default\n", + "gateway.clusters: [default, default]\n", + ] { + assert!( + load_file(declaration) + .unwrap_err() + .to_string() + .contains("duplicate cluster ID"), + "{declaration}" + ); + } + + // The implicit default needs no declaration, so a single-cluster deployment configures no list. + let config = load_file("gateway.cluster.default.bootstrap.servers: only:9123\n").unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["default"]); + + { + for contents in [ + "gateway.clusters: Default\n", + "gateway.clusters: 1st\n", + "gateway.clusters: eu-west\n", + "gateway.cluster.EU.bootstrap.servers: eu:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("cluster ID"), + "{contents}: {error}" + ); + } + } + + { + let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__CLUSTERS".to_string(), + "analytics".to_string(), + ); + let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); + + env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("not declared"), "got: {error}"); + } + } + + #[test] + fn native_client_options_are_parsed_and_bounded_by_native_types() { + assert_eq!( + parse_client_option("writer.batch-size", "2MiB").unwrap(), + ClientOptionValue::Bytes(2 * 1024 * 1024) + ); + assert_eq!( + parse_client_option("writer.batch-timeout", "50ms").unwrap(), + ClientOptionValue::Millis(50) + ); + assert_eq!( + parse_client_option("connect-timeout", "2s").unwrap(), + ClientOptionValue::Millis(2_000) + ); + assert!(parse_client_option("connect.timeout", "2s").is_err()); + assert!(parse_client_option("request-timeout", "30s").is_err()); + assert_eq!( + parse_client_option("writer.dynamic-batch-size.enabled", "false").unwrap(), + ClientOptionValue::Boolean(false) + ); + assert_eq!( + parse_client_option("lookup.max-retries", "3").unwrap(), + ClientOptionValue::Integer(3) + ); + assert_eq!( + parse_client_option("security.protocol", "sasl").unwrap(), + ClientOptionValue::Text("sasl".to_string()) + ); + + let config = load_file( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.lookup.max-retries: 3\n", + ) + .unwrap(); + let default = cluster(&config, "default"); + assert_eq!(default.client_option("writer.batch-size"), Some("2MiB")); + assert_eq!(default.client_option("lookup.max-retries"), Some("3")); + + { + for (option, value) in [ + ("lookup.queue-size", "0"), + ("lookup.max-batch-size", "-1"), + ("lookup.max-retries", "-1"), + ("writer.request-max-size", "3GiB"), + ("writer.dynamic-batch-size.enabled", "maybe"), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.starts_with(&format!("client.{option}")), "{error}"); + assert!( + load_file(&format!( + "gateway.cluster.default.client.{option}: \"{value}\"\n" + )) + .is_err(), + "accepted client.{option} = {value}" + ); + } + } + + { + let over_i32 = u64::from(i32::MAX as u32) + 1; + + for option in ["writer.batch-size", "writer.request-max-size"] { + let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must not exceed"), "{option}: {error}"); + } + assert_eq!( + parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), + ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) + ); + for option in [ + "lookup.queue-size", + "lookup.max-batch-size", + "lookup.max-inflight-requests", + ] { + assert_eq!( + parse_client_option(option, &format!("{over_i32}")).unwrap(), + ClientOptionValue::Integer(over_i32), + "{option} is stored as usize and must accept this" + ); + } + let error = + parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("expected a non-negative integer"), "{error}"); + assert_eq!( + parse_client_option("writer.batch-timeout", "0ms").unwrap(), + ClientOptionValue::Millis(0) + ); + assert_eq!( + parse_client_option("lookup.batch-timeout", "0ms").unwrap(), + ClientOptionValue::Millis(0) + ); + } + } + + /// The gateway advertises the write guarantees, so the client options that would weaken them, and the + /// authorization ID that user mode supplies per request, are refused rather than silently honoured. + #[test] + fn client_options_cannot_override_gateway_owned_guarantees_or_the_identity() { + for (option, value, expected) in [ + ("writer.acks", "0", "owned by the Gateway"), + ("writer.retries", "0", "owned by the Gateway"), + ("writer.enable-idempotence", "false", "owned by the Gateway"), + ( + "security.sasl.authorization-id", + "alice", + "cannot be configured statically", + ), + ( + "writer.unknown-knob", + "1", + "is not a supported native-client option", + ), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.contains(expected), "{option}: {error}"); + + let problems = problems( + load_file(&format!( + "gateway.cluster.default.client.{option}: {value}\n" + )) + .unwrap_err(), + ); + assert!( + problems.iter().any(|problem| { + problem.starts_with("gateway.cluster.default.client.") + && problem.contains(option) + }), + "{option}: {problems:?}" + ); + } + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), + "0".to_string(), + )]); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("writer.acks"), "got: {error}"); + } + + #[test] + fn cross_field_cluster_and_security_constraints_fail_before_startup() { + for contents in [ + // An account without its secret, and the reverse. + "gateway.cluster.default.connection.service.account: gateway_svc\n", + "gateway.cluster.default.connection.service.secret: gw-pass\n", + "gateway.cluster.default.connection.max: 0\n", + "gateway.cluster.default.bootstrap.servers: \" \"\n", + // The mode's credential table is missing. + "gateway.security.authentication: password\n", + "gateway.security.authentication: token\n", + "gateway.security.authentication: trusted-header\n\ + gateway.security.trusted-header.name: \"bad header\"\n", + "gateway.rest.lookup.max-keys: 0\n", + "gateway.rest.prefix-lookup.max-prefixes: 0\n", + ] { + assert!(load_file(contents).is_err(), "accepted: {contents}"); + } + + // Directly constructed configurations are subject to the same validation. + let mut config = GatewayConfig::default(); + config.clusters.clear(); + assert!( + problems(config.validate().unwrap_err()) + .iter() + .any(|error| error == "gateway.clusters must declare at least one cluster") + ); + + let mut config = GatewayConfig::default(); + config + .clusters + .get_mut(DEFAULT_CLUSTER_ID) + .expect("default cluster") + .identity_mode = IdentityMode::User; + assert!( + problems(config.validate().unwrap_err()) + .iter() + .any(|error| error.contains("identity-mode user requires")) + ); + } + + /// The legacy SASL options stay usable and keep winning, because silently changing which credential a + /// running deployment authenticates with would be worse than the deprecation. + #[test] + fn warnings_cover_legacy_credentials_and_ignored_pool_settings() { + let config = load_file( + "gateway.clusters: default,analytics\n\ + gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.analytics.client.security.sasl.username: other-user\n\ + gateway.cluster.analytics.client.security.sasl.password: other-secret\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.effective_service_account(), Some("legacy-user")); + assert_eq!(default.effective_service_secret(), Some("legacy-secret")); + + for (legacy, expected_account, expected_secret) in [ + ( + "gateway.cluster.default.client.security.sasl.username: legacy-user\n", + "legacy-user", + "canonical-secret", + ), + ( + "gateway.cluster.default.client.security.sasl.password: legacy-secret\n", + "canonical-user", + "legacy-secret", + ), + ] { + let config = load_file(&format!( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n{legacy}" + )) + .unwrap(); + let cluster = cluster(&config, "default"); + assert_eq!(cluster.effective_service_account(), Some(expected_account)); + assert_eq!(cluster.effective_service_secret(), Some(expected_secret)); + assert_eq!( + config + .warnings() + .iter() + .filter(|warning| warning.contains("is deprecated")) + .count(), + 1 + ); + } + + for option in [LEGACY_SERVICE_ACCOUNT_OPTION, LEGACY_SERVICE_SECRET_OPTION] { + let error = load_file(&format!( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.{option}: \" \"\n" + )) + .unwrap_err(); + assert!(error.to_string().contains(option), "{error}"); + assert!(error.to_string().contains("must not be blank"), "{error}"); + } + + let file = write_temp_config( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: file-user\n", + ); + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD".to_string(), + "env-secret".to_string(), + )]); + let mixed = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + let cluster = cluster(&mixed, "default"); + assert_eq!(cluster.effective_service_account(), Some("file-user")); + assert_eq!(cluster.effective_service_secret(), Some("env-secret")); + + let mut blank_env = env; + blank_env.insert( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD".to_string(), + " ".to_string(), + ); + let error = load(Some(file.path()), &blank_env, &CliOverrides::default()).unwrap_err(); + assert!( + error + .to_string() + .contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD"), + "{error}" + ); + + let deprecations: Vec = config + .warnings() + .into_iter() + .filter(|warning| warning.contains("is deprecated")) + .collect(); + assert_eq!(deprecations.len(), 2, "one per cluster: {deprecations:?}"); + for warning in &deprecations { + for secret in [ + "canonical-user", + "canonical-secret", + "legacy-user", + "legacy-secret", + "other-user", + "other-secret", + ] { + assert!(!warning.contains(secret), "leaked {secret}: {warning}"); + } + } + + { + let config = load_file( + "gateway.cluster.default.connection.max: 8\n\ + gateway.cluster.default.connection.idle-timeout: 5m\n", + ) + .unwrap(); + assert!( + config.warnings().iter().any(|warning| warning + .contains("ignored because connection.identity-mode is service")), + "{:?}", + config.warnings() + ); + } + } + + #[test] + fn diagnostics_redact_every_credential_and_keep_the_identities() { + let config = load_file( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.default.client.writer.batch-size: 4MiB\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:user-secret\n\ + gateway.security.tokens: token-secret:alice\n", + ) + .unwrap(); + + for diagnostic in [config.redacted_debug(), format!("{config:?}")] { + for credential in [ + "canonical-secret", + "legacy-secret", + "user-secret", + "token-secret", + ] { + assert!( + !diagnostic.contains(credential), + "leaked {credential}: {diagnostic}" + ); + } + // The identities and the tuning stay readable: they are what an operator came to look at. + for readable in ["canonical-user", "legacy-user", "4MiB"] { + assert!(diagnostic.contains(readable), "{readable}: {diagnostic}"); + } + assert!(diagnostic.contains(REDACTED), "{diagnostic}"); + } + + // The credentials are still reachable by the components that authenticate with them. + assert_eq!( + config.security.users.as_ref().map(Secret::expose), + Some("alice:user-secret") + ); + assert_eq!( + cluster(&config, "default").effective_service_secret(), + Some("legacy-secret") + ); + + // Startup errors name the bad option without quoting credentials from the same input. + let error = load_file( + "gateway.security.authentication: token\n\ + gateway.security.tokens: do-not-leak\n\ + gateway.cluster.default.connection.service.secret: also-secret\n\ + gateway.cluster.default.client.writer.acks: 0\n", + ) + .unwrap_err() + .to_string(); + assert!(error.contains("writer.acks"), "{error}"); + assert!(!error.contains("do-not-leak"), "{error}"); + assert!(!error.contains("also-secret"), "{error}"); + } + + /// Every dynamic cluster/client option follows environment-over-file precedence. + #[test] + fn the_environment_overrides_the_file_for_every_cluster_and_client_option() { + // Keeps every variant loadable: user identity mode needs usable credentials over SASL. The key + // under test is removed from the base so the file never carries it twice. + let base = [ + ("connection.service.account", "base-account"), + ("connection.service.secret", "base-secret"), + ("client.security.protocol", "sasl"), + ]; + let security = "gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n"; + + let mut cases: Vec<(String, String, &str, &str)> = Vec::new(); + for entry in CLUSTER_ENTRIES { + let values = match entry.kind { + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Integer => ("11", "22"), + ValueKind::Text if entry.key == "connection.identity-mode" => ("service", "user"), + ValueKind::Text if entry.key.ends_with("timeout") => ("11s", "22s"), + ValueKind::Text => ("file-value", "env-value"), + ValueKind::Bool | ValueKind::Bytes => { + unreachable!("no per-cluster option uses {:?}", entry.kind) + } + }; + cases.push(( + entry.key.to_string(), + environment_suffix(entry.key), + values.0, + values.1, + )); + } + for spec in CLIENT_OPTIONS { + let values = match spec.kind { + // The only legal value is PLAIN, so the two spellings differ only in case. + _ if spec.option == "security.sasl.mechanism" => ("PLAIN", "plain"), + _ if spec.option == "security.protocol" => ("plaintext", "sasl"), + ClientOptionKind::Text | ClientOptionKind::Secret => ("file-value", "env-value"), + ClientOptionKind::Boolean => ("true", "false"), + ClientOptionKind::I32Count { .. } | ClientOptionKind::UsizeCount { .. } => { + ("5", "6") + } + ClientOptionKind::I32Bytes if spec.option.ends_with("dynamic-batch-size.min") => { + ("1MiB", "2MiB") + } + ClientOptionKind::I32Bytes | ClientOptionKind::UsizeBytes => ("4MiB", "8MiB"), + ClientOptionKind::I64Millis | ClientOptionKind::U64Millis => ("11s", "22s"), + }; + cases.push(( + format!("{CLIENT_OPTION_PREFIX}{}", spec.option), + format!("CLIENT__{}", environment_suffix(spec.option)), + values.0, + values.1, + )); + } + + for (key, env_suffix, file_value, env_value) in cases { + let contents = |value: &str| { + base.iter() + .filter(|(base_key, _)| *base_key != key) + .map(|(base_key, base_value)| (*base_key, *base_value)) + .chain([(key.as_str(), value)]) + .map(|(key, value)| { + format!("{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{key}: \"{value}\"\n") + }) + .collect::() + }; + let load_valid = |contents: &str, env: &BTreeMap| { + let file = write_temp_config(&format!("{security}{contents}")); + load(Some(file.path()), env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{key}: {error}\n{contents}")) + }; + + let env = BTreeMap::from([( + format!( + "{ENV_PREFIX}CLUSTER__{}__{env_suffix}", + DEFAULT_CLUSTER_ID.to_ascii_uppercase() + ), + env_value.to_string(), + )]); + let from_file = load_valid(&contents(file_value), &no_env()); + let overridden = load_valid(&contents(file_value), &env); + let as_written = load_valid(&contents(env_value), &no_env()); + + assert_ne!( + from_file, overridden, + "{key} ignores its environment variable" + ); + assert_eq!( + overridden, as_written, + "{key} from the environment differs from the same value in the file" + ); + } + } + + /// Registry invariants keep public keys unique, non-overlapping, and environment-compatible. + #[test] + fn option_registries_are_safe_and_environment_compatible() { + for spec in CLIENT_OPTIONS { + let suffix = environment_suffix(spec.option); + assert_eq!( + environment_suffix_to_option(&suffix), + spec.option, + "{} does not survive the environment mapping", + spec.option + ); + assert!( + !spec.option.contains('_'), + "{} must spell words with '-', which the environment mapping reserves '_' for", + spec.option + ); + let file_key = format!( + "{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{CLIENT_OPTION_PREFIX}{}", + spec.option + ); + assert!( + matches!(resolve_key(&file_key), Ok(ResolvedKey::ClientOption { .. })), + "{file_key} is not reachable as a file key" + ); + } + + { + for spec in CLIENT_OPTIONS { + let looks_sensitive = ["password", "secret", "token"] + .iter() + .any(|part| spec.option.contains(part)); + assert_eq!( + spec.kind.is_sensitive(), + looks_sensitive, + "{}: the kind and the name disagree about being a credential", + spec.option + ); + assert!( + !RESERVED_CLIENT_OPTIONS + .iter() + .any(|(reserved, _)| *reserved == spec.option), + "{} is both allowed and refused", + spec.option + ); + } + // An option the gateway never validated must not be rendered on the chance that it holds a secret. + assert!(client_option_is_sensitive("some.unknown.option")); + assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); + assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); + + // A parsed credential stays wrapped, so printing the parse result cannot leak it either. + let parsed = + parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); + assert_eq!( + parsed, + ClientOptionValue::Secret(Secret::new("legacy-secret")) + ); + assert!(!format!("{parsed:?}").contains("legacy-secret")); + assert!(format!("{parsed:?}").contains(REDACTED)); + } + + { + let mut keys = std::collections::BTreeSet::new(); + let mut fields = std::collections::BTreeSet::new(); + let mut suffixes = std::collections::BTreeSet::new(); + + for entry in CLUSTER_ENTRIES { + assert!(!entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + !entry.key.starts_with(CLIENT_OPTION_PREFIX), + "{} collides with the client namespace", + entry.key + ); + assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); + assert!( + fields.insert(entry.internal_path), + "duplicate field: {}", + entry.internal_path + ); + assert!( + suffixes.insert(environment_suffix(entry.key)), + "duplicate environment suffix for {}", + entry.key + ); + } + + assert_eq!(CLUSTER_ENTRIES.len(), 8); + } + } + + /// Precedence is stated once for the whole vocabulary rather than sampled on one key, so a per-kind + /// conversion that only reads one source cannot hide: every option is driven from the file, then + /// overridden from the environment, and the environment value has to win in the loaded config. + #[test] + fn the_environment_overrides_the_file_for_every_option() { + for entry in CONFIG_ENTRIES { + let (file_value, env_value) = match entry.kind { + ValueKind::Bool => ("true", "false"), + ValueKind::Integer => ("11", "22"), + ValueKind::Bytes => ("1MiB", "2MiB"), + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Text => match entry.key { + REST_LISTEN_KEY => ("127.0.0.1:11111", "127.0.0.1:22222"), + METRICS_LISTEN_KEY => ("127.0.0.1:11112", "127.0.0.1:22223"), + REST_HEADER_READ_TIMEOUT_KEY + | REST_REQUEST_TIMEOUT_KEY + | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"), + // Both modes must be valid on their own: the file value is loaded without the + // environment override, and password and token modes need a credential table. + SECURITY_AUTHENTICATION_KEY => ("trusted-header", "trust"), + _ => ("file-value", "env-value"), + }, + }; + + let file = write_temp_config(&format!("{}: \"{file_value}\"\n", entry.key)); + let from_file = load(Some(file.path()), &no_env(), &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + let env = BTreeMap::from([(environment_variable(entry.key), env_value.to_string())]); + let from_env = load(Some(file.path()), &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + + assert_ne!( + from_file, from_env, + "{} ignores its environment variable", + entry.key + ); + let only_env = load(None, &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + assert_eq!( + from_env, only_env, + "{} lets the file value survive the environment override", + entry.key + ); + } + } + + /// User identity requires verified callers and usable service credentials over SASL. + #[test] + fn user_identity_mode_requires_usable_credentials_over_sasl() { + let user_mode = "gateway.cluster.default.connection.identity-mode: user\n"; + let credentials = "gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n"; + let sasl = "gateway.cluster.default.client.security.protocol: sasl\n"; + + for (contents, expected) in [ + ( + format!("{user_mode}{credentials}"), + "requires client.security.protocol sasl", + ), + ( + format!("{user_mode}{sasl}"), + "requires connection.service.account", + ), + ( + format!( + "{user_mode}{sasl}gateway.cluster.default.connection.service.account: \"\"\n\ + gateway.cluster.default.connection.service.secret: \" \"\n" + ), + "must not be blank", + ), + ( + format!( + "{user_mode}{sasl}{credentials}\ + gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n" + ), + "must be PLAIN", + ), + ] { + let problems = problems(load_file(&contents).unwrap_err()); + assert!( + problems.iter().any(|problem| problem.contains(expected)), + "expected {expected:?} for:\n{contents}got: {problems:?}" + ); + } + + assert!( + problems(load_file(&format!("{user_mode}{sasl}{credentials}")).unwrap_err()) + .iter() + .any(|problem| problem.contains("verified client identities")) + ); + assert!( + load_file(&format!( + "gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n{user_mode}{sasl}{credentials}" + )) + .is_ok() + ); + // Service mode needs no SASL: it authenticates as itself, with no principal to propagate. + assert!(load_file("gateway.cluster.default.connection.identity-mode: service\n").is_ok()); + } + + /// Native writer size relationships apply to configured values and native defaults. + #[test] + fn writer_size_pairs_must_fit_including_against_the_native_defaults() { + for (contents, rejected) in [ + // Both sides configured. + ( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.batch-size: 65MiB\n\ + gateway.cluster.default.client.writer.buffer.memory-size: 64MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 4MiB\n\ + gateway.cluster.default.client.writer.batch-size: 2MiB\n", + true, + ), + // One side only: the other is the native default, 10MiB request-max and 2MiB batch-size. + ( + "gateway.cluster.default.client.writer.batch-size: 128MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 3MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + // Coherent against the defaults, and coherent as a pair. + ( + "gateway.cluster.default.client.writer.batch-size: 4MiB\n", + false, + ), + ( + "gateway.cluster.default.client.writer.batch-size: 32MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 64MiB\n", + false, + ), + ] { + let result = load_file(contents); + assert_eq!( + result.is_err(), + rejected, + "unexpected outcome for:\n{contents}" + ); + if rejected { + assert!( + problems(result.unwrap_err()) + .iter() + .any(|problem| problem.contains("must be <=")), + "{contents}" + ); + } + } + } + #[test] fn options_are_complete_and_unambiguous() { let mut public_keys = std::collections::BTreeSet::new(); @@ -1163,6 +3333,6 @@ mod tests { ); } - assert_eq!(CONFIG_ENTRIES.len(), 8); + assert_eq!(CONFIG_ENTRIES.len(), 20); } } diff --git a/fluss-gateway/src/lifecycle.rs b/fluss-gateway/src/lifecycle.rs index 03a304e1c3..4d810304e8 100644 --- a/fluss-gateway/src/lifecycle.rs +++ b/fluss-gateway/src/lifecycle.rs @@ -245,6 +245,9 @@ pub async fn start(config: GatewayConfig) -> Result { /// Binds the listeners, installs the router, and spawns every process-owned task. async fn start_internal(config: GatewayConfig) -> Result { + // The effective configuration is what an operator needs when a deployment misbehaves, and the + // redacted rendering is the only form of it that is safe to write to a log. + log::debug!("effective configuration: {}", config.redacted_debug()); for warning in config.warnings() { log::warn!("{warning}"); } diff --git a/fluss-gateway/tests/process.rs b/fluss-gateway/tests/process.rs index 80ca730d99..346346ad5b 100644 --- a/fluss-gateway/tests/process.rs +++ b/fluss-gateway/tests/process.rs @@ -23,6 +23,7 @@ mod support; +use std::io::Read; use std::time::Duration; use support::{ChildGuard, await_http_ok, binary, free_port, write_config}; @@ -40,6 +41,98 @@ async fn an_invalid_configuration_fails_before_binding_with_exit_code_2() { ); } +/// A rejected `client.*` option is what an operator hits when migrating a native-client configuration, and +/// the file that carries it usually carries credentials too: the process must name the option on stderr +/// while leaking none of them. +#[tokio::test] +async fn a_restricted_client_option_fails_before_binding_without_leaking_credentials() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("gateway.yaml"); + std::fs::write( + &path, + "gateway.cluster.default.connection.service.account: gateway-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.security.authentication: token\n\ + gateway.security.tokens: token-secret:alice\n\ + gateway.cluster.default.client.writer.acks: 0\n", + ) + .expect("write"); + + let output = binary().arg("--config").arg(&path).output().expect("run"); + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("writer.acks"), "{stderr}"); + for credential in ["canonical-secret", "legacy-secret", "token-secret"] { + assert!( + !stderr.contains(credential), + "stderr leaked {credential}: {stderr}" + ); + } +} + +/// The deprecated SASL options keep working, so the only signal an operator gets is the startup warning: +/// it has to reach stderr, once per cluster, with the credentials redacted. Debug logging is on, so the +/// effective-configuration dump is on stderr too and is held to the same rule. +#[tokio::test] +async fn legacy_credentials_emit_one_redacted_warning_per_cluster() { + let dir = tempfile::tempdir().expect("tempdir"); + let port = free_port(); + let path = dir.path().join("gateway.yaml"); + std::fs::write( + &path, + format!( + "gateway.rest.listen: 127.0.0.1:{port}\n\ + gateway.metrics.enabled: false\n\ + gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.security.authentication: token\n\ + gateway.security.tokens: token-secret:alice\n" + ), + ) + .expect("write"); + + // Only the few startup log lines are piped, so the pipe buffer cannot fill and stall the child. + let child = binary() + .arg("--config") + .arg(&path) + .env("RUST_LOG", "debug") + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn"); + let mut guard = ChildGuard(child); + assert!( + await_http_ok( + &format!("http://127.0.0.1:{port}/health"), + Duration::from_secs(15) + ) + .await, + "health" + ); + guard.send_sigterm(); + let status = guard.wait_for_exit(Duration::from_secs(35)).await; + assert_eq!(status.code(), Some(0)); + + let mut stderr = String::new(); + guard + .0 + .stderr + .take() + .expect("piped stderr") + .read_to_string(&mut stderr) + .expect("read stderr"); + assert_eq!(stderr.matches("is deprecated").count(), 1, "{stderr}"); + assert!(stderr.contains("effective configuration"), "{stderr}"); + for credential in ["canonical-secret", "legacy-secret", "token-secret"] { + assert!( + !stderr.contains(credential), + "stderr leaked {credential}: {stderr}" + ); + } +} + #[tokio::test] async fn the_binary_starts_serves_health_and_drains_on_sigterm_with_exit_code_0() { let dir = tempfile::tempdir().expect("tempdir");