diff --git a/Cargo.lock b/Cargo.lock index dabb520b..63efa90f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "annotate-snippets" -version = "0.12.17" +version = "0.12.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a5b0a5ab8561c603f16c3b78c24a386b1f70e7fc1b1b1e8c15738828e4a176" +checksum = "0258346f92c78dbc5b336b9a5b27c64447aa10643bc64e651db2edb145361564" dependencies = [ "anstyle", "memchr", @@ -114,18 +114,18 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "ar_archive_writer" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +checksum = "523e5e187fff36fcb8d1d19ccf0ca4ca43ec3d158316057f66dc6f72716bc5ec" dependencies = [ "object", ] [[package]] name = "arc-swap" -version = "1.9.3" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" dependencies = [ "rustversion", ] @@ -178,7 +178,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -280,12 +280,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.2" @@ -299,6 +293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -313,21 +308,21 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.3" +version = "3.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +checksum = "87a0cf66fd1eb2fb7b822cb78370d82303750809bc6ba855f3f58e517d3238db" [[package]] name = "bytes" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "776dfebd015e2d8732c684a2c71003674ccb49457b539b9e82151ba38e96dd7c" [[package]] name = "cc" -version = "1.11.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" dependencies = [ "find-msvc-tools", "jobserver", @@ -365,9 +360,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +checksum = "ac230478587d0269d8d6998b82ae69ea51509d55df66d9e92edd1f986904cd4a" dependencies = [ "clap_builder", "clap_derive", @@ -387,21 +382,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +checksum = "895c9d8d2ac2dd8b8a29f3724a14e72aaaa6debcf8c38c82f97305c6a172767a" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "clap_lex" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +checksum = "e0921b3876b3a1df8a1c9f912e7f94342458022c2e84288015893d83c4d62a79" [[package]] name = "cmov" @@ -427,9 +422,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" [[package]] name = "const_format" @@ -500,9 +495,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" dependencies = [ "cfg-if", ] @@ -524,9 +519,9 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-bigint" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" +checksum = "914c195c4d7fbc5ed7f88d14cbf85914d5c664967658d43404b263913a477e82" dependencies = [ "cpubits", "ctutils", @@ -590,7 +585,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -601,38 +596,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.9", -] - -[[package]] -name = "defmt" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "defmt-parser" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" -dependencies = [ - "thiserror 2.0.21", + "syn 3.0.10", ] [[package]] @@ -707,6 +671,7 @@ dependencies = [ "const-oid", "crypto-common", "ctutils", + "zeroize", ] [[package]] @@ -717,7 +682,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -763,14 +728,14 @@ dependencies = [ [[package]] name = "educe" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebdb1f8f8d2815bbe7b369a7da10343c82026a4503b68bd1279ba557611e96d" +checksum = "a00371d2607edea062396ef1ce6ade65c8e915ac7a1277daa198208420f682c6" dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -849,7 +814,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -860,9 +825,9 @@ checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776" dependencies = [ "libc", "windows-sys 0.61.2", @@ -918,9 +883,9 @@ checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" dependencies = [ "crc32fast", "miniz_oxide", @@ -1010,7 +975,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -1091,7 +1056,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.2", + "bitflags", "libc", "libgit2-sys", "log", @@ -1117,9 +1082,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" +checksum = "9bd04d0d9c7079b416b458d2317aea4aa964c81938cb7c20f559758c4f422dc8" dependencies = [ "arraydeque", "smallvec", @@ -1501,15 +1466,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.15.0" @@ -1521,9 +1477,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" [[package]] name = "java-properties" @@ -1538,13 +1494,26 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.39" +version = "0.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac703058abac89e5c39b0b879dce2b6b14a4ba36600c7e7e3d0c60a6883ef3c" +checksum = "175dc66334db3b9ab9985bf8b61cfed50dc16d20964bb47f9e479adcae984a7f" dependencies = [ - "defmt", - "jiff-core", - "jiff-static", + "jiff-core 0.1.8", + "jiff-static 0.2.46", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5d126d72eb017f10b302e73bdeca446bea405ee449af4d9e7866d6b082d83e" +dependencies = [ + "jiff-core 1.0.1", + "jiff-static 1.0.1", "jiff-tzdb-platform", "log", "portable-atomic", @@ -1555,37 +1524,57 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.2" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e599b80292e597019b8b3e392de3f4bea2959ddfb8c7dff60e035e9e311a2c1b" +dependencies = [ + "log", +] + +[[package]] +name = "jiff-core" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee43309eb53bafb8c36d35d6e444769c8f8e6206a9fbf7546f5db9d7d4a0e93" +checksum = "b6b46a105a9874634dd96231021526bf24ca04955a5998c59a9c7666e634b7b8" dependencies = [ - "defmt", "log", ] [[package]] name = "jiff-static" -version = "0.2.39" +version = "0.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc32470c67eb7d0e9ccb74303fa925313dc04cecb1aa048beaa6f391e36a93d" +checksum = "c7b1af7547d47354bc438d797e3854601bde8f8d185780c5ba4ab27afa54f1ba" dependencies = [ - "jiff-core", + "jiff-core 0.1.8", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "jiff-static" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0127e9bde3c6130d6169e7d8d68676e4be69c4bf7573c308b6f960a5089c428" +dependencies = [ + "jiff-core 1.0.1", + "proc-macro2", + "quote", + "syn 3.0.10", +] + [[package]] name = "jiff-tzdb" -version = "0.1.9" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" +checksum = "12e71a1b68a875a44a33bb7a1dd84392cb77145a3367a552a3cfe6a4f5cf3afc" [[package]] name = "jiff-tzdb-platform" -version = "0.1.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +checksum = "dc65b284afcfa8f6cf189c7274c805505314a7ab975c60e1f548bff56525420e" dependencies = [ "jiff-tzdb", ] @@ -1675,9 +1664,9 @@ dependencies = [ [[package]] name = "jsonpath-rust" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c4668c489390e66d586fe2238ee8568473f7ac1607b411342f787c5e1bedd7" +checksum = "c83bc96c838e7a3de9dc89ea291363ac073371959637e3b2207556e9320547b8" dependencies = [ "pest", "pest_derive", @@ -1703,7 +1692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab0d13801242174b1f3966907c3f32039f2d6b36c7ce82c61225d66ed5d1eb70" dependencies = [ "base64 0.22.1", - "jiff", + "jiff 0.2.46", "schemars", "serde", "serde_json", @@ -1712,7 +1701,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "darling", "regex", @@ -1736,9 +1725,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "kube" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c73b082f549411623a4f8949bd24dcaedacb1511379c461c76d579fde301d021" +checksum = "ee2daae7debf056a191329a20b5ba6f6de6d437e87d14e586453633b3de04f7f" dependencies = [ "k8s-openapi", "kube-client", @@ -1749,9 +1738,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae56b223bca490db1f1b115bfc606d411177511d7748d8cc019a7e3b2935430b" +checksum = "23c0cbc565d35cd1e1997537a22d386abedc62cc2e7e70f91c08f9928012d845" dependencies = [ "base64 0.23.1", "bytes", @@ -1764,7 +1753,7 @@ dependencies = [ "hyper-rustls", "hyper-timeout", "hyper-util", - "jiff", + "jiff 0.2.46", "jsonpath-rust", "k8s-openapi", "kube-core", @@ -1786,14 +1775,14 @@ dependencies = [ [[package]] name = "kube-core" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de84b47fc1aa35ba0ef9059ef54b47c8b47f0979afd15487e525282f2d73fb9" +checksum = "ca27486a6fbf5e74ec7b411faf1d5bab8a3e489b93f958391c698f27d693014f" dependencies = [ "derive_more", "form_urlencoded", "http", - "jiff", + "jiff 0.2.46", "json-patch", "k8s-openapi", "schemars", @@ -1805,23 +1794,23 @@ dependencies = [ [[package]] name = "kube-derive" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de1fb46ac34371491ac0277c13e0e1bb9288000e64c86ca3d91343754818e64e" +checksum = "b9f4ad02a27b7385ca9da44d83788aca4f11feeebf2ee57ded3003037051019f" dependencies = [ "darling", "proc-macro2", "quote", "serde", "serde_json", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "kube-runtime" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dea60005024b6ec1819c426209aa29b9eaca64f6b3988ab73c55f43fa484392" +checksum = "aa7b047ec6a5081f57fcd656308d640a9796d7684f56cc01a7a3131f92050478" dependencies = [ "ahash", "async-broadcast", @@ -1834,7 +1823,7 @@ dependencies = [ "json-patch", "k8s-openapi", "kube-client", - "parking_lot", + "parking_lot 0.13.0", "pin-project", "serde", "serde_json", @@ -1876,9 +1865,9 @@ checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" [[package]] name = "libz-sys" -version = "1.1.31" +version = "1.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" +checksum = "fee3f59f4321dab0c78d828942e7aa09d0bd8ad6e78a501977e6bf3d02938045" dependencies = [ "cc", "libc", @@ -1901,6 +1890,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d91b22da179613dcf4d6ec8349563f52a363f6849e03b00dd0a185ec4a59af" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.35" @@ -1930,9 +1928,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "d5848f5a9e8117cf94f2f90faac24a87ff81600ff8628c1e3deaab572e76bfa1" [[package]] name = "miniz_oxide" @@ -1946,9 +1944,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" dependencies = [ "libc", "wasi", @@ -2148,8 +2146,18 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.14", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cf937ca4e1d0b7578784559063422fc39ae7c5a00f02471f2614d5486fd226" +dependencies = [ + "lock_api 0.5.0", + "parking_lot_core 0.10.0", ] [[package]] @@ -2160,7 +2168,19 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "parking_lot_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfff060068cc18aa7815010fcaf80c68c7fb9e85fc7f5c0f9bf142c21d19826" +dependencies = [ + "libc", + "redox_syscall 0.9.5", "smallvec", "windows-link", ] @@ -2192,9 +2212,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.5" +version = "2.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8db75153506001cb36ed924f08cadc862eb3f8c4d0d534139f29074769d7660" +checksum = "5ec53fd1d256900a9a771bf2720d42cef6f8bc91b4e25065fbd291e3e7222576" dependencies = [ "memchr", "pest_stack_guard", @@ -2203,9 +2223,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.5" +version = "2.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20aa17e11fb35664b67d5214a0afac57a360d231b6ec53d6eba0700b27d933b4" +checksum = "adcf794fd301ddc20818a6b4de85ed507f4b45300245cec948982e69e12e3bd7" dependencies = [ "pest", "pest_generator", @@ -2213,22 +2233,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.5" +version = "2.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ec5afccf44ccbfeb2b232316510bb55b4bf34d891afcb0fb07d2b17d7fe9d" +checksum = "c3073afae5db334c25dea9fc3e230a4f191e920c1df080cb9c0b37c9ce1dbb41" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "pest_meta" -version = "2.9.5" +version = "2.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3f45b6caa65c1b08186832745159c2529994146340a7bf44168095469720f5d" +checksum = "7bc7f614b3742541642af1685faaa910bf6333e3cd3729c2cccd118146a57327" dependencies = [ "pest", ] @@ -2381,9 +2401,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +checksum = "3ef0b569d1d907ab3a28596aee62cf1f61dacb0529a4bedcd8e7460e2d5228b4" dependencies = [ "bytes", "prost-derive", @@ -2391,12 +2411,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +checksum = "1ad9c5423459fa2f729b8d9235fab810aa29628b171467744fc86d21460ad70d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools", "proc-macro2", "quote", "syn 2.0.119", @@ -2404,9 +2424,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +checksum = "f29a7c4c7a84cef41a53d338a2d10d4f1b043ea5c3dde26db22ee2c9c1b1c75b" dependencies = [ "prost", ] @@ -2454,9 +2474,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -2494,7 +2514,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.2", + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags", ] [[package]] @@ -2514,14 +2543,14 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "regex" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +checksum = "9807af0e638c12e753aa1b4e601791bf2a8f71274023c8bbeec12fc149950857" dependencies = [ "aho-corasick", "memchr", @@ -2531,9 +2560,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "41d60ef07df457506a0a1a2073112eb4aae5733dbd2813c3da77d1b0fd441158" dependencies = [ "aho-corasick", "memchr", @@ -2542,9 +2571,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +checksum = "d902046e6a28a1e4c3cab97258a0ce11e994903f01c2d97a9447a0b3df85f4f6" [[package]] name = "relative-path" @@ -2609,14 +2638,16 @@ dependencies = [ [[package]] name = "rsa" -version = "0.10.0-rc.19" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33405e321e16ef941ca66e254f6904c3aafd3c7ff001fb88bf11c73eb75c2b85" +checksum = "23e77fd77e0dea28b3011fd62d15445b7de7959333b0276d930e03be0f755699" dependencies = [ "const-oid", "crypto-bigint", + "crypto-common", "crypto-primes", "digest", + "hmac", "pkcs1", "pkcs8", "rand_core 0.10.1", @@ -2693,9 +2724,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" dependencies = [ "zeroize", ] @@ -2729,9 +2760,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.16" +version = "0.103.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" dependencies = [ "ring", "rustls-pki-types", @@ -2791,7 +2822,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2829,7 +2860,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -2868,9 +2899,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "969cde0349344dbbf58b5f9badbfb8f806db13969364eb8007c057ef687007d9" dependencies = [ "annotate-snippets", "base64 0.23.1", @@ -2910,7 +2941,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2921,7 +2952,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -3057,9 +3088,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.4" +version = "1.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" +checksum = "3244542535d5de959d90ec55d1b5ff0fd2bb7b1310223907d5f11932847f7b59" [[package]] name = "snafu" @@ -3132,14 +3163,14 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.5.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "const-oid", "ecdsa", "k8s-openapi", "kube", "p256", - "rand 0.10.3", + "rand 0.10.4", "rand_core 0.10.1", "rsa", "sha2", @@ -3174,13 +3205,14 @@ dependencies = [ "strum", "tokio", "tracing", + "url", "yaml_serde", ] [[package]] name = "stackable-operator" version = "0.122.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "base16ct", "base64 0.23.1", @@ -3194,11 +3226,11 @@ dependencies = [ "http", "indexmap", "java-properties", - "jiff", + "jiff 1.0.1", "json-patch", "k8s-openapi", "kube", - "rand 0.10.3", + "rand 0.10.4", "regex", "schemars", "semver", @@ -3225,20 +3257,20 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "darling", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "stackable-shared" version = "0.2.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ - "jiff", + "jiff 1.0.1", "k8s-openapi", "kube", "schemars", @@ -3253,7 +3285,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.6" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "axum", "clap", @@ -3277,7 +3309,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.14.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "kube", "schemars", @@ -3290,24 +3322,24 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.14.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "convert_case", "darling", "indoc", - "itertools 0.15.0", + "itertools", "k8s-openapi", "k8s-version", "kube", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] name = "stackable-webhook" version = "0.11.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#837182338040c290701232ea3bcb4fb1d71f635a" dependencies = [ "arc-swap", "async-trait", @@ -3320,7 +3352,7 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.10.3", + "rand 0.10.4", "serde", "serde_json", "snafu 0.9.2", @@ -3412,9 +3444,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" dependencies = [ "proc-macro2", "quote", @@ -3438,7 +3470,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -3478,7 +3510,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -3560,7 +3592,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3576,7 +3608,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -3617,18 +3649,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.18+spec-1.1.0" +version = "0.25.19+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec6e78bd70db9420d8d212fa9dc97f116448a1ceff5d00eabb991d407e21396" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" dependencies = [ "indexmap", "toml_datetime", @@ -3721,7 +3753,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.2", + "bitflags", "bytes", "futures-util", "http", @@ -3740,7 +3772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ "base64 0.22.1", - "bitflags 2.13.2", + "bitflags", "bytes", "http", "http-body", @@ -3883,9 +3915,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +checksum = "8f20bdbfc9915686faa18fb5bbe8b4552778505defbf9f12289beefbc44092a6" [[package]] name = "unicode-ident" @@ -3953,9 +3985,9 @@ checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" [[package]] name = "uuid" -version = "1.29.0" +version = "1.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +checksum = "3c14b15dffa2f4bd712103e857140aa04a4f2ff6e955c77d7eb8e53ee02a3b64" dependencies = [ "js-sys", "wasm-bindgen", @@ -4055,7 +4087,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "wasm-bindgen-shared", ] @@ -4090,9 +4122,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" dependencies = [ "rustls-pki-types", ] @@ -4249,9 +4281,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" dependencies = [ "memchr", ] @@ -4332,7 +4364,7 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "synstructure", ] @@ -4373,15 +4405,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +checksum = "85ccff8a4805aba1eaa4cd4afcd49f7557dd2cf5fd79ffc37f8959466effc5aa" dependencies = [ "zeroize_derive", ] @@ -4427,7 +4459,7 @@ checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index 5175eb5f..53d4866f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -221,9 +221,9 @@ rec { }; "annotate-snippets" = rec { crateName = "annotate-snippets"; - version = "0.12.17"; + version = "0.12.18"; edition = "2024"; - sha256 = "0xm1whl8hwqmihg1n6zwwxq1ysrq9b17hfvcy41wcqc5mfjv19dj"; + sha256 = "0r0m6r2v3vdj3mjlxiivch8alis4qqkmp6kb6ddvr3f7j9pk8n02"; libName = "annotate_snippets"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -387,9 +387,9 @@ rec { }; "ar_archive_writer" = rec { crateName = "ar_archive_writer"; - version = "0.5.3"; + version = "0.5.5"; edition = "2024"; - sha256 = "1dhjwdapx0ydx38r7aj8w4rqmdhxs1xl2zp8xala0h11zzg5ikbk"; + sha256 = "1v65ddqp4vywcrzha5l32lyyqhyalh6cz76is6wgqdpzgwc5wgjj"; license = "Apache-2.0 WITH LLVM-exception"; dependencies = [ { @@ -407,13 +407,14 @@ rec { features = [ "write" "xcoff" ]; } ]; - + features = { + }; }; "arc-swap" = rec { crateName = "arc-swap"; - version = "1.9.3"; + version = "1.9.5"; edition = "2018"; - sha256 = "0vmfc2wjnn7gyjfawvnq6a5pxsbiq1rbzzmra0p6969vv6rn429q"; + sha256 = "199jy040f53mq2a2mr13i2l8vjqa68yv781p7pl6v7r0a5fjp8l4"; libName = "arc_swap"; license = "MIT OR Apache-2.0"; authors = [ @@ -551,7 +552,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; usesDefaultFeatures = false; features = [ "clone-impls" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -927,23 +928,7 @@ rec { }; resolvedDefaultFeatures = [ "alloc" ]; }; - "bitflags 1.3.2" = rec { - crateName = "bitflags"; - version = "1.3.2"; - edition = "2018"; - sha256 = "12ki6w8gn1ldq7yz9y680llwk5gmrhrzszaa17g1sbrw2r2qvwxy"; - license = "MIT/Apache-2.0"; - authors = [ - "The Rust Project Developers" - ]; - features = { - "compiler_builtins" = [ "dep:compiler_builtins" ]; - "core" = [ "dep:core" ]; - "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "bitflags 2.13.2" = rec { + "bitflags" = rec { crateName = "bitflags"; version = "2.13.2"; edition = "2021"; @@ -974,10 +959,17 @@ rec { name = "hybrid-array"; packageId = "hybrid-array"; } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "zeroize" = [ "dep:zeroize" ]; }; + resolvedDefaultFeatures = [ "zeroize" ]; }; "built" = rec { crateName = "built"; @@ -1015,9 +1007,9 @@ rec { }; "bumpalo" = rec { crateName = "bumpalo"; - version = "3.20.3"; + version = "3.20.4"; edition = "2021"; - sha256 = "0jc6va3nwcqikm7chnpdv1s87my3gs2j7g1sc7g3k91brg3arxbj"; + sha256 = "1nrq69ym33pmydashsxw1447a0r3v1q87drch9xzpchyzmkcz847"; license = "MIT OR Apache-2.0"; authors = [ "Nick Fitzgerald " @@ -1031,9 +1023,9 @@ rec { }; "bytes" = rec { crateName = "bytes"; - version = "1.12.1"; + version = "1.12.2"; edition = "2021"; - sha256 = "017z19dpg4f942h051m7bpnzcgng042hhcpd7bmg7bjjqd42lrgw"; + sha256 = "0z6xjs7a66qmhag9nlvv8m4wnk370c8cg8l4qqr8fbay06yzwvbp"; license = "MIT"; authors = [ "Carl Lerche " @@ -1048,9 +1040,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.11.0"; + version = "1.11.2"; edition = "2021"; - sha256 = "0ssz5frnjb3x873yhp2790xf9v8jz8hdbgj28i1gm861rvg8jn9p"; + sha256 = "0i8xv6psbly5zn2vpmv283cnl402jwlw3x6gjmx5r92lh4z9xvrk"; license = "MIT OR Apache-2.0"; dependencies = [ { @@ -1183,10 +1175,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.6.7"; + version = "4.6.8"; edition = "2024"; crateBin = []; - sha256 = "0il98y2rfw75984ck59znd4n592p07bxz8yy3a9blddb02rpd25a"; + sha256 = "0jnd0ilrh7yx5vlxjrnzanfm0lgad6p852wrsvc6j0kxb1w088xc"; license = "MIT OR Apache-2.0"; dependencies = [ { @@ -1266,9 +1258,9 @@ rec { }; "clap_derive" = rec { crateName = "clap_derive"; - version = "4.6.7"; + version = "4.6.8"; edition = "2024"; - sha256 = "1q36bgbkfb9rdz7jcvd0ffv0wwlyw0lg3z6iwdcybm0mjjvm3izr"; + sha256 = "0yknfahwc1bkz618rhzqpkgadaiawwa4lwpk5658ppf25a6rsp49"; procMacro = true; license = "MIT OR Apache-2.0"; dependencies = [ @@ -1286,7 +1278,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "full" ]; } ]; @@ -1299,9 +1291,9 @@ rec { }; "clap_lex" = rec { crateName = "clap_lex"; - version = "1.1.1"; + version = "1.1.2"; edition = "2024"; - sha256 = "11k4m4ngn39iz39lnq6nn6kyxzp62mnm1d86qfad3q0vlk33n4qw"; + sha256 = "0y9asv286gc92n02i11f5h15h91ljizjx4cz3j5dz8dkfqw1p4p0"; license = "MIT OR Apache-2.0"; }; @@ -1374,9 +1366,9 @@ rec { }; "const-oid" = rec { crateName = "const-oid"; - version = "0.10.2"; + version = "0.10.3"; edition = "2024"; - sha256 = "0p7m286mp8aai4sa72g7ji6qm0d4ns8wg4i4b2hj9p9615zm3vx6"; + sha256 = "16is3l224m0m7lc2zqhmnfhv8mljkv1274v8k6hj32imhjld95dy"; libName = "const_oid"; license = "Apache-2.0 OR MIT"; authors = [ @@ -1584,9 +1576,9 @@ rec { }; "crc32fast" = rec { crateName = "crc32fast"; - version = "1.5.4"; + version = "1.5.5"; edition = "2021"; - sha256 = "017g8wq37261s7i21v2p7wacl8pr3p4why0d9dfd2dskdm0x75sz"; + sha256 = "0r5dgm4p0hi5bhg90qbw1f09rmgfhlzhgldbmx01yxrml799v4ib"; license = "MIT OR Apache-2.0"; authors = [ "Sam Rijs " @@ -1638,9 +1630,9 @@ rec { }; "crypto-bigint" = rec { crateName = "crypto-bigint"; - version = "0.7.10"; + version = "0.7.11"; edition = "2024"; - sha256 = "04dyl8pzc00l7bmsa3br86dczs45hwhkw7wjzg9dl0xs9jv5q0ww"; + sha256 = "10ky8wx92qxj0hsd8n3njrjcdm8lb7wcn54dz3bmxg3z9mf1jk4i"; libName = "crypto_bigint"; license = "Apache-2.0 OR MIT"; authors = [ @@ -1873,7 +1865,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "full" "extra-traits" ]; } ]; @@ -1905,88 +1897,11 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; }; - "defmt" = rec { - crateName = "defmt"; - version = "1.2.0"; - edition = "2021"; - links = "defmt"; - sha256 = "1qg3w0w7f0fn74xp60aci37swsjz6hbrqp3pngbm9js0qxd7k874"; - license = "MIT OR Apache-2.0"; - authors = [ - "The Knurling-rs developers" - ]; - dependencies = [ - { - name = "bitflags"; - packageId = "bitflags 1.3.2"; - } - { - name = "defmt-macros"; - packageId = "defmt-macros"; - } - ]; - features = { - "unstable-test" = [ "defmt-macros/unstable-test" ]; - }; - resolvedDefaultFeatures = [ "alloc" ]; - }; - "defmt-macros" = rec { - crateName = "defmt-macros"; - version = "1.2.0"; - edition = "2021"; - sha256 = "0rl3dlnl8lc4dqzdnr6q0vrpjz3qmdca5641334rbi4rpmbb02w4"; - procMacro = true; - libName = "defmt_macros"; - license = "MIT OR Apache-2.0"; - authors = [ - "The Knurling-rs developers" - ]; - dependencies = [ - { - name = "defmt-parser"; - packageId = "defmt-parser"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 3.0.9"; - features = [ "full" "extra-traits" ]; - } - ]; - features = { - }; - }; - "defmt-parser" = rec { - crateName = "defmt-parser"; - version = "1.1.0"; - edition = "2021"; - sha256 = "1849z1vp502s77sip4wzj6jimjl6x5vr7b2ivp2mds1dzdlqz5bi"; - libName = "defmt_parser"; - license = "MIT OR Apache-2.0"; - authors = [ - "The Knurling-rs developers" - ]; - dependencies = [ - { - name = "thiserror"; - packageId = "thiserror 2.0.21"; - } - ]; - features = { - }; - }; "delegate" = rec { crateName = "delegate"; version = "0.13.5"; @@ -2248,6 +2163,12 @@ rec { packageId = "ctutils"; optional = true; } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "blobby" = [ "dep:blobby" ]; @@ -2260,7 +2181,7 @@ rec { "rand_core" = [ "common/rand_core" ]; "zeroize" = [ "dep:zeroize" "block-buffer?/zeroize" ]; }; - resolvedDefaultFeatures = [ "alloc" "block-api" "default" "mac" "oid" ]; + resolvedDefaultFeatures = [ "alloc" "block-api" "default" "mac" "oid" "zeroize" ]; }; "displaydoc" = rec { crateName = "displaydoc"; @@ -2283,7 +2204,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; features = { @@ -2431,9 +2352,9 @@ rec { }; "educe" = rec { crateName = "educe"; - version = "0.8.1"; + version = "0.8.2"; edition = "2024"; - sha256 = "0vg925v5bfkr2aynhfshlhk21j230fhpv6ingfz5p0fjz3wb3g8y"; + sha256 = "1il2yqh8884ql7d7f4ksmhaykj35vrmcxwbf75ia1pkyc39720x0"; procMacro = true; license = "MIT"; authors = [ @@ -2456,7 +2377,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "visit" "visit-mut" ]; } ]; @@ -2716,7 +2637,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; features = { @@ -2732,9 +2653,9 @@ rec { }; "errno" = rec { crateName = "errno"; - version = "0.3.14"; + version = "0.3.15"; edition = "2018"; - sha256 = "1szgccmh8vgryqyadg8xd58mnwwicf39zmin3bsn63df2wbbgjir"; + sha256 = "0xj7g0gdrgylxchyzxpfvvb5322j87zbhgal3pjr1fyfyiw6wpg3"; license = "MIT OR Apache-2.0"; authors = [ "Chris Wong " @@ -2911,9 +2832,9 @@ rec { }; "flate2" = rec { crateName = "flate2"; - version = "1.1.10"; + version = "1.1.11"; edition = "2018"; - sha256 = "1jvd2cl8j5hyf8imi62y1x7gwzz1hajirni0801yxhds1qp4wqvf"; + sha256 = "1i44w2j9sw5lqvkrw0d0d3dnn84zrz3nzlsyyrq6wr3b68l4jvb2"; license = "MIT OR Apache-2.0"; authors = [ "Alex Crichton " @@ -3193,7 +3114,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "full" ]; } ]; @@ -3550,7 +3471,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.2"; + packageId = "bitflags"; } { name = "libc"; @@ -3627,10 +3548,10 @@ rec { }; "granit-parser" = rec { crateName = "granit-parser"; - version = "2.1.0"; + version = "2.2.0"; edition = "2021"; crateBin = []; - sha256 = "1pq1ljzfc7n0sfx0f6la8f5p4lmj81n841rj293kvpylsj35bw05"; + sha256 = "1j1d897qqxarylh7rjrq37469aaax9x33ljqnhbb8ybhkh6lvl4v"; libName = "granit_parser"; license = "MIT OR Apache-2.0"; authors = [ @@ -3863,6 +3784,7 @@ rec { features = { "zeroize" = [ "digest/zeroize" ]; }; + resolvedDefaultFeatures = [ "zeroize" ]; }; "hostname" = rec { crateName = "hostname"; @@ -4966,29 +4888,7 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "itertools 0.14.0" = rec { - crateName = "itertools"; - version = "0.14.0"; - edition = "2018"; - sha256 = "118j6l1vs2mx65dqhwyssbrxpawa90886m3mzafdvyip41w2q69b"; - license = "MIT OR Apache-2.0"; - authors = [ - "bluss" - ]; - dependencies = [ - { - name = "either"; - packageId = "either"; - usesDefaultFeatures = false; - } - ]; - features = { - "default" = [ "use_std" ]; - "use_std" = [ "use_alloc" "either/use_std" ]; - }; - resolvedDefaultFeatures = [ "default" "use_alloc" "use_std" ]; - }; - "itertools 0.15.0" = rec { + "itertools" = rec { crateName = "itertools"; version = "0.15.0"; edition = "2018"; @@ -5012,9 +4912,9 @@ rec { }; "itoa" = rec { crateName = "itoa"; - version = "1.0.18"; + version = "1.0.19"; edition = "2021"; - sha256 = "10jnd1vpfkb8kj38rlkn2a6k02afvj3qmw054dfpzagrpl6achlg"; + sha256 = "1m44ca6ihrapjhlfy1vn4zj207qyhizbv0c1iwgnr6ksc575gl9v"; license = "MIT OR Apache-2.0"; authors = [ "David Tolnay " @@ -5049,35 +4949,107 @@ rec { ]; }; - "jiff" = rec { + "jiff 0.2.46" = rec { crateName = "jiff"; - version = "0.2.39"; + version = "0.2.46"; edition = "2021"; - sha256 = "0g7ghdl0minhwgkwf036ld5lmcdnwbf7kf5h75f9xj5si82h7irs"; + sha256 = "0zsak2pdr6j7krzv8jwn41nw23fmzqfbdy2vk2wrlfyv6iiwcp8p"; license = "Unlicense OR MIT"; authors = [ "Andrew Gallant " ]; dependencies = [ { - name = "defmt"; - packageId = "defmt"; + name = "jiff-core"; + packageId = "jiff-core 0.1.8"; + rename = "jcore"; + usesDefaultFeatures = false; + } + { + name = "jiff-static"; + packageId = "jiff-static 0.2.46"; + optional = true; + } + { + name = "jiff-static"; + packageId = "jiff-static 0.2.46"; + target = { target, features }: false; + } + { + name = "log"; + packageId = "log"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("ptr" == target."has_atomic" or null)); + } + { + name = "portable-atomic-util"; + packageId = "portable-atomic-util"; + usesDefaultFeatures = false; + target = { target, features }: (!("ptr" == target."has_atomic" or null)); + } + { + name = "serde_core"; + packageId = "serde_core"; optional = true; + usesDefaultFeatures = false; } + ]; + devDependencies = [ + { + name = "log"; + packageId = "log"; + } + ]; + features = { + "alloc" = [ "jcore/alloc" "serde_core?/alloc" "portable-atomic-util/alloc" ]; + "arbitrary" = [ "dep:arbitrary" "jcore/arbitrary" "std" ]; + "default" = [ "std" "tz-system" "tz-fat" "tzdb-bundle-platform" "tzdb-zoneinfo" "tzdb-concatenated" "perf-inline" ]; + "defmt" = [ "dep:defmt" "jcore/defmt" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "logging" = [ "dep:log" "jcore/logging" ]; + "serde" = [ "dep:serde_core" ]; + "static" = [ "static-tz" "jiff-static?/tzdb" ]; + "static-tz" = [ "dep:jiff-static" ]; + "std" = [ "alloc" "jcore/std" "log?/std" "serde_core?/std" ]; + "tz-fat" = [ "jcore/tz-fat" "jiff-static?/tz-fat" ]; + "tz-system" = [ "std" "dep:windows-link" ]; + "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; + "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; + "tzdb-concatenated" = [ "std" ]; + "tzdb-zoneinfo" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "serde" "std" ]; + }; + "jiff 1.0.1" = rec { + crateName = "jiff"; + version = "1.0.1"; + edition = "2024"; + sha256 = "0gnqhaqdcrkqkr6syjg4br0flss4rbg3prq2nc87y0gbf9ni4pgd"; + license = "Unlicense OR MIT"; + authors = [ + "Andrew Gallant " + ]; + dependencies = [ { name = "jiff-core"; - packageId = "jiff-core"; + packageId = "jiff-core 1.0.1"; rename = "jcore"; usesDefaultFeatures = false; } { name = "jiff-static"; - packageId = "jiff-static"; + packageId = "jiff-static 1.0.1"; optional = true; } { name = "jiff-static"; - packageId = "jiff-static"; + packageId = "jiff-static 1.0.1"; target = { target, features }: false; } { @@ -5124,7 +5096,7 @@ rec { } ]; features = { - "alloc" = [ "jcore/alloc" "serde_core?/alloc" "portable-atomic-util/alloc" "defmt?/alloc" ]; + "alloc" = [ "jcore/alloc" "serde_core?/alloc" "portable-atomic-util/alloc" ]; "arbitrary" = [ "dep:arbitrary" "jcore/arbitrary" "std" ]; "default" = [ "std" "tz-system" "tz-fat" "tzdb-bundle-platform" "tzdb-zoneinfo" "tzdb-concatenated" "perf-inline" ]; "defmt" = [ "dep:defmt" "jcore/defmt" ]; @@ -5141,13 +5113,13 @@ rec { "tzdb-concatenated" = [ "std" ]; "tzdb-zoneinfo" = [ "std" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "perf-inline" "serde" "std" "tz-fat" "tz-system" "tzdb-bundle-platform" "tzdb-concatenated" "tzdb-zoneinfo" ]; + resolvedDefaultFeatures = [ "alloc" "default" "perf-inline" "std" "tz-fat" "tz-system" "tzdb-bundle-platform" "tzdb-concatenated" "tzdb-zoneinfo" ]; }; - "jiff-core" = rec { + "jiff-core 0.1.8" = rec { crateName = "jiff-core"; - version = "0.1.2"; + version = "0.1.8"; edition = "2021"; - sha256 = "14qf99yrvnzm8rsvz7va43kgij398x26wpfk6s6gpfjkxc4k7r5f"; + sha256 = "06rc38qrwph31vvdzixqvyfrb8myykijsf9yifdh35z5j81bi6g5"; libName = "jiff_core"; license = "Unlicense OR MIT"; authors = [ @@ -5155,10 +5127,32 @@ rec { ]; dependencies = [ { - name = "defmt"; - packageId = "defmt"; + name = "log"; + packageId = "log"; optional = true; + usesDefaultFeatures = false; } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" "std" ]; + "default" = [ "std" "tz-fat" ]; + "defmt" = [ "dep:defmt" ]; + "logging" = [ "dep:log" ]; + "std" = [ "alloc" "log?/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" "tz-fat" ]; + }; + "jiff-core 1.0.1" = rec { + crateName = "jiff-core"; + version = "1.0.1"; + edition = "2024"; + sha256 = "1f5p6kk6cxlwkb2rhnasjl2cl95z4qah4cb2v56n6x4qb886md5n"; + libName = "jiff_core"; + license = "Unlicense OR MIT"; + authors = [ + "Andrew Gallant " + ]; + dependencies = [ { name = "log"; packageId = "log"; @@ -5167,7 +5161,6 @@ rec { } ]; features = { - "alloc" = [ "defmt?/alloc" ]; "arbitrary" = [ "dep:arbitrary" "std" ]; "default" = [ "std" "tz-fat" ]; "defmt" = [ "dep:defmt" ]; @@ -5176,11 +5169,11 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" "tz-fat" ]; }; - "jiff-static" = rec { + "jiff-static 0.2.46" = rec { crateName = "jiff-static"; - version = "0.2.39"; + version = "0.2.46"; edition = "2021"; - sha256 = "0gd96qg3jvxapr4a06nbrq2dq4skjbx06hxprklx1dvyqrq29hvd"; + sha256 = "1fpiakx7mcjapb2q0mqqin7xw6v0ahw7wycd8fy58wyl8xsszcf7"; procMacro = true; libName = "jiff_static"; license = "Unlicense OR MIT"; @@ -5190,7 +5183,7 @@ rec { dependencies = [ { name = "jiff-core"; - packageId = "jiff-core"; + packageId = "jiff-core 0.1.8"; rename = "jcore"; } { @@ -5209,13 +5202,48 @@ rec { features = { "tzdb" = [ "dep:jiff-tzdb" ]; }; + resolvedDefaultFeatures = [ "default" ]; + }; + "jiff-static 1.0.1" = rec { + crateName = "jiff-static"; + version = "1.0.1"; + edition = "2024"; + sha256 = "0a64i580m5kgicq3qmzp9fffcjvfcxl8szcy2vb30q9wvsdpw4n0"; + procMacro = true; + libName = "jiff_static"; + license = "Unlicense OR MIT"; + authors = [ + "Andrew Gallant " + ]; + dependencies = [ + { + name = "jiff-core"; + packageId = "jiff-core 1.0.1"; + rename = "jcore"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 3.0.10"; + } + ]; + features = { + "tzdb" = [ "dep:jiff-tzdb" ]; + }; resolvedDefaultFeatures = [ "default" "tz-fat" ]; }; "jiff-tzdb" = rec { crateName = "jiff-tzdb"; - version = "0.1.9"; - edition = "2021"; - sha256 = "19rd52w3il7603xln19s5yvz113dyrvmlpj4b63qdbkb1h3pg0zs"; + version = "1.0.0"; + edition = "2024"; + sha256 = "1z1srzss9rngld9aarrkb8a7gjwj8gc1symv6d5a8xd8d0dimrqj"; libName = "jiff_tzdb"; libPath = "lib.rs"; license = "Unlicense OR MIT"; @@ -5226,9 +5254,9 @@ rec { }; "jiff-tzdb-platform" = rec { crateName = "jiff-tzdb-platform"; - version = "0.1.3"; - edition = "2021"; - sha256 = "1s1ja692wyhbv7f60mc0x90h7kn1pv65xkqi2y4imarbmilmlnl7"; + version = "1.0.0"; + edition = "2024"; + sha256 = "03j24mjzbgs8yphn0p4pmfki8lsh0p478wlw337zda6gmy2b4rfw"; libName = "jiff_tzdb_platform"; libPath = "lib.rs"; license = "Unlicense OR MIT"; @@ -5487,9 +5515,9 @@ rec { }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; - version = "1.0.12"; + version = "1.0.13"; edition = "2021"; - sha256 = "1mzd3dg7qy1g6h8v81qnmkvp712nx2727qkgb1nyd44k9266di5i"; + sha256 = "1f270lrfjmkm42rf6dwnjmqk61xccc9jksl9vklksylfhdncjfy8"; libName = "jsonpath_rust"; license = "MIT"; authors = [ @@ -5576,7 +5604,7 @@ rec { } { name = "jiff"; - packageId = "jiff"; + packageId = "jiff 0.2.46"; usesDefaultFeatures = false; features = [ "alloc" "serde" ]; } @@ -5612,9 +5640,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "k8s_version"; license = "Apache-2.0"; @@ -5693,9 +5721,9 @@ rec { }; "kube" = rec { crateName = "kube"; - version = "5.0.0"; + version = "5.1.0"; edition = "2024"; - sha256 = "08fh07izsyfmfqf4d71p24awpnmfvhjbsjc99wx644clahphhfy7"; + sha256 = "0zsgw0yknqskcic4xlc7gr1nvppnlrdhp8i92ccnl1dzvvkslbgf"; license = "Apache-2.0"; authors = [ "clux " @@ -5770,9 +5798,9 @@ rec { }; "kube-client" = rec { crateName = "kube-client"; - version = "5.0.0"; + version = "5.1.0"; edition = "2024"; - sha256 = "02s36llknzls076dhj3p3m8pf4a1dmhgqnqi3cgxp454phiv4mmf"; + sha256 = "0ifq2a095y883kwp0zifriidrgka70ns4dvmk7hx2p6kcp2wph13"; libName = "kube_client"; license = "Apache-2.0"; authors = [ @@ -5843,7 +5871,7 @@ rec { } { name = "jiff"; - packageId = "jiff"; + packageId = "jiff 0.2.46"; optional = true; usesDefaultFeatures = false; features = [ "std" "serde" ]; @@ -6010,9 +6038,9 @@ rec { }; "kube-core" = rec { crateName = "kube-core"; - version = "5.0.0"; + version = "5.1.0"; edition = "2024"; - sha256 = "1f9zszr84ljjgr41bzcsjzq4g2vwniafynchxyh5p8qszi3lps3d"; + sha256 = "0kq1jgb2g3v93hwmiyckkd43x2mbbcfsy7s1ggn78pmzdxm4h9ya"; libName = "kube_core"; license = "Apache-2.0"; authors = [ @@ -6036,7 +6064,7 @@ rec { } { name = "jiff"; - packageId = "jiff"; + packageId = "jiff 0.2.46"; usesDefaultFeatures = false; features = [ "std" ]; } @@ -6094,9 +6122,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "5.0.0"; + version = "5.1.0"; edition = "2024"; - sha256 = "0kp63147ahqkv6inrj341q08i4mvw7h16z17q0d4jwa3qdmb87yy"; + sha256 = "17q1a5q060rhxmyyabmzxvz12kyai9w86kd4kp58awvvl81avx5r"; procMacro = true; libName = "kube_derive"; license = "Apache-2.0"; @@ -6129,7 +6157,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "extra-traits" ]; } ]; @@ -6144,9 +6172,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "5.0.0"; + version = "5.1.0"; edition = "2024"; - sha256 = "14j393x46py5ffmqhfbb9ykar7lvlad209n434cfrdi4a0061slx"; + sha256 = "0y040n91y4x3lw0wqmjgd3brd5qacj6k0mnnzibiy255qrz08yxa"; libName = "kube_runtime"; license = "Apache-2.0"; authors = [ @@ -6208,7 +6236,7 @@ rec { } { name = "parking_lot"; - packageId = "parking_lot"; + packageId = "parking_lot 0.13.0"; } { name = "pin-project"; @@ -6351,10 +6379,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.31"; + version = "1.1.33"; edition = "2018"; links = "z"; - sha256 = "1cjw8nisxy5iv8yms7dsxwhclb26djl38n6xq4nmlqxxra73087z"; + sha256 = "0ic0jc13vgz6fwcm12p7ss5bvl09mbkl52c2ip3v1ni18fgzbqzy"; libName = "libz_sys"; license = "MIT OR Apache-2.0"; authors = [ @@ -6409,7 +6437,7 @@ rec { "yoke" = [ "dep:yoke" ]; }; }; - "lock_api" = rec { + "lock_api 0.4.14" = rec { crateName = "lock_api"; version = "0.4.14"; edition = "2021"; @@ -6432,6 +6460,29 @@ rec { }; resolvedDefaultFeatures = [ "atomic_usize" "default" ]; }; + "lock_api 0.5.0" = rec { + crateName = "lock_api"; + version = "0.5.0"; + edition = "2024"; + sha256 = "1bsr9bn8b8fh1nq077l4yris6lizar4q7v6nykf175hpv8i1pnb5"; + license = "MIT OR Apache-2.0"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "scopeguard"; + packageId = "scopeguard"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "atomic_usize" ]; + "owning_ref" = [ "dep:owning_ref" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "atomic_usize" "default" ]; + }; "log" = rec { crateName = "log"; version = "0.4.35"; @@ -6514,9 +6565,9 @@ rec { }; "mime" = rec { crateName = "mime"; - version = "0.3.17"; + version = "0.3.18"; edition = "2015"; - sha256 = "16hkibgvb9klh0w0jk5crr5xv90l3wlf77ggymzjmvl1818vnxv8"; + sha256 = "18dzfqp5gaza7lg8qqpq1xh83zw79b1al3zryaacy5w1krd8z16m"; license = "MIT OR Apache-2.0"; authors = [ "Sean McArthur " @@ -6561,9 +6612,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.4"; + version = "1.2.5"; edition = "2021"; - sha256 = "1vl6px9zy0pwhlwnw3cknsqyzkg8bcpiwiq4cgicf2fwgywfv20p"; + sha256 = "0jlwrib8a3yp5h4j9s8z6qyh9k50lp8fjh38rmrsbc027y81lli0"; license = "MIT"; authors = [ "Carl Lerche " @@ -7356,7 +7407,7 @@ rec { "loom" = [ "dep:loom" ]; }; }; - "parking_lot" = rec { + "parking_lot 0.12.5" = rec { crateName = "parking_lot"; version = "0.12.5"; edition = "2021"; @@ -7368,11 +7419,11 @@ rec { dependencies = [ { name = "lock_api"; - packageId = "lock_api"; + packageId = "lock_api 0.4.14"; } { name = "parking_lot_core"; - packageId = "parking_lot_core"; + packageId = "parking_lot_core 0.9.12"; } ]; features = { @@ -7384,7 +7435,76 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "parking_lot_core" = rec { + "parking_lot 0.13.0" = rec { + crateName = "parking_lot"; + version = "0.13.0"; + edition = "2024"; + sha256 = "09njdx4da5163x3h43x0qpkrmhrg89ir0mc4hxbvgl71liy97kx9"; + license = "MIT OR Apache-2.0"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "lock_api"; + packageId = "lock_api 0.5.0"; + } + { + name = "parking_lot_core"; + packageId = "parking_lot_core 0.10.0"; + } + ]; + features = { + "arc_lock" = [ "lock_api/arc_lock" ]; + "deadlock_detection" = [ "parking_lot_core/deadlock_detection" ]; + "nightly" = [ "parking_lot_core/nightly" ]; + "owning_ref" = [ "lock_api/owning_ref" ]; + "serde" = [ "lock_api/serde" ]; + }; + resolvedDefaultFeatures = [ "default" ]; + }; + "parking_lot_core 0.10.0" = rec { + crateName = "parking_lot_core"; + version = "0.10.0"; + edition = "2024"; + sha256 = "09lqs4hjq55zz70gbiszx2wpz366h2pzq42hh6kqmhcc0rhg1zww"; + license = "MIT OR Apache-2.0"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "libc"; + packageId = "libc"; + target = { target, features }: (((target."unix" or false) && (!("aix" == target."os" or null))) || ("wasi" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + target = { target, features }: ("aix" == target."os" or null); + } + { + name = "redox_syscall"; + packageId = "redox_syscall 0.9.5"; + target = { target, features }: ("redox" == target."os" or null); + } + { + name = "smallvec"; + packageId = "smallvec"; + } + { + name = "windows-link"; + packageId = "windows-link"; + target = { target, features }: (target."windows" or false); + } + ]; + features = { + "backtrace" = [ "dep:backtrace" ]; + "deadlock_detection" = [ "petgraph" "backtrace" ]; + "petgraph" = [ "dep:petgraph" ]; + }; + }; + "parking_lot_core 0.9.12" = rec { crateName = "parking_lot_core"; version = "0.9.12"; edition = "2021"; @@ -7405,7 +7525,7 @@ rec { } { name = "redox_syscall"; - packageId = "redox_syscall"; + packageId = "redox_syscall 0.5.18"; target = { target, features }: ("redox" == target."os" or null); } { @@ -7494,9 +7614,9 @@ rec { }; "pest" = rec { crateName = "pest"; - version = "2.9.5"; - edition = "2021"; - sha256 = "0q3nkmv7947j750m63adihzynqn8mn6g096rdsriq0066lapbnx8"; + version = "2.9.6"; + edition = "2024"; + sha256 = "0xi54bky74fjzdjm1qmlj6ygixnf886p5whvfyd0m42nsb8kziay"; license = "MIT OR Apache-2.0"; authors = [ "DragoÈ™ Tiselice " @@ -7530,9 +7650,9 @@ rec { }; "pest_derive" = rec { crateName = "pest_derive"; - version = "2.9.5"; - edition = "2021"; - sha256 = "1d1kv4khnw50xgb57v5n679618spmjps052jgnv68mmk3zhigai0"; + version = "2.9.6"; + edition = "2024"; + sha256 = "1mrv5vhnjblq934wwi82612lnzshxn2xxd56304c5p81sd7pkkxd"; procMacro = true; license = "MIT OR Apache-2.0"; authors = [ @@ -7559,9 +7679,9 @@ rec { }; "pest_generator" = rec { crateName = "pest_generator"; - version = "2.9.5"; - edition = "2021"; - sha256 = "17gyswbjnzdh1z5sz4fq6jzv8mdv21jk2cmjnbzcnk7lrjpwavk2"; + version = "2.9.6"; + edition = "2024"; + sha256 = "0hdv3p7cjdqbkk5q1w0x1j91w6ag18ikxz59vqjlqcyvwpx3l1y3"; license = "MIT OR Apache-2.0"; authors = [ "DragoÈ™ Tiselice " @@ -7586,7 +7706,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; features = { @@ -7598,9 +7718,9 @@ rec { }; "pest_meta" = rec { crateName = "pest_meta"; - version = "2.9.5"; - edition = "2021"; - sha256 = "0p8gf9lm82b887s7n2il8r0rja95khala9w3hs0v1hb5m9n5px5k"; + version = "2.9.6"; + edition = "2024"; + sha256 = "09vklm3824fdrk12jdydwcrn7gqhm6m5ys7i59j429blncagdivv"; license = "MIT OR Apache-2.0"; authors = [ "DragoÈ™ Tiselice " @@ -8062,9 +8182,9 @@ rec { }; "prost" = rec { crateName = "prost"; - version = "0.14.4"; + version = "0.14.6"; edition = "2021"; - sha256 = "1qas5v5rap45f43v3ja0jngxrrafrkcwl0iw5a3ld1pz2rscd2jj"; + sha256 = "1d18a8nhwip7v3fbx9190p5xlq8zrxifwsjr50xan1yrs5lvbw1y"; license = "Apache-2.0"; authors = [ "Dan Burkert " @@ -8092,9 +8212,9 @@ rec { }; "prost-derive" = rec { crateName = "prost-derive"; - version = "0.14.4"; + version = "0.14.6"; edition = "2021"; - sha256 = "1pqa77d7da5pf6ba3kjj7510m5cynz6902ax01ckvr0pfrgv4w5m"; + sha256 = "03fp19322vf89xs6f50pidi2kahhp3x3b4ldkdr2zyjr6i1cbn8s"; procMacro = true; libName = "prost_derive"; license = "Apache-2.0"; @@ -8111,7 +8231,7 @@ rec { } { name = "itertools"; - packageId = "itertools 0.14.0"; + packageId = "itertools"; } { name = "proc-macro2"; @@ -8131,9 +8251,9 @@ rec { }; "prost-types" = rec { crateName = "prost-types"; - version = "0.14.4"; + version = "0.14.6"; edition = "2021"; - sha256 = "02ivjvc4cwl5bfgjs3l00hwlrk74z8zlg1xcgx60bww8fvf6fjgr"; + sha256 = "0ny7n70wkqifn9ny5pf3llz086sg1p8s4f6kacdg9kl4g967r6pj"; libName = "prost_types"; license = "Apache-2.0"; authors = [ @@ -8226,11 +8346,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.10.3" = rec { + "rand 0.10.4" = rec { crateName = "rand"; - version = "0.10.3"; + version = "0.10.4"; edition = "2024"; - sha256 = "1bxlhj4m9zrgfgk1yirf0nny86izrngscydfx9w387n9rfbgpjb5"; + sha256 = "04cv998yj3apwm1hcizych49yl6bhbzc613g8s8aavkyqrvs4vwg"; license = "MIT OR Apache-2.0"; authors = [ "The Rand Project Developers" @@ -8372,7 +8492,7 @@ rec { }; resolvedDefaultFeatures = [ "os_rng" "std" ]; }; - "redox_syscall" = rec { + "redox_syscall 0.5.18" = rec { crateName = "redox_syscall"; version = "0.5.18"; edition = "2021"; @@ -8385,7 +8505,30 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.2"; + packageId = "bitflags"; + } + ]; + features = { + "core" = [ "dep:core" ]; + "default" = [ "userspace" ]; + "rustc-dep-of-std" = [ "core" "bitflags/rustc-dep-of-std" ]; + }; + resolvedDefaultFeatures = [ "default" "userspace" ]; + }; + "redox_syscall 0.9.5" = rec { + crateName = "redox_syscall"; + version = "0.9.5"; + edition = "2021"; + sha256 = "1zb8g34sfgrbvja7lgay2isggkj461qf86mdchawn7ml8mc8f32q"; + libName = "syscall"; + license = "MIT"; + authors = [ + "Jeremy Soller " + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags"; } ]; features = { @@ -8435,16 +8578,16 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; }; "regex" = rec { crateName = "regex"; - version = "1.13.1"; + version = "1.13.2"; edition = "2021"; - sha256 = "1391a0a4100ik8cp7l577p3ip3haqq03rd9c5vdr7vcfdixj687h"; + sha256 = "0mq8jm4w2by1xsxwh8s04xqqyamzj4bn0khvm99yf4lccc7ay1wq"; license = "MIT OR Apache-2.0"; authors = [ "The Rust Project Developers" @@ -8501,9 +8644,9 @@ rec { }; "regex-automata" = rec { crateName = "regex-automata"; - version = "0.4.18"; + version = "0.4.19"; edition = "2021"; - sha256 = "1cml0rm0ssqfkibh9nh3gy4b6hbsbicj1rihpwf2a4v4nawm71dd"; + sha256 = "0n0i8kyv1lbpvb1i6a5x7mrybaml5q8p680s19m50mzlgpq0xmj1"; libName = "regex_automata"; license = "MIT OR Apache-2.0"; authors = [ @@ -8563,9 +8706,9 @@ rec { }; "regex-syntax" = rec { crateName = "regex-syntax"; - version = "0.8.11"; + version = "0.8.12"; edition = "2021"; - sha256 = "1m25h5q2wp976fb9gc3dsc9l99svcvd5cri8lncb51c46ydgzxnn"; + sha256 = "1xplhpgv7827jixdkhh17y899s8irsh5hwmrrb1y9898d9p080nr"; libName = "regex_syntax"; license = "MIT OR Apache-2.0"; authors = [ @@ -8898,9 +9041,9 @@ rec { }; "rsa" = rec { crateName = "rsa"; - version = "0.10.0-rc.19"; - edition = "2021"; - sha256 = "119bbjvkxiqipy4gn0ghgwygvan30illy9bflqf99vqn3qr5wh1k"; + version = "0.10.0"; + edition = "2024"; + sha256 = "16anfl7vw0qfjdnjgc1kjfayfzav8hajvmhz06rjishdgvbpzrr3"; license = "MIT OR Apache-2.0"; authors = [ "RustCrypto Developers" @@ -8918,6 +9061,12 @@ rec { usesDefaultFeatures = false; features = [ "zeroize" "alloc" ]; } + { + name = "crypto-common"; + packageId = "crypto-common"; + usesDefaultFeatures = false; + features = [ "rand_core" ]; + } { name = "crypto-primes"; packageId = "crypto-primes"; @@ -8929,6 +9078,12 @@ rec { usesDefaultFeatures = false; features = [ "alloc" "oid" ]; } + { + name = "hmac"; + packageId = "hmac"; + usesDefaultFeatures = false; + features = [ "zeroize" ]; + } { name = "pkcs1"; packageId = "pkcs1"; @@ -8951,9 +9106,8 @@ rec { { name = "sha2"; packageId = "sha2"; - optional = true; usesDefaultFeatures = false; - features = [ "oid" ]; + features = [ "oid" "zeroize" ]; } { name = "signature"; @@ -8980,25 +9134,17 @@ rec { packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } - { - name = "sha2"; - packageId = "sha2"; - usesDefaultFeatures = false; - features = [ "oid" ]; - } ]; features = { - "crypto-common" = [ "dep:crypto-common" ]; - "default" = [ "std" "encoding" ]; + "default" = [ "std" "encoding" "getrandom" ]; "encoding" = [ "dep:pkcs1" "dep:pkcs8" "dep:spki" ]; - "getrandom" = [ "crypto-bigint/getrandom" "crypto-common" ]; + "getrandom" = [ "crypto-bigint/getrandom" "crypto-common/getrandom" ]; "pkcs5" = [ "pkcs8/encryption" ]; "serde" = [ "encoding" "dep:serde" "dep:serdect" "crypto-bigint/serde" ]; "sha1" = [ "dep:sha1" ]; - "sha2" = [ "dep:sha2" ]; "std" = [ "pkcs1?/std" "pkcs8?/std" ]; }; - resolvedDefaultFeatures = [ "default" "encoding" "sha2" "std" ]; + resolvedDefaultFeatures = [ "default" "encoding" "getrandom" "pkcs1v15" "std" ]; }; "rstest" = rec { crateName = "rstest"; @@ -9214,9 +9360,9 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.15.1"; + version = "1.16.0"; edition = "2021"; - sha256 = "15hakk4pcvr5278cazgw9qf2r7gdg09rg5pivbyd3dbyih12aj9g"; + sha256 = "1yv38vs71zaxmmx8cx0w8gm0sgi5yn96bzqw3z3dh1kvvvvkabqh"; libName = "rustls_pki_types"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -9372,9 +9518,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.103.16"; + version = "0.103.17"; edition = "2021"; - sha256 = "10djm2f00xzi5qlbcfqjhhn2smk2h9aw8b4l74qncz4xsfvb2hs4"; + sha256 = "1m9zhn7hxx8xkn16833cd6634z6l045rf72sr8zq453wi057d873"; libName = "webpki"; license = "ISC"; dependencies = [ @@ -9582,13 +9728,13 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "extra-traits" ]; } ]; @@ -9702,7 +9848,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.2"; + packageId = "bitflags"; } { name = "core-foundation"; @@ -9824,10 +9970,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.2.0"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1n87f1lfymy00y0fnr4kjq9xn1pqp2zsv6szigsvnk9l941xx74n"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -9973,7 +10119,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -10006,7 +10152,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -10237,7 +10383,7 @@ rec { "oid" = [ "digest/oid" ]; "zeroize" = [ "digest/zeroize" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "oid" ]; + resolvedDefaultFeatures = [ "alloc" "default" "oid" "zeroize" ]; }; "sharded-slab" = rec { crateName = "sharded-slab"; @@ -10403,9 +10549,9 @@ rec { }; "smallvec" = rec { crateName = "smallvec"; - version = "1.16.4"; + version = "1.16.6"; edition = "2018"; - sha256 = "1j772prjisi2872lxry7d9fkss3dfbjwl16jmsvvp2qvk2qx9mfh"; + sha256 = "0nbvgy2346gisl3kj8hh2dxvplhgzysx2mgcj2frbpnm6ljm8i1j"; license = "MIT OR Apache-2.0"; authors = [ "The Servo Project Developers" @@ -10630,9 +10776,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_certs"; license = "Apache-2.0"; @@ -10669,7 +10815,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.3"; + packageId = "rand 0.10.4"; } { name = "rand_core"; @@ -10678,7 +10824,7 @@ rec { { name = "rsa"; packageId = "rsa"; - features = [ "sha2" ]; + features = [ "pkcs1v15" ]; } { name = "sha2"; @@ -10810,6 +10956,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "url"; + packageId = "url"; + } ]; buildDependencies = [ { @@ -10836,9 +10986,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_operator"; license = "Apache-2.0"; @@ -10899,7 +11049,7 @@ rec { } { name = "jiff"; - packageId = "jiff"; + packageId = "jiff 1.0.1"; } { name = "json-patch"; @@ -10920,7 +11070,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.3"; + packageId = "rand 0.10.4"; } { name = "regex"; @@ -11041,9 +11191,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11066,7 +11216,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; @@ -11077,9 +11227,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_shared"; license = "Apache-2.0"; @@ -11089,7 +11239,7 @@ rec { dependencies = [ { name = "jiff"; - packageId = "jiff"; + packageId = "jiff 1.0.1"; optional = true; } { @@ -11159,9 +11309,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_telemetry"; license = "Apache-2.0"; @@ -11270,9 +11420,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_versioned"; license = "Apache-2.0"; @@ -11317,9 +11467,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11342,7 +11492,7 @@ rec { } { name = "itertools"; - packageId = "itertools 0.15.0"; + packageId = "itertools"; } { name = "k8s-openapi"; @@ -11371,7 +11521,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; @@ -11382,9 +11532,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "837182338040c290701232ea3bcb4fb1d71f635a"; + sha256 = "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf"; }; libName = "stackable_webhook"; license = "Apache-2.0"; @@ -11444,7 +11594,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.3"; + packageId = "rand 0.10.4"; } { name = "serde"; @@ -11715,11 +11865,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; - "syn 3.0.9" = rec { + "syn 3.0.10" = rec { crateName = "syn"; - version = "3.0.9"; + version = "3.0.10"; edition = "2021"; - sha256 = "0fw28lhl90kls24q2h2sp39yjb0lsvz5cwh9fd3f3w3v9kp8v36p"; + sha256 = "0ygaz8ccgap0vb0faifk1xv1p2sayg490r038fab1c57lfzcrsaa"; license = "MIT OR Apache-2.0"; authors = [ "David Tolnay " @@ -11794,7 +11944,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -11891,7 +12041,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; } ]; @@ -12169,7 +12319,7 @@ rec { } { name = "parking_lot"; - packageId = "parking_lot"; + packageId = "parking_lot 0.12.5"; optional = true; } { @@ -12264,7 +12414,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "full" ]; } ]; @@ -12427,9 +12577,9 @@ rec { }; "toml_datetime" = rec { crateName = "toml_datetime"; - version = "1.1.2+spec-1.1.0"; + version = "1.1.3+spec-1.1.0"; edition = "2024"; - sha256 = "0lrhcmqvhjr259w4f2kijya7fgi0kpmhg9fb3m144v3cj1kxg1ib"; + sha256 = "0jbfdgsl0vw6f394cifqm62d8q0aiiima891v2i7c8fdhg42agqp"; license = "MIT OR Apache-2.0"; dependencies = [ { @@ -12449,9 +12599,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.18+spec-1.1.0"; + version = "0.25.19+spec-1.1.0"; edition = "2024"; - sha256 = "15hkw83x94drmc7d1xgg3j5485pijzfajbr1il6l5f8dsy5ygihy"; + sha256 = "063j62l4wsmxpvbiqfw89avzxhv79a3rz6ngw552dmsq9266hhyx"; license = "MIT OR Apache-2.0"; dependencies = [ { @@ -12831,7 +12981,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.2"; + packageId = "bitflags"; } { name = "bytes"; @@ -12950,7 +13100,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags 2.13.2"; + packageId = "bitflags"; } { name = "bytes"; @@ -13516,9 +13666,9 @@ rec { }; "ucd-trie" = rec { crateName = "ucd-trie"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "0wc9p07sqwz320848i52nvyjvpsxkx3kv5bfbmm6s35809fdk5i8"; + sha256 = "19lj832gpvlv5099zgzgbm87h9smnklbpdcgl7x8cmlir6zvs84g"; libName = "ucd_trie"; license = "MIT OR Apache-2.0"; authors = [ @@ -13703,41 +13853,35 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.29.0"; + version = "1.29.3"; edition = "2021"; - sha256 = "0525458ww4jwa0nw513jdrhgwxrldglzvfrbw5gnjl0cc88js3ja"; + sha256 = "0r1v5bh3xrdqgrywfmg9yqplyjm018a5gs0345qvvx52zxfv251w"; license = "Apache-2.0 OR MIT"; - authors = [ - "Ashley Mannix" - "Dylan DPC" - "Hunar Roop Kahlon" - ]; dependencies = [ { name = "js-sys"; packageId = "js-sys"; optional = true; usesDefaultFeatures = false; - target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + target = { target, features }: ((builtins.elem "wasm" target."family") && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; optional = true; usesDefaultFeatures = false; - target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + target = { target, features }: ((builtins.elem "wasm" target."family") && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } ]; devDependencies = [ { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + target = { target, features }: ((builtins.elem "wasm" target."family") && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } ]; features = { "arbitrary" = [ "dep:arbitrary" ]; - "atomic" = [ "dep:atomic" ]; "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; "bytemuck" = [ "dep:bytemuck" ]; "default" = [ "std" ]; @@ -13752,11 +13896,9 @@ rec { "slog" = [ "dep:slog" ]; "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; - "v1" = [ "atomic" ]; "v3" = [ "md5" ]; "v4" = [ "rng" ]; "v5" = [ "sha1" ]; - "v6" = [ "atomic" ]; "v7" = [ "rng" ]; "zerocopy" = [ "dep:zerocopy" ]; }; @@ -14002,7 +14144,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "visit" "visit-mut" "full" "extra-traits" ]; } { @@ -14561,9 +14703,9 @@ rec { }; "webpki-root-certs" = rec { crateName = "webpki-root-certs"; - version = "1.0.11"; + version = "1.0.12"; edition = "2021"; - sha256 = "0apn96djpc8yj1cp9scvqd8a5wd93ncxla4liwp18360mdjhx7g4"; + sha256 = "11d5s7zs8rkfppris96pvl71izw2b95vjah9nmai30pvxgf9hc99"; libName = "webpki_root_certs"; license = "CDLA-Permissive-2.0"; dependencies = [ @@ -15395,9 +15537,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.4"; + version = "1.0.5"; edition = "2021"; - sha256 = "10fzxipa7lx16172p3aca9j60hzbqgjki2f95kqksd5qywcp7f93"; + sha256 = "1vvs4zmisyxxp1igpjpi87z26d6wl3q98shcvx68gazhklcr6fwc"; license = "MIT"; dependencies = [ { @@ -15660,7 +15802,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "fold" "visit" ]; } { @@ -15780,7 +15922,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "fold" "visit" ]; } { @@ -15792,9 +15934,9 @@ rec { }; "zeroize" = rec { crateName = "zeroize"; - version = "1.9.1"; + version = "1.9.2"; edition = "2024"; - sha256 = "0yb8iykihpl3hfw5c4silw2lklpfxajkaa9yj1qw6jsy5hwq8c71"; + sha256 = "1an5zxp4cnc9gz1zyygxylndsmvmkzagqjndlkma3aq5925gzk45"; license = "Apache-2.0 OR MIT"; authors = [ "The RustCrypto Project Developers" @@ -15952,7 +16094,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.9"; + packageId = "syn 3.0.10"; features = [ "extra-traits" ]; } ]; diff --git a/Cargo.toml b/Cargo.toml index 4f2d809f..d5ed0893 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,10 +28,11 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.54", features = ["full"] } tracing = "0.1" +url = "2.5" yaml_serde = "0.10" [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "fix/versioned-any-changed-value-type" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } [workspace.lints.clippy] diff --git a/crate-hashes.json b/crate-hashes.json index e59dc466..d2846765 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#k8s-version@0.1.5": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-certs@0.5.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator-derive@0.3.2": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator@0.122.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-shared@0.2.1": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-telemetry@0.6.6": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned-macros@0.14.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned@0.14.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-webhook@0.11.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj" + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#k8s-version@0.1.5": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-certs@0.5.0": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-operator-derive@0.3.2": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-operator@0.122.0": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-shared@0.2.1": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-telemetry@0.6.6": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-versioned-macros@0.14.0": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-versioned@0.14.0": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf", + "git+https://github.com/stackabletech//operator-rs.git?branch=fix%2Fversioned-any-changed-value-type#stackable-webhook@0.11.0": "1jbv3604gc6azyz96q4lbz4nraq5hmj030gbr9m1vm4zr31gw6yf" } \ No newline at end of file diff --git a/deploy/helm/hive-operator/Chart.yaml b/deploy/helm/hive-operator/Chart.yaml index d63e1ed4..ccb223c7 100644 --- a/deploy/helm/hive-operator/Chart.yaml +++ b/deploy/helm/hive-operator/Chart.yaml @@ -38,7 +38,7 @@ annotations: url: https://quay.io/repository/stackable/sdp/hive-operator artifacthub.io/crds: |- - kind: HiveCluster - version: v1alpha1 + version: v1alpha2 name: hiveclusters.hive.stackable.tech displayName: HiveCluster description: A Hive cluster stacklet. This resource is managed by the Stackable operator for Apache Hive. diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml index 34aff112..c9355530 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive-postgres-s3 @@ -14,6 +14,7 @@ spec: credentialsSecretName: hive-credentials s3: reference: minio + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 index 34aff112..c9355530 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive-postgres-s3 @@ -14,6 +14,7 @@ spec: credentialsSecretName: hive-credentials s3: reference: minio + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: diff --git a/docs/modules/hive/pages/getting_started/first_steps.adoc b/docs/modules/hive/pages/getting_started/first_steps.adoc index 5a18f979..aad330b1 100644 --- a/docs/modules/hive/pages/getting_started/first_steps.adoc +++ b/docs/modules/hive/pages/getting_started/first_steps.adoc @@ -40,7 +40,8 @@ A SecretClass for the credentials to the Minio. The credentials were defined in include::example$getting_started/hive-minio-credentials-secret-class.yaml[] ---- -And lastly the actual Apache Hive cluster definition (`hive-postgres-s3.yaml`). Note how it references the `minio` S3 object +And lastly the actual Apache Hive cluster definition (`hive-postgres-s3.yaml`). +Note how it references the `minio` S3Connection and stores the xref:usage-guide/data-storage.adoc#warehouse-directory[warehouse directory] in the `my-bucket` bucket via `warehouseDir`: [source,yaml] ---- diff --git a/docs/modules/hive/pages/index.adoc b/docs/modules/hive/pages/index.adoc index c6f23618..05a0aa46 100644 --- a/docs/modules/hive/pages/index.adoc +++ b/docs/modules/hive/pages/index.adoc @@ -4,7 +4,7 @@ :hive: https://hive.apache.org :github: https://github.com/stackabletech/hive-operator/ :crd: {crd-ref}{crd-ref-release}&operator=hive-operator -:crd-hivecluster: {crd-ref}/hive.stackable.tech/v1alpha1/hivecluster{crd-ref-release} +:crd-hivecluster: {crd-ref}/hive.stackable.tech/v1alpha2/hivecluster{crd-ref-release} :feature-tracker: https://features.stackable.tech/unified [.link-bar] @@ -37,6 +37,8 @@ Every role group is accessible through its own Service, and there is a Service f The operator creates a xref:concepts:service_discovery.adoc[service discovery ConfigMap] for the Hive metastore instance. The discovery ConfigMap contains information on how to connect to the HMS. +Besides its Thrift API, the HMS can also serve the xref:usage-guide/iceberg-rest-catalog.adoc[Iceberg REST catalog API] (experimental). + == Dependencies The Stackable operator for Apache Hive depends on the Stackable xref:commons-operator:index.adoc[commons], xref:secret-operator:index.adoc[secret] and xref:listener-operator:index.adoc[listener] operators. diff --git a/docs/modules/hive/pages/reference/discovery.adoc b/docs/modules/hive/pages/reference/discovery.adoc index bb91d356..8392d5b5 100644 --- a/docs/modules/hive/pages/reference/discovery.adoc +++ b/docs/modules/hive/pages/reference/discovery.adoc @@ -9,7 +9,7 @@ The Stackable Operator for Apache Hive publishes a discovery https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#configmap-v1-core[ConfigMap], which exposes a client configuration bundle that allows access to the Apache Hive cluster. -The bundle includes an Apache Thrift connection string to access the Hive Metastore service. This string may be used by other operators or tools to configure their products with access to Hive. Access is limited to services within the same Kubernetes cluster. +The bundle includes an Apache Thrift connection string to access the Hive Metastore service. This string may be used by other operators or tools to configure their products with access to Hive. == Example @@ -17,7 +17,7 @@ Given the following Hive cluster: [source,yaml,subs="normal,callouts"] ---- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: {clusterName} # <1> @@ -29,6 +29,7 @@ spec: host: postgresql database: hive credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: # <3> @@ -51,27 +52,20 @@ The resulting discovery ConfigMap is `{namespace}/{clusterName}`. == Contents -=== Internal access - -The `{namespace}/{clusterName}` discovery ConfigMap contains the following fields where `{clusterName}` represents the name, `{namespace}` the namespace of the cluster and `{roleGroup}` a role group of the `metastore` role: +The `{namespace}/{clusterName}` discovery ConfigMap contains the following fields where `{clusterName}` represents the name and `{namespace}` the namespace of the cluster: `HIVE`:: ==== -Contains the thrift protocol connection string for the Hive metastore service: +Contains the thrift protocol connection string for the Hive metastore service. +The address and port are the ones published by the `{clusterName}-metastore` xref:listener-operator:listener.adoc[Listener], so they depend on the xref:usage-guide/listenerclass.adoc[ListenerClass] in use. +For the default `cluster-internal` ListenerClass this is: [subs="attributes"] - thrift://{clusterName}.{namespace}.svc.cluster.local:{metastorePort} -==== - -WARNING: Using the Hive metastore in high availability mode (replicas > 1) does not work with Derby but instead requires a properly configured database like PostgreSQL or MySQL. - -=== External access - -If `spec.clusterConfig.listenerClass` is set to `external-unstable` an additional ConfigMap is generated to expose external access to the cluster. This discovery ConfigMap is reachable via `{namespace}/{clusterName}-nodeport`. + thrift://{clusterName}-metastore.{namespace}.svc.cluster.local:{metastorePort} +This entry is only present if the Thrift API is enabled (`spec.clusterConfig.thriftApi.enabled`, which is the default). ==== -Contains the thrift protocol connection string for the Hive metastore NodePort service: - thrift://: - thrift://: -==== +The discovery ConfigMap does not contain an entry for the xref:usage-guide/iceberg-rest-catalog.adoc[Iceberg REST catalog] yet. + +WARNING: Using the Hive metastore in high availability mode (replicas > 1) does not work with Derby but instead requires a properly configured database like PostgreSQL or MySQL. diff --git a/docs/modules/hive/pages/usage-guide/data-storage.adoc b/docs/modules/hive/pages/usage-guide/data-storage.adoc index 037b29cd..a987407a 100644 --- a/docs/modules/hive/pages/usage-guide/data-storage.adoc +++ b/docs/modules/hive/pages/usage-guide/data-storage.adoc @@ -9,6 +9,32 @@ However, as soon as you start storing metadata in the HMS that refers to `s3://` So if you are storing tables in S3 (or HDFS for that matter), you need to give the HMS access to that filesystem as well. The Stackable Operator currently supports S3 and HDFS. +[#warehouse-directory] +== Warehouse directory + +The warehouse directory is the default location for databases and managed tables that are created without an explicit location. +It is required and configured in the top level `clusterConfig`: + +[source,yaml] +---- +clusterConfig: + warehouseDir: s3a://my-bucket/warehouse/ # <1> + warehouseExternalDir: s3a://my-bucket/warehouse/external/ # <2> +---- +<1> Maps to `hive.metastore.warehouse.dir`. +<2> Optional, maps to `hive.metastore.warehouse.external.dir`. + If it is not set, the operator does not configure it and the Hive metastore uses its internal default. + It is required when using the xref:usage-guide/iceberg-rest-catalog.adoc[Iceberg REST catalog], see that docs for details. + +When using HDFS, use an HDFS location instead, e.g. `hdfs://my-hdfs-cluster/warehouse/`. + +The location must be on a filesystem the HMS has access to (see <<_s3_support,S3 support>> and <<_apache_hdfs_support,Apache HDFS support>> below). +The HMS only accesses it once databases or tables are created there. + +WARNING: Always specify the full URI including the scheme (`s3a://`, `s3://` or `hdfs://`). +A path without a scheme, such as `/warehouse`, is resolved against the default filesystem. +Without an HDFS connection this is the local filesystem of the HMS Pod, which is ephemeral: all data written there is lost when the Pod is restarted. + [s3] == S3 support diff --git a/docs/modules/hive/pages/usage-guide/database-driver.adoc b/docs/modules/hive/pages/usage-guide/database-driver.adoc index 3b6fa16b..b593bb3d 100644 --- a/docs/modules/hive/pages/usage-guide/database-driver.adoc +++ b/docs/modules/hive/pages/usage-guide/database-driver.adoc @@ -141,7 +141,7 @@ This is done by setting the value of the environment variable `METASTORE_AUX_JAR [source,yaml] ---- --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive-with-drivers @@ -156,6 +156,7 @@ spec: credentialsSecretName: hive-credentials # <2> s3: reference: minio # <3> + warehouseDir: s3a://my-bucket/warehouse/ # <3> metastore: roleGroups: default: @@ -186,7 +187,7 @@ stringData: <1> The database connection details matching those given when deploying the MySQL Helm chart <2> Hive credentials are retrieved from a Secret -<3> A reference to the file store using S3 (this has been omitted from this article for the sake of brevity, but is described in e.g. the xref:getting_started/first_steps.adoc[] guide) +<3> A reference to the file store using S3 and the xref:usage-guide/data-storage.adoc#warehouse-directory[warehouse directory] in it (this has been omitted from this article for the sake of brevity, but is described in e.g. the xref:getting_started/first_steps.adoc[] guide) <4> Use `envOverrides` to set the driver path <5> Use `podOverrides` to mount the driver @@ -216,7 +217,7 @@ docker build -f ./Dockerfile -t oci.stackable.tech/sdp/hive:4.2.1-stackable0.0.0 [source, yaml] ---- --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -229,6 +230,7 @@ spec: ... s3: ... + warehouseDir: ... metastore: config: logging: diff --git a/docs/modules/hive/pages/usage-guide/derby-example.adoc b/docs/modules/hive/pages/usage-guide/derby-example.adoc index 640e659e..63904cc1 100644 --- a/docs/modules/hive/pages/usage-guide/derby-example.adoc +++ b/docs/modules/hive/pages/usage-guide/derby-example.adoc @@ -9,7 +9,7 @@ It should generally be safe to simply use the latest image version that is avail .Create a single node Apache Hive Metastore cluster using Derby: [source,yaml] ---- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive-derby @@ -19,11 +19,15 @@ spec: clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ # <1> metastore: roleGroups: default: replicas: 1 ---- +<1> The default location for databases and tables. +The Hive metastore only accesses it once databases or tables are created there, which requires access to the bucket as shown below. +See xref:usage-guide/data-storage.adoc#warehouse-directory[Warehouse directory] for details. WARNING: You should not use the `Derby` database in production. Derby stores data locally which does not work in high availability setups (multiple replicas) and all data is lost after Pod restarts. @@ -46,12 +50,12 @@ In order to upload data to minio we need a port-forward to access the web ui. ---- kubectl port-forward service/minio 9001 ---- -Then, connect to localhost:9001 and login with the user `minio-access-key` and password `minio-secret-key`. Create a bucket and upload data. +Then, connect to localhost:9001 and login with the user `minio-access-key` and password `minio-secret-key`. Create a bucket named `hive` and upload data. Deploy the hive cluster: [source,yaml] ---- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive-derby @@ -68,6 +72,7 @@ spec: accessStyle: Path credentials: secretClass: simple-hive-s3-secret-class + warehouseDir: s3a://hive/warehouse/ metastore: roleGroups: default: @@ -121,7 +126,7 @@ helm install hive bitnami/postgresql --version=12.1.5 \ .Create Hive Metastore using a PostgreSQL database [source,yaml] ---- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive-postgres @@ -134,6 +139,7 @@ spec: host: hive-postgresql.default.svc.cluster.local database: hive credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/warehouse/ # <1> metastore: roleGroups: default: @@ -148,3 +154,4 @@ stringData: username: hive password: hive ---- +<1> As in the first example, the Hive metastore needs access to this bucket once databases or tables are created there. diff --git a/docs/modules/hive/pages/usage-guide/iceberg-rest-catalog.adoc b/docs/modules/hive/pages/usage-guide/iceberg-rest-catalog.adoc new file mode 100644 index 00000000..60da5fed --- /dev/null +++ b/docs/modules/hive/pages/usage-guide/iceberg-rest-catalog.adoc @@ -0,0 +1,165 @@ += Iceberg REST catalog +:description: Serve the Apache Iceberg REST catalog API from the Hive metastore, secure it with OAuth 2 and optionally disable the Thrift API. +:hive-iceberg-rest-docs: https://hive.apache.org/docs/latest/admin/iceberg-rest-catalog/ +:hive-keycloak-docs: https://hive.apache.org/docs/latest/admin/oauth2/keycloak/ +:rfc-9068: https://www.rfc-editor.org/rfc/rfc9068 + +WARNING: Support for the Iceberg REST catalog is experimental and may change in future releases. + +The Hive metastore (HMS) can serve the {hive-iceberg-rest-docs}[Apache Iceberg REST catalog API] in addition to its Thrift API. +The catalog runs inside the HMS process and uses the same metadata database and xref:usage-guide/data-storage.adoc#warehouse-directory[warehouse directory]. +Iceberg clients that support the REST catalog can therefore access the Iceberg tables in the HMS without a Hive or Thrift client. + +The Iceberg REST catalog requires Hive 4.2.0 or newer. + +== Enabling the REST catalog + +[source,yaml] +---- +spec: + clusterConfig: + warehouseDir: s3a://my-bucket/warehouse/ + warehouseExternalDir: s3a://my-bucket/warehouse/external/ # <1> + icebergRestCatalog: + enabled: true +---- +<1> Namespaces created via the REST catalog without an explicit location are placed in this directory. + +IMPORTANT: Set `warehouseExternalDir` when using the REST catalog. +Without it, the REST catalog can not create any namespace, not even one with an explicit location, and fails with `java.lang.NullPointerException: Warehouse location is not set: hive.metastore.warehouse.external.dir=null`. +We (or Hive upstream) might improve the behavior in the https://github.com/stackabletech/docker-images/pull/1722[future], so this might become irrelevant. + +The catalog is served over plain HTTP on port `9001` (named `iceberg-rest`). +The port is exposed by the role xref:usage-guide/listenerclass.adoc[Listener] and the headless role group Services, next to the Thrift port. +Clients use the URI `http://
:9001/iceberg`, where `
` is the address of the Listener, e.g. `-metastore..svc.cluster.local` for the default `cluster-internal` ListenerClass. + +The xref:reference/discovery.adoc[discovery ConfigMap] does not contain an entry for the REST catalog yet. + +[#migration] +== Migrating existing Iceberg tables + +The REST catalog serves the same metadata as the Thrift API. +Iceberg tables that were created via the Thrift API, e.g. by Trino using the Hive metastore, therefore show up in the REST catalog without any migration and keep their locations. +Their schema, partitioning and history (e.g. for time travel) carry over as well. +Hive databases show up as namespaces. + +Plain (non-Iceberg) Hive tables are not served by the REST catalog: they are neither listed nor can they be loaded. +Keep the Thrift API enabled as long as clients need to access them. + +Views are not shared between the two APIs: + +* Trino views created via the Thrift API are stored as Hive views, which the REST catalog does not serve. + Recreate them via the REST catalog. +* Iceberg views created via the REST catalog are listed by Trino via the Thrift API, but can not be read. + +You can move clients from the Thrift API to the REST catalog step by step: + +. Enable the REST catalog next to the Thrift API and set `warehouseExternalDir`, as described in <<_enabling_the_rest_catalog>>. + This restarts the HMS, existing tables are not modified. +. Switch the clients to the REST catalog one by one. + Clients can use both APIs at the same time. + Changes made via one API are visible via the other one, including schema changes, row-level deletes and updates as well as table maintenance such as compaction. +. Once no client uses the Thrift API anymore, disable it as described in <<_disabling_the_thrift_api>>. + +The https://github.com/stackabletech/hive-operator/blob/main/tests/templates/kuttl/iceberg-rest-catalog/[iceberg-rest-catalog integration test] runs through these steps with Trino. + +[#authentication] +== Authentication + +WARNING: If no `authentication` is configured, the REST catalog does not authenticate requests at all and accepts every client that can reach the port. + +The REST catalog supports the authentication methods `oAuth2` (recommended), `jwt` and `simple`. +Exactly one of them can be configured. + +=== OAuth 2 + +The HMS validates the JWT access tokens that clients send as bearer tokens: + +[source,yaml] +---- +spec: + clusterConfig: + icebergRestCatalog: + enabled: true + authentication: + oAuth2: + issuerUrl: https://keycloak.example.com/realms/my-realm # <1> + audience: hive-metastore # <2> +---- +<1> The issuer of the access tokens. + On startup, the HMS fetches the server metadata from `/.well-known/oauth-authorization-server`, which must contain the `jwks_uri` used to validate the tokens. +<2> The audience (`aud` claim) the access tokens must be issued for. + +The access tokens must also + +* have the JOSE header type `at+jwt` as defined in {rfc-9068}[RFC 9068], which some authorization servers do not use by default, +* contain the scope `catalog`, which Iceberg clients request by default. + +The HMS uses the `sub` claim as the user name. +The Hive documentation describes how to {hive-keycloak-docs}[configure Keycloak] accordingly. + +NOTE: The HMS fetches the server metadata only on startup. +If that fails, e.g. because the issuer is unreachable, the HMS logs an error and starts *without* the REST catalog. +The Pod then does not become ready, as the readiness probe checks the REST catalog port. + +=== JWT + +The HMS accepts JSON Web Tokens (JWT) that clients send as bearer tokens and validates their signature using a JSON Web Key Set (JWKS): + +[source,yaml] +---- +spec: + clusterConfig: + icebergRestCatalog: + enabled: true + authentication: + jwt: + jwksUrls: # <1> + - https://keycloak.example.com/realms/my-realm/protocol/openid-connect/certs +---- +<1> The URLs of the JWKS used to validate the signature of the tokens. + The HMS only falls back to another URL if fetching the JWKS fails, so all URLs need to serve the same keys, e.g. replicas of the same authorization server. + They can not be used to accept tokens of different issuers. + The HMS does not preserve the order of the URLs. + +The HMS uses the `sub` claim as the user name. + +WARNING: In this mode the HMS only checks the signature, the expiry and the presence of the `sub` claim. +It does not check the issuer or the audience of a token. +Every token signed by a key from the JWKS is accepted, e.g. any token of a Keycloak realm, regardless of the client it was issued for. +Prefer <<_oauth_2,OAuth 2>>, which also checks the issuer, the audience and the scope. + +=== Simple + +The HMS takes the user name from the `x-actor-username` HTTP header of each request, without verifying it: + +[source,yaml] +---- +spec: + clusterConfig: + icebergRestCatalog: + enabled: true + authentication: + simple: {} +---- + +WARNING: Clients can claim to be any user, so this mode is about as secure as no authentication. +Only use it for testing, e.g. of authorization rules. + +== Disabling the Thrift API + +If clients only use the REST catalog, you can disable the Thrift API: + +[source,yaml] +---- +spec: + clusterConfig: + thriftApi: + enabled: false + icebergRestCatalog: + enabled: true +---- + +The HMS can not switch its Thrift server off, so the operator binds it to the loopback interface of the Pod instead. +The Thrift port is then neither exposed by the Listener nor by the Services, and the discovery ConfigMap does not contain the `HIVE` entry. +At least one of the two APIs must be enabled. diff --git a/docs/modules/hive/pages/usage-guide/listenerclass.adoc b/docs/modules/hive/pages/usage-guide/listenerclass.adoc index 50aa7579..dbee16ee 100644 --- a/docs/modules/hive/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hive/pages/usage-guide/listenerclass.adoc @@ -1,8 +1,8 @@ = Service exposition with ListenerClasses :description: Configure the Hive service exposure with listener classes: cluster-internal, external-unstable or external-stable -Apache Hive offers an API. -The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Metastore pods. +The Apache Hive metastore offers a Thrift API (port `9083`) and, optionally, an xref:usage-guide/iceberg-rest-catalog.adoc[Iceberg REST catalog] (port `9001`). +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Metastore pods, which exposes the enabled APIs. The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.metastore.roleConfig.listenerClass`: diff --git a/docs/modules/hive/pages/usage-guide/security.adoc b/docs/modules/hive/pages/usage-guide/security.adoc index fb9ddfe7..5742f6f2 100644 --- a/docs/modules/hive/pages/usage-guide/security.adoc +++ b/docs/modules/hive/pages/usage-guide/security.adoc @@ -1,9 +1,11 @@ = Security -:description: Secure Apache Hive with Kerberos authentication in Kubernetes. Configure Kerberos server, SecretClass, and access Hive securely with provided guides. +:description: Secure the Apache Hive metastore with Kerberos authentication for the Thrift API and OPA authorization in Kubernetes. Configure Kerberos server, SecretClass, and access Hive securely with provided guides. :opa-rego-docs: https://www.openpolicyagent.org/docs/latest/#rego == Authentication -Currently, the only supported authentication mechanism is Kerberos, which is disabled by default. +The Thrift API supports Kerberos authentication, which is disabled by default and described below. +Authentication for the experimental Iceberg REST catalog (OAuth 2, JWT or simple) is described in xref:usage-guide/iceberg-rest-catalog.adoc#authentication[Iceberg REST catalog]. + For Kerberos to work a Kerberos KDC is needed, which the users need to provide. The xref:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation] states which kind of Kerberos servers are supported and how they can be configured. @@ -36,17 +38,17 @@ IMPORTANT: HDFS and Hive need to use the same SecretClass (or at least use the s ---- spec: clusterConfig: - authentication: - kerberos: - secretClass: kerberos # Put your SecretClass name in here + thriftApi: + authentication: + kerberos: + secretClass: kerberos # Put your SecretClass name in here ---- The `kerberos.secretClass` is used to give Hive the possibility to request keytabs from the secret-operator. === 5. Access Hive In case you want to access Hive it is recommended to start up a client Pod that connects to Hive, rather than shelling into the master. -We have an https://github.com/stackabletech/hive-operator/blob/main/tests/templates/kuttl/kerberos/70-install-access-hive.yaml.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. - +We have an https://github.com/stackabletech/hive-operator/blob/main/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. == Authorization The Stackable Operator for Apache Hive supports the following authorization methods. diff --git a/docs/modules/hive/partials/nav.adoc b/docs/modules/hive/partials/nav.adoc index aeca34a7..ffeadaf1 100644 --- a/docs/modules/hive/partials/nav.adoc +++ b/docs/modules/hive/partials/nav.adoc @@ -5,6 +5,7 @@ * xref:hive:usage-guide/index.adoc[] ** xref:hive:usage-guide/listenerclass.adoc[] ** xref:hive:usage-guide/data-storage.adoc[] +** xref:hive:usage-guide/iceberg-rest-catalog.adoc[] ** xref:hive:usage-guide/derby-example.adoc[] ** xref:hive:usage-guide/database-driver.adoc[] ** xref:hive:usage-guide/thrift-api-compatibility.adoc[] @@ -20,7 +21,7 @@ *** xref:hive:usage-guide/operations/graceful-shutdown.adoc[] * xref:hive:reference/index.adoc[] ** xref:hive:reference/crds.adoc[] -*** {crd-ref}/hive.stackable.tech/v1alpha1/hivecluster{crd-ref-release}[HiveCluster {external-link-icon}^] +*** {crd-ref}/hive.stackable.tech/v1alpha2/hivecluster{crd-ref-release}[HiveCluster {external-link-icon}^] ** xref:hive:reference/discovery.adoc[] ** xref:hive:reference/commandline-parameters.adoc[] ** xref:hive:reference/environment-variables.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index 7783eae9..680deb55 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -13,6 +13,1616 @@ spec: singular: hivecluster scope: Namespaced versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: A Hive cluster stacklet. This resource is managed by the Stackable operator for Apache Hive. + properties: + spec: + description: |- + A Hive cluster stacklet. This resource is managed by the Stackable operator for Apache Hive. + Find more information on how to use it and the resources that the operator generates in the + [operator documentation](https://docs.stackable.tech/home/nightly/hive/). + properties: + clusterConfig: + description: |- + Hive metastore settings that affect all roles and role groups. + The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. + properties: + authorization: + description: |- + Authorization options for Hive. + Learn more in the [Hive authorization usage guide](https://docs.stackable.tech/home/nightly/hive/usage-guide/security#authorization). + nullable: true + properties: + opa: + description: |- + Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + and the name of the Rego package containing your authorization rules. + Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) + to learn how to deploy Rego authorization rules with OPA. + nullable: true + properties: + configMapName: + description: |- + The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + for the OPA stacklet that should be used for authorization requests. + type: string + package: + description: The name of the Rego package containing the Rego rules for the product. + nullable: true + type: string + required: + - configMapName + type: object + type: object + hdfs: + description: HDFS connection specification. + nullable: true + properties: + configMap: + description: |- + Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + providing information about the HDFS cluster. + See also the [Stackable Operator for HDFS](https://docs.stackable.tech/home/nightly/hdfs/) to learn + more about setting up an HDFS cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + icebergRestCatalog: + default: + authentication: null + enabled: false + description: Experimental support for starting an Iceberg REST catalog server (disabled by default). + properties: + authentication: + description: |- + Settings related to user [authentication](https://docs.stackable.tech/home/nightly/hive/usage-guide/iceberg-rest-catalog#authentication). + + If not specified, the Iceberg REST catalog does not authenticate requests at all. + nullable: true + oneOf: + - required: + - simple + - required: + - oAuth2 + - required: + - jwt + properties: + jwt: + description: |- + Configure Hive Metastore so that its Iceberg REST API accepts a JSON Web Token (JWT) as a + bearer token in the Authorization header. Hive Metastore derives the username from the `sub` + claim of a properly-signed JWT. + + Only the signature and the expiry of the JWT are checked, not its issuer or audience. + Prefer `oAuth2` where possible. + properties: + jwksUrls: + description: |- + URLs of the JSON Web Key Set (JWKS) used to validate the signature of the JWTs. + + Maps to the `metastore.authentication.jwt.jwks.url` setting. Hive only falls back to + another URL if fetching the JWKS fails, so all URLs need to serve the same keys. Hive does + not preserve the order of the URLs, and the URLs must not contain commas. + + TODO: Should this be a struct instead? + items: + format: uri + type: string + minItems: 1 + type: array + required: + - jwksUrls + type: object + oAuth2: + description: |- + OAuth 2 is the industry standard for authenticating Iceberg client usernames. You can + integrate Hive Metastore with your Authorization Server, e.g., Keycloak, to protect Iceberg + resources. + properties: + audience: + description: The audience the JWT access tokens must be issued for. + type: string + issuerUrl: + description: |- + URL of the OAuth 2 authorization server (issuer), e.g. a Keycloak realm. + + The Hive metastore fetches the server metadata from + `/.well-known/oauth-authorization-server` on startup, which must include the + `jwks_uri` used to validate access tokens. + + TODO: Should this be a struct instead? + format: uri + type: string + required: + - audience + - issuerUrl + type: object + simple: + description: |- + Hive Metastore assumes that the value of the `x-actor-username` HTTP header is the + authenticated username, without verifying it. You may use it to test authorized access, + but this mode is not recommended in a production environment. + type: object + type: object + enabled: + default: false + description: |- + Whether the Iceberg REST catalog is started (defaults to `false`). + Requires Hive 4.2.0 or newer. + type: boolean + type: object + metadataDatabase: + description: |- + Configure the database where the Hive metastore stores all it's internal metadata, such + as schema and tables. + oneOf: + - required: + - postgresql + - required: + - mysql + - required: + - derby + properties: + derby: + description: |- + Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database. + + Derby is an embedded, file-based Java database engine that requires no separate server process. + It is typically used for development, testing, or as a lightweight metastore backend (e.g. for + Apache Hive). + properties: + location: + description: |- + Path on the filesystem where Derby stores its database files. + + If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`. + The `{unique_database_name}` part is automatically handled by the operator and is added to + prevent clashing database files. The `create=true` flag is always appended to the JDBC URL, + so the database is created automatically if it does not yet exist at this location. + nullable: true + type: string + type: object + mysql: + description: |- + Connection settings for a [MySQL](https://www.mysql.com/) database. + + Please note that - due to license issues - we don't ship the mysql driver, you need to add + it it yourself. + See for details. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the MySQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the MySQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 3306 + description: Port the MySQL server is listening on. Defaults to `3306`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + type: object + s3: + description: |- + S3 connection specification. This can be either `inline` or a `reference` to an + S3Connection object. Read the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3) to learn more. + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing the requested secrets.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + thriftApi: + default: + authentication: null + enabled: true + description: Start the Hive metastore Thrift API (enabled by default). + properties: + authentication: + description: |- + Settings related to user [authentication](https://docs.stackable.tech/home/nightly/hive/usage-guide/security). + + Only Kerberos is supported. + nullable: true + properties: + kerberos: + description: Kerberos configuration. + properties: + secretClass: + description: Name of the SecretClass providing the keytab for the Hive metastore. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - secretClass + type: object + required: + - kerberos + type: object + enabled: + default: true + description: |- + Whether the Thrift API is exposed (defaults to `true`). + + Hive can not switch the Thrift server off, so if disabled it is bound to the loopback + interface and not exposed via the Service and Listener. + type: boolean + type: object + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + warehouseDir: + description: |- + The location of default database for the Hive warehouse (e.g. `s3a://my-bucket/my-warehouse`). + + Maps to the `hive.metastore.warehouse.dir` setting. + type: string + warehouseExternalDir: + description: |- + Optional external location of Hive warehouse (e.g. `s3a://my-bucket/my-warehouse/external`). + + Maps to the `hive.metastore.warehouse.external.dir` setting. If not specified the operator + will *not* pass the setting to the Hive metastore and will let it default internally. + nullable: true + type: string + required: + - metadataDatabase + - warehouseDir + type: object + clusterOperation: + default: + reconciliationPaused: false + stopped: false + description: |- + [Cluster operations](https://docs.stackable.tech/home/nightly/concepts/operations/cluster_operations) + properties, allow stopping the product instance as well as pausing reconciliation. + properties: + reconciliationPaused: + default: false + description: |- + Flag to stop cluster reconciliation by the operator. This means that all changes in the + custom resource spec are ignored until this flag is set to false or removed. The operator + will however still watch the deployed resources at the time and update the custom resource + status field. + If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over + `stopped` and stop the reconciliation immediately. + type: boolean + stopped: + default: false + description: |- + Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, + ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 + and therefore stopped and removed. + If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation + and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. + type: boolean + type: object + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + - null + nullable: true + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `26.7.0` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `26.7.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + + If this is used in combination with `stackableVersionPolicy: LatestPatch`, the correct + floating tag is computed automatically, e.g. `26.7.0` becomes `26.7` for the product image. + nullable: true + type: string + stackableVersionPolicy: + default: Exact + description: |- + Configure the Stackable version policy. Defaults to `Exact`. + + Currently, two variants are supported: + + - `Exact`, which uses the exact, fully-qualified, canonical version of a product image. + - `LatestPatch`, referencing a floating tag which always points to the latest patch version + in the current release line. The current release line is either automatically derived by + the operator based on its own version, or can be overridden with `stackableVersion`. + + A potential newer image is only pulled when Pods are rotated or their containers are + restarted. Pods are NOT rotated and containers are NOT restarted automatically when a new + image is available. This behaviour makes this a passive update mechanism, rather than an + active one. + + It should be noted that when this field is set to `LatestPatch`, the operator automatically + uses `Always` as the pull policy for product images. If set to `Exact`, `IfNotPresent` is + used. Explicitly setting `pullPolicy` takes precedence. + + ### Examples + + - The `stackableVersion` field is not set, the operator falls back to its own version, eg. + 26.7.0. If this field is set to `LatestPatch`, the `26.7` floating tag will be used for + product images, else, `26.7.0` will be used. + - The `stackableVersion` field is set to `26.3.0`. If this field is set to `LatestPatch`, + the `26.3` floating tag will be used for product images, else, `26.3.0` will be used. + enum: + - Exact + - LatestPatch + type: string + type: object + metastore: + description: |- + This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing + all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable + at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured + on role level. There is also a second form of config, which can only be configured + at role level, the `roleConfig`. + You can learn more about this in the + [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + hive: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: + data: + capacity: null + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + properties: + data: + default: + capacity: null + description: |- + This field is deprecated. It was never used by Hive and will be removed in a future + CRD version. The controller will warn if it's set to a non zero value. + properties: + capacity: + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + nullable: true + x-kubernetes-int-or-string: true + selectors: + description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + storageClass: + nullable: true + type: string + type: object + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + hive-site.xml: + additionalProperties: + type: string + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. + type: object + security.properties: + additionalProperties: + type: string + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the metastore. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + hive: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: + data: + capacity: null + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + properties: + data: + default: + capacity: null + description: |- + This field is deprecated. It was never used by Hive and will be removed in a future + CRD version. The controller will warn if it's set to a non zero value. + properties: + capacity: + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + nullable: true + x-kubernetes-int-or-string: true + selectors: + description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + storageClass: + nullable: true + type: string + type: object + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + hive-site.xml: + additionalProperties: + type: string + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. + type: object + security.properties: + additionalProperties: + type: string + default: {} + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + description: |- + The set of role groups for this role, keyed by their name. + + A role group is a subset of the replicas of a role that share the same configuration, + allowing finer-grained control than the role level. This is useful to e.g. schedule groups + onto different classes of nodes or into different regions, or to run them with different + settings. Configuration set on a role group is merged on top of the role-level `config`, + with the more specific role group values taking precedence. + + Every role needs at least one role group. A role with a single role group conventionally + names it `default`. + + Read the + [roles and role groups concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups) + for more details. + type: object + required: + - roleGroups + type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - clusterConfig + - image + - metastore + type: object + status: + nullable: true + properties: + changedValues: + description: Contains changed values during upgrades and downgrades of CRDs. + properties: + downgrades: + additionalProperties: + items: + description: Contains a changed value for a single field of the CRD. + properties: + downgradedValue: + description: |- + The value the field was converted to during the downgrade. Only set for fields which + changed their type. + + When upgrading again, the tracked `value` is only applied if the field still contains this + value. Otherwise, the field was changed by a user in the older version and that change is + kept instead. + nullable: true + x-kubernetes-preserve-unknown-fields: true + jsonPath: + description: The name of the field of the custom resource this value is for. + type: string + value: + description: The value to be used when upgrading or downgrading the custom resource. + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - jsonPath + - value + type: object + type: array + description: List of values needed when downgrading to a particular version. + type: object + upgrades: + additionalProperties: + items: + description: Contains a changed value for a single field of the CRD. + properties: + downgradedValue: + description: |- + The value the field was converted to during the downgrade. Only set for fields which + changed their type. + + When upgrading again, the tracked `value` is only applied if the field still contains this + value. Otherwise, the field was changed by a user in the older version and that change is + kept instead. + nullable: true + x-kubernetes-preserve-unknown-fields: true + jsonPath: + description: The name of the field of the custom resource this value is for. + type: string + value: + description: The value to be used when upgrading or downgrading the custom resource. + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - jsonPath + - value + type: object + type: array + description: List of values needed when upgrading to a particular version. + type: object + type: object + conditions: + default: [] + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + nullable: true + type: string + message: + description: A human readable message indicating details about the transition. + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - 'True' + - 'False' + - Unknown + type: string + type: + description: Type of deployment condition. + enum: + - Available + - Degraded + - Progressing + - ReconciliationPaused + - Stopped + type: string + required: + - status + - type + type: object + type: array + discoveryHash: + description: An opaque value that changes every time a discovery detail does + nullable: true + type: string + type: object + required: + - spec + title: HiveCluster + type: object + served: true + storage: true + subresources: + status: {} - name: v1alpha1 schema: openAPIV3Schema: @@ -30,14 +1640,17 @@ spec: The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. properties: authentication: - description: Settings related to user [authentication](https://docs.stackable.tech/home/nightly/hive/usage-guide/security). + description: |- + Settings related to user [authentication](https://docs.stackable.tech/home/nightly/hive/usage-guide/security). + + Moved to `.spec.clusterConfig.thriftApi.authentication` in v1alpha2. nullable: true properties: kerberos: description: Kerberos configuration. properties: secretClass: - description: Name of the SecretClass providing the keytab for the HBase services. + description: Name of the SecretClass providing the keytab for the Hive metastore. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ @@ -827,6 +2440,8 @@ spec: description: |- The location of default database for the Hive warehouse. Maps to the `hive.metastore.warehouse.dir` setting. + + Moved to `.spec.clusterConfig.warehouseDir` in v1alpha2. nullable: true type: string type: object @@ -1280,6 +2895,8 @@ spec: description: |- The location of default database for the Hive warehouse. Maps to the `hive.metastore.warehouse.dir` setting. + + Moved to `.spec.clusterConfig.warehouseDir` in v1alpha2. nullable: true type: string type: object @@ -1405,6 +3022,68 @@ spec: status: nullable: true properties: + changedValues: + description: Contains changed values during upgrades and downgrades of CRDs. + properties: + downgrades: + additionalProperties: + items: + description: Contains a changed value for a single field of the CRD. + properties: + downgradedValue: + description: |- + The value the field was converted to during the downgrade. Only set for fields which + changed their type. + + When upgrading again, the tracked `value` is only applied if the field still contains this + value. Otherwise, the field was changed by a user in the older version and that change is + kept instead. + nullable: true + x-kubernetes-preserve-unknown-fields: true + jsonPath: + description: The name of the field of the custom resource this value is for. + type: string + value: + description: The value to be used when upgrading or downgrading the custom resource. + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - jsonPath + - value + type: object + type: array + description: List of values needed when downgrading to a particular version. + type: object + upgrades: + additionalProperties: + items: + description: Contains a changed value for a single field of the CRD. + properties: + downgradedValue: + description: |- + The value the field was converted to during the downgrade. Only set for fields which + changed their type. + + When upgrading again, the tracked `value` is only applied if the field still contains this + value. Otherwise, the field was changed by a user in the older version and that change is + kept instead. + nullable: true + x-kubernetes-preserve-unknown-fields: true + jsonPath: + description: The name of the field of the custom resource this value is for. + type: string + value: + description: The value to be used when upgrading or downgrading the custom resource. + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - jsonPath + - value + type: object + type: array + description: List of values needed when upgrading to a particular version. + type: object + type: object conditions: default: [] items: @@ -1453,6 +3132,6 @@ spec: title: HiveCluster type: object served: true - storage: true + storage: false subresources: status: {} diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index cd10a5bb..ddc3792e 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -25,6 +25,7 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +url.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 167cf6ea..fc923d91 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,4 +1,4 @@ -//! Ensures that `Pod`s are configured and running for each [`v1alpha1::HiveCluster`] +//! Ensures that `Pod`s are configured and running for each [`v1alpha2::HiveCluster`] mod apply; mod build; @@ -41,6 +41,7 @@ use stackable_operator::{ role_group_utils::ResourceNames, role_utils, types::{ + common::Port, kubernetes::{ListenerClassName, SecretClassName}, operator::{ControllerName, OperatorName, ProductName, ProductVersion}, }, @@ -51,7 +52,12 @@ use strum::EnumDiscriminants; use crate::{ HIVE_OPERATOR_NAME, controller::{apply::Applier, update_status::update_status}, - crd::{APP_NAME, HdfsConnection, HiveRole, MetaStoreConfig, v1alpha1}, + crd::{ + APP_NAME, HIVE_PORT, HIVE_PORT_NAME, HdfsConnection, HiveRole, ICEBERG_REST_CATALOG_PORT, + ICEBERG_REST_CATALOG_PORT_NAME, + security::IcebergRestCatalogAuthenticationConfig, + v1alpha2::{self, MetaStoreConfig}, + }, }; pub const HIVE_CONTROLLER_NAME: &str = "hivecluster"; @@ -105,7 +111,7 @@ impl ReconcilerError for Error { pub type HiveRoleGroupConfig = stackable_operator::v2::role_utils::RoleGroupConfig< ValidatedMetaStoreConfig, stackable_operator::v2::role_utils::JavaCommonConfig, - v1alpha1::HiveConfigOverrides, + v1alpha2::HiveConfigOverrides, >; /// A validated Hive metastore config: the merged [`MetaStoreConfig`] with its raw `logging` @@ -118,7 +124,6 @@ pub struct ValidatedMetaStoreConfig { pub graceful_shutdown_timeout: Option, pub logging: validate::ValidatedLogging, pub resources: Resources, - pub warehouse_dir: Option, } impl ValidatedMetaStoreConfig { @@ -126,34 +131,12 @@ impl ValidatedMetaStoreConfig { /// already-validated logging. fn from_merged(merged: MetaStoreConfig, logging: validate::ValidatedLogging) -> Self { Self { - warehouse_dir: merged.warehouse_dir, resources: merged.resources, logging, affinity: merged.affinity, graceful_shutdown_timeout: merged.graceful_shutdown_timeout, } } - - /// Wraps a merged [`MetaStoreConfig`] with trivial automatic logging, for builder tests - /// that only exercise the non-logging parts of the config. - #[cfg(test)] - pub(crate) fn from_merged_for_test(merged: MetaStoreConfig) -> Self { - use stackable_operator::{ - product_logging::spec::AutomaticContainerLogConfig, - v2::product_logging::framework::ValidatedContainerLogConfigChoice, - }; - - Self::from_merged( - merged, - validate::ValidatedLogging { - hive_container: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), - vector_container: None, - enable_vector_agent: false, - }, - ) - } } /// The validated cluster: the typed, merged result of the validate step. Subsequent @@ -161,7 +144,7 @@ impl ValidatedMetaStoreConfig { /// /// The cluster identity (`name`, `namespace`, `uid`) is captured here so that owner /// references for child objects can be built straight from this struct -/// (via its [`Resource`] impl) without threading the raw [`v1alpha1::HiveCluster`] +/// (via its [`Resource`] impl) without threading the raw [`v1alpha2::HiveCluster`] /// around. pub struct ValidatedCluster { /// `ObjectMeta` carrying `name`, `namespace` and `uid`, so this struct can act as the @@ -246,33 +229,34 @@ impl ValidatedCluster { } } - /// Whether Kerberos is enabled for this cluster (a Kerberos `SecretClass` was configured). - pub fn has_kerberos_enabled(&self) -> bool { - self.cluster_config.kerberos_secret_class.is_some() + /// Whether Kerberos is enabled for this cluster (a Kerberos `SecretClass` was configured for + /// the thrift API). + pub fn has_thrift_kerberos_enabled(&self) -> bool { + self.cluster_config.thrift_kerberos_secret_class.is_some() } } -/// Lets [`ValidatedCluster`] stand in for the raw [`v1alpha1::HiveCluster`] when building owner +/// Lets [`ValidatedCluster`] stand in for the raw [`v1alpha2::HiveCluster`] when building owner /// references and metadata for child objects. Kind/group/version are delegated to the CRD; the /// `metadata` (name, namespace, uid) is captured during validation. impl Resource for ValidatedCluster { - type DynamicType = ::DynamicType; - type Scope = ::Scope; + type DynamicType = ::DynamicType; + type Scope = ::Scope; fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HiveCluster::kind(dt) + v1alpha2::HiveCluster::kind(dt) } fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HiveCluster::group(dt) + v1alpha2::HiveCluster::group(dt) } fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HiveCluster::version(dt) + v1alpha2::HiveCluster::version(dt) } fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HiveCluster::plural(dt) + v1alpha2::HiveCluster::plural(dt) } fn meta(&self) -> &ObjectMeta { @@ -305,7 +289,7 @@ impl NameIsValidLabelValue for ValidatedCluster { /// Cluster-wide settings resolved during validation and dereferencing. /// /// Everything the config-file builders need is resolved here so they never have to -/// read the raw [`v1alpha1::HiveCluster`] spec. +/// read the raw [`v1alpha2::HiveCluster`] spec. pub struct ValidatedClusterConfig { pub metadata_database_connection_details: JdbcDatabaseConnectionDetails, /// The resolved JDBC driver class (Derby version special-casing already applied). @@ -316,15 +300,35 @@ pub struct ValidatedClusterConfig { pub hdfs: Option, pub s3_connection_spec: Option, pub hive_opa_config: Option, - /// The Kerberos `SecretClass` name, if Kerberos is enabled. - pub kerberos_secret_class: Option, + + pub thrift_api_enabled: bool, + pub iceberg_rest_catalog_enabled: bool, + + pub warehouse_dir: String, + pub warehouse_external_dir: Option, + + /// The Kerberos `SecretClass` name, if Kerberos is enabled for the Thrift API. + pub thrift_kerberos_secret_class: Option, + pub iceberg_rest_catalog_security: Option, } impl ValidatedClusterConfig { /// Whether a `core-site.xml` with `hadoop.security.authentication=kerberos` is required: /// Kerberos is enabled and there is no HDFS backend (i.e. S3). pub fn needs_kerberos_core_site(&self) -> bool { - self.kerberos_secret_class.is_some() && self.hdfs.is_none() + self.thrift_kerberos_secret_class.is_some() && self.hdfs.is_none() + } + + /// The ports of the enabled APIs (Thrift and/or Iceberg REST catalog), as `(name, port)`. + pub fn api_ports(&self) -> Vec<(&'static str, Port)> { + let mut ports = Vec::new(); + if self.thrift_api_enabled { + ports.push((HIVE_PORT_NAME, HIVE_PORT)); + } + if self.iceberg_rest_catalog_enabled { + ports.push((ICEBERG_REST_CATALOG_PORT_NAME, ICEBERG_REST_CATALOG_PORT)); + } + ports } } @@ -358,7 +362,7 @@ pub struct KubernetesResources { } pub async fn reconcile_hive( - hive: Arc>, + hive: Arc>, ctx: Arc, ) -> Result { tracing::info!("Starting reconcile"); @@ -408,7 +412,7 @@ pub async fn reconcile_hive( } pub fn error_policy( - _obj: Arc>, + _obj: Arc>, error: &Error, _ctx: Arc, ) -> Action { @@ -424,7 +428,7 @@ pub(crate) mod test_support { use stackable_operator::utils::yaml_from_str_singleton_map; use super::{ValidatedCluster, dereference::DereferencedObjects, validate::validate_cluster}; - use crate::crd::v1alpha1; + use crate::crd::v1alpha2; /// The expected `app.kubernetes.io/version` label value for the given product version. /// @@ -444,7 +448,7 @@ pub(crate) mod test_support { /// (`simple-hive`) deliberately differs from the product name (`hive`), so tests asserting /// recommended labels catch swapped `name`/`instance` values. pub const DERBY_YAML: &str = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -456,18 +460,19 @@ pub(crate) mod test_support { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: replicas: 1 "#; - pub fn minimal_hive(yaml: &str) -> v1alpha1::HiveCluster { + pub fn minimal_hive(yaml: &str) -> v1alpha2::HiveCluster { yaml_from_str_singleton_map(yaml).expect("invalid test HiveCluster YAML") } /// Runs the real validate step against a minimal (S3/OPA-free) fixture. - pub fn validated_cluster(hive: &v1alpha1::HiveCluster) -> ValidatedCluster { + pub fn validated_cluster(hive: &v1alpha2::HiveCluster) -> ValidatedCluster { validate_cluster( hive, "oci.example.org", @@ -510,7 +515,7 @@ mod tests { fn reconcile_exits_early_for_deleted_cluster() { let hive = yaml_serde::from_str( r#" -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index b3132329..aba80e6a 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -35,7 +35,7 @@ fn construct_jvm_args(cluster: &ValidatedCluster, rg: &HiveRoleGroupConfig) -> V "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), ]; - if cluster.has_kerberos_enabled() { + if cluster.has_thrift_kerberos_enabled() { jvm_args.push(format!( "-Djava.security.krb5.conf={STACKABLE_KERBEROS_DIR}/krb5.conf" )); @@ -92,7 +92,7 @@ mod tests { }; fn metastore_default( - hive: &crate::crd::v1alpha1::HiveCluster, + hive: &crate::crd::v1alpha2::HiveCluster, ) -> (ValidatedCluster, HiveRoleGroupConfig) { let validated = validated_cluster(hive); let rg = validated @@ -107,7 +107,7 @@ mod tests { #[test] fn test_construct_jvm_arguments_defaults() { let input = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -119,6 +119,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: @@ -145,7 +146,7 @@ mod tests { #[test] fn test_construct_jvm_argument_overrides() { let input = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -157,6 +158,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: config: resources: diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index cc79b209..9799b9d8 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -66,7 +66,7 @@ pub fn add_kerberos_pod_config( cb: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<(), Error> { - if let Some(kerberos_secret_class) = &cluster.cluster_config.kerberos_secret_class { + if let Some(kerberos_secret_class) = &cluster.cluster_config.thrift_kerberos_secret_class { // Mount keytab let kerberos_secret_operator_volume = SecretOperatorVolumeSourceBuilder::new( kerberos_secret_class.clone(), @@ -96,7 +96,7 @@ pub fn add_kerberos_pod_config( /// Returned as an [`EnvVarSet`] (rather than added to the container directly) so the caller can /// merge the user's `envOverrides` on top, letting an override win on a name collision. pub fn kerberos_env_vars(cluster: &ValidatedCluster) -> EnvVarSet { - if !cluster.has_kerberos_enabled() { + if !cluster.has_thrift_kerberos_enabled() { return EnvVarSet::new(); } EnvVarSet::new().with_value(&KRB5_CONFIG, format!("{STACKABLE_KERBEROS_DIR}/krb5.conf")) @@ -137,7 +137,7 @@ pub fn kerberos_config_properties( } pub fn kerberos_container_start_commands(cluster: &ValidatedCluster) -> String { - if !cluster.has_kerberos_enabled() { + if !cluster.has_thrift_kerberos_enabled() { return String::new(); } diff --git a/rust/operator-binary/src/controller/build/properties/core_site.rs b/rust/operator-binary/src/controller/build/properties/core_site.rs index f068a313..284e692e 100644 --- a/rust/operator-binary/src/controller/build/properties/core_site.rs +++ b/rust/operator-binary/src/controller/build/properties/core_site.rs @@ -38,7 +38,7 @@ mod tests { // Kerberos enabled (a SecretClass is set) and no HDFS backend (the fixture has none). let mut cluster_config = derby_cluster_config(); - cluster_config.kerberos_secret_class = + cluster_config.thrift_kerberos_secret_class = Some(SecretClassName::from_str("kerberos").expect("valid secret class name")); let data = build(&cluster_config).expect("core-site present"); assert_eq!( diff --git a/rust/operator-binary/src/controller/build/properties/hive_site.rs b/rust/operator-binary/src/controller/build/properties/hive_site.rs index 448a7327..6bf5a8b9 100644 --- a/rust/operator-binary/src/controller/build/properties/hive_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hive_site.rs @@ -5,14 +5,21 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{crd::s3, k8s_openapi::api::core::v1::EnvVar}; use crate::{ - controller::{ - ValidatedClusterConfig, ValidatedMetaStoreConfig, build::opa::build_opa_hive_site_config, - }, - crd::HIVE_PORT, + controller::{ValidatedClusterConfig, build::opa::build_opa_hive_site_config}, + crd::{HIVE_PORT, ICEBERG_REST_CATALOG_PORT, security::IcebergRestCatalogAuthenticationConfig}, }; -const DEFAULT_WAREHOUSE_DIR: &str = "/stackable/warehouse"; const HIVE_METASTORE_PORT: &str = "hive.metastore.port"; +const METASTORE_THRIFT_BIND_HOST: &str = "hive.metastore.thrift.bind.host"; +const METASTORE_CATALOG_SERVLET_PORT: &str = "metastore.catalog.servlet.port"; + +// Iceberg REST catalog authentication property keys. +const METASTORE_CATALOG_SERVLET_AUTH: &str = "metastore.catalog.servlet.auth"; +const METASTORE_CATALOG_SERVLET_AUTH_OAUTH2_ISSUER: &str = + "metastore.catalog.servlet.auth.oauth2.issuer"; +const METASTORE_CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE: &str = + "metastore.catalog.servlet.auth.oauth2.audience"; +const METASTORE_AUTHENTICATION_JWT_JWKS_URL: &str = "metastore.authentication.jwt.jwks.url"; // Metastore property keys. const CONNECTION_DRIVER_NAME: &str = "javax.jdo.option.ConnectionDriverName"; @@ -21,6 +28,7 @@ const CONNECTION_URL: &str = "javax.jdo.option.ConnectionURL"; const CONNECTION_USER_NAME: &str = "javax.jdo.option.ConnectionUserName"; const METASTORE_METRICS_ENABLED: &str = "hive.metastore.metrics.enabled"; const METASTORE_WAREHOUSE_DIR: &str = "hive.metastore.warehouse.dir"; +const METASTORE_WAREHOUSE_EXTERNAL_DIR: &str = "hive.metastore.warehouse.external.dir"; // S3 property keys. const S3_ACCESS_KEY: &str = "fs.s3a.access.key"; @@ -47,23 +55,69 @@ type Result = std::result::Result; pub fn build( cluster_config: &ValidatedClusterConfig, product_version: &str, - merged_config: &ValidatedMetaStoreConfig, kerberos_config: BTreeMap, overrides: BTreeMap, ) -> Result> { let database_connection_details = &cluster_config.metadata_database_connection_details; let mut data: BTreeMap = BTreeMap::new(); - // 1. Defaults. + // Defaults. + data.insert(METASTORE_METRICS_ENABLED.to_string(), "true".to_string()); + + // HMS can not disable the Thrift server: it always listens (on 9083 if no port is set) and + // its main loop is what the background threads (compaction, housekeeping) wait for. If the + // Thrift API is disabled, we bind it to the loopback interface so it can not be reached from + // outside the Pod. The embedded Iceberg REST catalog talks to HMS in-process, not via Thrift. + // Note that the bind host is only honoured by the (default) binary transport mode. data.insert(HIVE_METASTORE_PORT.to_string(), HIVE_PORT.to_string()); + if !cluster_config.thrift_api_enabled { + data.insert( + METASTORE_THRIFT_BIND_HOST.to_string(), + "127.0.0.1".to_string(), + ); + } - // 2. Automatic / operator-injected. - data.insert( - METASTORE_WAREHOUSE_DIR.to_string(), - DEFAULT_WAREHOUSE_DIR.to_string(), - ); - data.insert(METASTORE_METRICS_ENABLED.to_string(), "true".to_string()); + if cluster_config.iceberg_rest_catalog_enabled { + // The REST catalog is only started for a non-negative port (defaults to -1). + data.insert( + METASTORE_CATALOG_SERVLET_PORT.to_string(), + ICEBERG_REST_CATALOG_PORT.to_string(), + ); + + let auth_type = match &cluster_config.iceberg_rest_catalog_security { + None => "none", + Some(IcebergRestCatalogAuthenticationConfig::Simple {}) => "simple", + Some(IcebergRestCatalogAuthenticationConfig::OAuth2(oauth2)) => { + data.insert( + METASTORE_CATALOG_SERVLET_AUTH_OAUTH2_ISSUER.to_string(), + oauth2.issuer_url.to_string(), + ); + data.insert( + METASTORE_CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE.to_string(), + oauth2.audience.to_string(), + ); + "oauth2" + } + Some(IcebergRestCatalogAuthenticationConfig::Jwt(jwt)) => { + // Hive splits the setting on commas. + data.insert( + METASTORE_AUTHENTICATION_JWT_JWKS_URL.to_string(), + jwt.jwks_urls + .iter() + .map(|url| url.as_str()) + .collect::>() + .join(","), + ); + "jwt" + } + }; + data.insert( + METASTORE_CATALOG_SERVLET_AUTH.to_string(), + auth_type.to_string(), + ); + } + // Metadata database connection data.insert( CONNECTION_DRIVER_NAME.to_string(), cluster_config.connection_driver.clone(), @@ -79,6 +133,7 @@ pub fn build( data.insert(CONNECTION_PASSWORD.to_string(), format!("${{env:{name}}}")); } + // S3 connection if let Some(s3) = cluster_config.s3_connection_spec.as_ref() { data.insert( S3_ENDPOINT.to_string(), @@ -124,12 +179,19 @@ pub fn build( data.extend(build_opa_hive_site_config(opa_config, product_version)); } - // 3. Spec: warehouse dir from the merged CRD config (overrides the default). - if let Some(warehouse_dir) = &merged_config.warehouse_dir { - data.insert(METASTORE_WAREHOUSE_DIR.to_string(), warehouse_dir.clone()); + // Warehouse dir from the clusterConfig. + data.insert( + METASTORE_WAREHOUSE_DIR.to_string(), + cluster_config.warehouse_dir.clone(), + ); + if let Some(warehouse_external_dir) = &cluster_config.warehouse_external_dir { + data.insert( + METASTORE_WAREHOUSE_EXTERNAL_DIR.to_string(), + warehouse_external_dir.clone(), + ); } - // 4. User overrides (highest precedence). + // User overrides (highest precedence). data.extend(overrides); Ok(data) @@ -138,24 +200,16 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::{ - controller::build::properties::test_support::{derby_cluster_config, s3_cluster_config}, - crd::MetaStoreConfig, + use crate::controller::build::properties::test_support::{ + derby_cluster_config, s3_cluster_config, }; #[test] fn defaults_present_for_minimal_derby_cluster() { let cluster_config = derby_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default()); - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - BTreeMap::new(), - ) - .expect("build hive-site"); + let data = build(&cluster_config, "4.0.0", BTreeMap::new(), BTreeMap::new()) + .expect("build hive-site"); assert_eq!(data.get("hive.metastore.port"), Some(&"9083".to_string())); assert_eq!( @@ -164,49 +218,42 @@ mod tests { ); assert_eq!( data.get("hive.metastore.warehouse.dir"), - Some(&"/stackable/warehouse".to_string()) + Some(&"s3a://my-bucket/warehouse/".to_string()) ); assert!(data.contains_key("javax.jdo.option.ConnectionDriverName")); // No env credentials for an embedded Derby database. assert!(!data.contains_key("javax.jdo.option.ConnectionUserName")); + // Thrift is reachable from outside the Pod and the REST catalog is not started. + assert!(!data.contains_key("hive.metastore.thrift.bind.host")); + assert!(!data.contains_key("metastore.catalog.servlet.port")); } #[test] - fn warehouse_dir_spec_overrides_default() { - let cluster_config = derby_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig { - warehouse_dir: Some("/custom/warehouse".to_string()), - ..MetaStoreConfig::default() - }); - - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - BTreeMap::new(), - ) - .expect("build hive-site"); + fn thrift_disabled_binds_to_loopback() { + let mut cluster_config = derby_cluster_config(); + cluster_config.thrift_api_enabled = false; + cluster_config.iceberg_rest_catalog_enabled = true; + let data = build(&cluster_config, "4.2.0", BTreeMap::new(), BTreeMap::new()) + .expect("build hive-site"); + + assert_eq!(data.get("hive.metastore.port"), Some(&"9083".to_string())); assert_eq!( - data.get("hive.metastore.warehouse.dir"), - Some(&"/custom/warehouse".to_string()) + data.get("hive.metastore.thrift.bind.host"), + Some(&"127.0.0.1".to_string()) + ); + assert_eq!( + data.get("metastore.catalog.servlet.port"), + Some(&"9001".to_string()) ); } #[test] fn s3_connection_emits_s3a_settings() { let cluster_config = s3_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default()); - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - BTreeMap::new(), - ) - .expect("build hive-site"); + let data = build(&cluster_config, "4.0.0", BTreeMap::new(), BTreeMap::new()) + .expect("build hive-site"); assert_eq!( data.get("fs.s3a.endpoint"), @@ -225,16 +272,9 @@ mod tests { #[test] fn s3_connection_registers_the_s3_scheme() { let cluster_config = s3_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default()); - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - BTreeMap::new(), - ) - .expect("build hive-site"); + let data = build(&cluster_config, "4.0.0", BTreeMap::new(), BTreeMap::new()) + .expect("build hive-site"); assert_eq!( data.get("fs.s3.impl"), @@ -245,16 +285,9 @@ mod tests { #[test] fn no_s3_connection_leaves_the_s3_scheme_unregistered() { let cluster_config = derby_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default()); - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - BTreeMap::new(), - ) - .expect("build hive-site"); + let data = build(&cluster_config, "4.0.0", BTreeMap::new(), BTreeMap::new()) + .expect("build hive-site"); assert!(!data.contains_key("fs.s3.impl")); } @@ -262,19 +295,12 @@ mod tests { #[test] fn user_override_wins_over_everything() { let cluster_config = derby_cluster_config(); - let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default()); let overrides = [("hive.metastore.port".to_string(), "1234".to_string())] .into_iter() .collect(); - let data = build( - &cluster_config, - "4.0.0", - &merged, - BTreeMap::new(), - overrides, - ) - .expect("build hive-site"); + let data = + build(&cluster_config, "4.0.0", BTreeMap::new(), overrides).expect("build hive-site"); assert_eq!(data.get("hive.metastore.port"), Some(&"1234".to_string())); } diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 93181588..d337baa2 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -45,13 +45,13 @@ pub(crate) mod test_support { controller::{ValidatedClusterConfig, test_support::DERBY_YAML}, crd::{ databases::{MetadataDatabaseConnection, derby_driver_class}, - v1alpha1, + v1alpha2, }, }; /// Build a minimal Derby-backed [`ValidatedClusterConfig`] for builder tests. pub fn derby_cluster_config() -> ValidatedClusterConfig { - let hive: v1alpha1::HiveCluster = + let hive: v1alpha2::HiveCluster = stackable_operator::utils::yaml_from_str_singleton_map(DERBY_YAML) .expect("valid HiveCluster YAML"); let metadata_database_connection_details = hive @@ -77,7 +77,21 @@ pub(crate) mod test_support { hdfs: None, s3_connection_spec: None, hive_opa_config: None, - kerberos_secret_class: None, + thrift_api_enabled: hive.spec.cluster_config.thrift_api.enabled, + iceberg_rest_catalog_enabled: hive.spec.cluster_config.iceberg_rest_catalog.enabled, + warehouse_dir: hive.spec.cluster_config.warehouse_dir.clone(), + warehouse_external_dir: hive.spec.cluster_config.warehouse_external_dir.clone(), + thrift_kerberos_secret_class: hive + .spec + .cluster_config + .thrift_api + .authentication + .map(|a| a.kerberos.secret_class), + iceberg_rest_catalog_security: hive + .spec + .cluster_config + .iceberg_rest_catalog + .authentication, } } diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 08c2d3cf..104a4c14 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -52,7 +52,7 @@ pub fn build_metastore_rolegroup_config_map( ) -> Result { // Kerberos-related hive-site.xml entries (empty when Kerberos is disabled). Computed here, in // the build step, from the validated cluster identity and the controller's cluster info. - let kerberos_config = if cluster.has_kerberos_enabled() { + let kerberos_config = if cluster.has_thrift_kerberos_enabled() { kerberos_config_properties( cluster.name.as_ref(), cluster.namespace.as_ref(), @@ -67,7 +67,6 @@ pub fn build_metastore_rolegroup_config_map( let hive_site_data = hive_site::build( &cluster.cluster_config, &cluster.image.product_version, - &rg.config, kerberos_config, hive_site_overrides, ) diff --git a/rust/operator-binary/src/controller/build/resource/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs index fb16d1b0..d2afbb92 100644 --- a/rust/operator-binary/src/controller/build/resource/discovery.rs +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -16,7 +16,7 @@ use crate::{ resource::listener::build_listener_connection_string, }, }, - crd::{HiveRole, v1alpha1}, + crd::{HiveRole, v1alpha2}, }; #[derive(Snafu, Debug)] @@ -24,7 +24,7 @@ pub enum Error { #[snafu(display("could not build discovery config map for {obj_ref}"))] DiscoveryConfigMap { source: stackable_operator::builder::configmap::Error, - obj_ref: ObjectRef, + obj_ref: ObjectRef, }, #[snafu(display("failed to configure listener discovery configmap"))] @@ -33,9 +33,9 @@ pub enum Error { }, } -/// An [`ObjectRef`] back to the owning [`v1alpha1::HiveCluster`], reconstructed from the validated +/// An [`ObjectRef`] back to the owning [`v1alpha2::HiveCluster`], reconstructed from the validated /// cluster identity for use in error messages. -fn cluster_object_ref(cluster: &ValidatedCluster) -> ObjectRef { +fn cluster_object_ref(cluster: &ValidatedCluster) -> ObjectRef { ObjectRef::new(cluster.name.as_ref()).within(cluster.namespace.as_ref()) } @@ -55,7 +55,7 @@ pub fn discovery_config_map_name(cluster_name: &ClusterName) -> ConfigMapName { } /// Builds the discovery [`ConfigMap`] containing information about how to connect to a certain -/// [`v1alpha1::HiveCluster`]. +/// [`v1alpha2::HiveCluster`]. /// /// The ConfigMap needs the role Listener's ingress address, which only the listener-operator /// writes. While the dereferenced Listener is absent or still address-less (around the first @@ -96,11 +96,17 @@ pub fn build_discovery_configmap( discovery_configmap.metadata(discovery_config_map_meta(cluster, hive_role)); - discovery_configmap.add_data( - "HIVE".to_string(), - build_listener_connection_string(listener_address, &hive_role.to_string()) - .context(ListenerConfigurationSnafu)?, - ); + // The Listener only exposes the Thrift port if the Thrift API is enabled. + if cluster.cluster_config.thrift_api_enabled { + discovery_configmap.add_data( + "HIVE".to_string(), + build_listener_connection_string(listener_address, &hive_role.to_string()) + .context(ListenerConfigurationSnafu)?, + ); + } + + // Note that we intentionally don't add an discovery entry for the Iceberg REST catalog, as we + // don't want to commit to more public API that will likely break with Discovery 2.0. let config_map = discovery_configmap .build() diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index c20a0548..0bb70542 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -14,7 +14,7 @@ use crate::{ ValidatedCluster, build::{object_meta, recommended_labels_for_role_resources}, }, - crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole}, + crd::{HIVE_PORT_NAME, HiveRole}, }; #[derive(Snafu, Debug)] @@ -77,7 +77,7 @@ pub fn build_role_listener( let spec = ListenerSpec { class_name: Some(listener_class.to_string()), - ports: Some(listener_ports()), + ports: Some(listener_ports(cluster)), ..Default::default() }; @@ -88,10 +88,15 @@ pub fn build_role_listener( } } -pub fn listener_ports() -> Vec { - vec![ListenerPort { - name: HIVE_PORT_NAME.to_owned(), - port: HIVE_PORT.into(), - protocol: Some("TCP".to_owned()), - }] +pub fn listener_ports(cluster: &ValidatedCluster) -> Vec { + cluster + .cluster_config + .api_ports() + .into_iter() + .map(|(name, port)| ListenerPort { + name: name.to_owned(), + port: port.into(), + protocol: Some("TCP".to_owned()), + }) + .collect() } diff --git a/rust/operator-binary/src/controller/build/resource/rbac.rs b/rust/operator-binary/src/controller/build/resource/rbac.rs index 608e2e25..7fafcb7c 100644 --- a/rust/operator-binary/src/controller/build/resource/rbac.rs +++ b/rust/operator-binary/src/controller/build/resource/rbac.rs @@ -60,7 +60,7 @@ mod tests { "namespace": "default", "ownerReferences": [ { - "apiVersion": "hive.stackable.tech/v1alpha1", + "apiVersion": "hive.stackable.tech/v1alpha2", "controller": true, "kind": "HiveCluster", "name": "simple-hive", @@ -94,7 +94,7 @@ mod tests { "namespace": "default", "ownerReferences": [ { - "apiVersion": "hive.stackable.tech/v1alpha1", + "apiVersion": "hive.stackable.tech/v1alpha2", "controller": true, "kind": "HiveCluster", "name": "simple-hive", diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index 3c6cf9fe..d38c076c 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -8,7 +8,7 @@ use crate::{ RoleGroupName, ValidatedCluster, build::{object_meta, recommended_labels_for_role_group_resources, role_group_selector}, }, - crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole, METRICS_PORT, METRICS_PORT_NAME}, + crd::{HiveRole, METRICS_PORT, METRICS_PORT_NAME}, }; /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup @@ -34,7 +34,7 @@ pub fn build_rolegroup_headless_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), // Expecting same ports as on listener service, just as a headless, internal service - ports: Some(service_ports()), + ports: Some(service_ports(cluster)), selector: Some(role_group_selector(cluster, hive_role, role_group_name).into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() @@ -88,13 +88,18 @@ fn metrics_ports() -> Vec { }] } -fn service_ports() -> Vec { - vec![ServicePort { - name: Some(HIVE_PORT_NAME.to_string()), - port: HIVE_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] +fn service_ports(cluster: &ValidatedCluster) -> Vec { + cluster + .cluster_config + .api_ports() + .into_iter() + .map(|(name, port)| ServicePort { + name: Some(name.to_string()), + port: port.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }) + .collect() } #[cfg(test)] @@ -145,7 +150,7 @@ mod tests { "namespace": "default", "ownerReferences": [ { - "apiVersion": "hive.stackable.tech/v1alpha1", + "apiVersion": "hive.stackable.tech/v1alpha2", "controller": true, "kind": "HiveCluster", "name": "simple-hive", diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 53d09f2f..03310577 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -67,10 +67,11 @@ use crate::{ }, }, crd::{ - Container, HIVE_PORT, HIVE_PORT_NAME, HiveRole, METRICS_PORT, METRICS_PORT_NAME, - STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, - STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, - STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR_NAME, + Container, HIVE_PORT_NAME, HiveRole, ICEBERG_REST_CATALOG_PORT_NAME, METRICS_PORT, + METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, + STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, + STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, + STACKABLE_LOG_DIR_NAME, }, }; @@ -277,6 +278,26 @@ pub(crate) fn build_metastore_rolegroup_statefulset( "} }; + // If the Iceberg REST catalog is enabled, we probe it rather than Thrift: + // - HMS binds the Thrift socket before it starts the REST catalog, so an open REST port also + // means that Thrift is listening (or bound to the loopback interface if disabled, see + // `hive_site.rs`, where the kubelet can not reach it anyway). + // - If the REST catalog fails to initialize (e.g. its authentication), HMS logs an error and + // keeps running with Thrift only. Probing Thrift would report such a Pod as ready. + // + // The REST catalog embedded in HMS has no health endpoint (only the standalone REST catalog + // server has Spring Boot actuator endpoints). Every request to `/iceberg/*` without a bearer + // token is rejected with a 401 when authentication is enabled, which an HTTP probe treats as a + // failure, so we use a TCP probe. The port is only opened if the catalog is served. + let probe_port_name = if cluster.cluster_config.iceberg_rest_catalog_enabled { + ICEBERG_REST_CATALOG_PORT_NAME + } else { + HIVE_PORT_NAME + }; + for (port_name, port) in cluster.cluster_config.api_ports() { + container_builder.add_container_port(port_name, port.into()); + } + let container_builder = container_builder .image_from_product_image(resolved_product_image) .command(vec![ @@ -324,7 +345,6 @@ pub(crate) fn build_metastore_rolegroup_statefulset( STACKABLE_LOG_CONFIG_MOUNT_DIR, ) .expect("The mount paths are statically defined and there should be no duplicates.") - .add_container_port(HIVE_PORT_NAME, HIVE_PORT.into()) .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) .resources(merged_config.resources.clone().into()) .readiness_probe(Probe { @@ -332,7 +352,7 @@ pub(crate) fn build_metastore_rolegroup_statefulset( period_seconds: Some(10), failure_threshold: Some(5), tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(HIVE_PORT_NAME.to_string()), + port: IntOrString::String(probe_port_name.to_string()), ..TCPSocketAction::default() }), ..Probe::default() @@ -341,7 +361,7 @@ pub(crate) fn build_metastore_rolegroup_statefulset( initial_delay_seconds: Some(30), period_seconds: Some(10), tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(HIVE_PORT_NAME.to_string()), + port: IntOrString::String(probe_port_name.to_string()), ..TCPSocketAction::default() }), ..Probe::default() @@ -447,7 +467,7 @@ pub(crate) fn build_metastore_rolegroup_statefulset( add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; - if cluster.has_kerberos_enabled() { + if cluster.has_thrift_kerberos_enabled() { add_kerberos_pod_config(cluster, hive_role, container_builder, &mut pod_builder) .context(AddKerberosConfigSnafu)?; } @@ -538,7 +558,7 @@ mod tests { /// [`DERBY_YAML`] with the operator-set `FILE_LOG_DIRECTORY` overridden on the /// metastore. const OVERRIDE_DERBY_YAML: &str = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -550,6 +570,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: envOverrides: FILE_LOG_DIRECTORY: /debug-override @@ -560,7 +581,7 @@ mod tests { /// [`DERBY_YAML`] with Kerberos enabled. const KERBEROS_DERBY_YAML: &str = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -570,11 +591,13 @@ mod tests { image: productVersion: "4.0.0" clusterConfig: - authentication: - kerberos: - secretClass: kerberos + thriftApi: + authentication: + kerberos: + secretClass: kerberos metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: default: @@ -583,7 +606,7 @@ mod tests { /// [`KERBEROS_DERBY_YAML`] with the operator-set `KRB5_CONFIG` overridden on the metastore. const KERBEROS_OVERRIDE_DERBY_YAML: &str = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -593,11 +616,13 @@ mod tests { image: productVersion: "4.0.0" clusterConfig: - authentication: - kerberos: - secretClass: kerberos + thriftApi: + authentication: + kerberos: + secretClass: kerberos metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: envOverrides: KRB5_CONFIG: /custom/krb5.conf diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index f5b37d34..de596b95 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -16,7 +16,7 @@ use crate::{ controller::build::resource::{ discovery::discovery_config_map_name, listener::role_listener_name, }, - crd::{HiveRole, v1alpha1}, + crd::{HiveRole, v1alpha2}, }; #[derive(Snafu, Debug)] @@ -91,7 +91,7 @@ pub struct ResolvedOpaConfig { impl ResolvedOpaConfig { pub async fn from_opa_config( client: &Client, - hive: &v1alpha1::HiveCluster, + hive: &v1alpha2::HiveCluster, opa_config: &OpaConfig, ) -> Result { // See: @@ -120,7 +120,7 @@ impl ResolvedOpaConfig { pub async fn dereference( client: &stackable_operator::client::Client, - hive: &v1alpha1::HiveCluster, + hive: &v1alpha2::HiveCluster, ) -> Result { let s3_connection_spec: Option = if let Some(s3) = &hive.spec.cluster_config.s3 { diff --git a/rust/operator-binary/src/controller/update_status.rs b/rust/operator-binary/src/controller/update_status.rs index 9e42c9da..a709555e 100644 --- a/rust/operator-binary/src/controller/update_status.rs +++ b/rust/operator-binary/src/controller/update_status.rs @@ -18,7 +18,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ HIVE_OPERATOR_NAME, controller::{Applied, KubernetesResources}, - crd::{HiveClusterStatus, v1alpha1}, + crd::{HiveClusterStatus, v1alpha2}, }; #[derive(Snafu, Debug, EnumDiscriminants)] @@ -38,11 +38,11 @@ pub enum Error { type Result = std::result::Result; /// Computes the cluster status from the applied resources and patches it onto the -/// [`v1alpha1::HiveCluster`]. Takes [`KubernetesResources`] so the type system proves +/// [`v1alpha2::HiveCluster`]. Takes [`KubernetesResources`] so the type system proves /// the status derives from applied resources, not merely built ones. pub async fn update_status( client: &Client, - hive: &v1alpha1::HiveCluster, + hive: &v1alpha2::HiveCluster, applied: &KubernetesResources, ) -> Result<()> { let mut ss_cond_builder = StatefulSetConditionBuilder::default(); @@ -62,6 +62,8 @@ pub async fn update_status( conditions: compute_conditions(hive, &[&ss_cond_builder, &cluster_operation_cond_builder]), }; + // Deliberately not a HiveClusterStatusWithChangedValues: The changedValues are owned by the + // conversion webhook. Server-side apply only touches the fields we send, so they are retained. client .apply_patch_status(HIVE_OPERATOR_NAME, hive, &status) .await diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 3ac69722..bee11b44 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, str::FromStr}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ commons::product_image_selection, config::fragment, @@ -23,9 +23,9 @@ use crate::{ ValidatedMetaStoreConfig, ValidatedRoleConfig, dereference::DereferencedObjects, }, crd::{ - HiveRole, MetaStoreConfig, + HiveRole, databases::{MetadataDatabaseConnection, derby_driver_class}, - v1alpha1::{self, HiveMetastoreRoleConfig}, + v1alpha2::{self, HiveMetastoreRoleConfig, MetaStoreConfig}, }, }; @@ -79,6 +79,23 @@ pub enum Error { "the Vector aggregator discovery ConfigMap name is required when the Vector agent is enabled" ))] MissingVectorAggregatorConfigMapName, + + #[snafu(display( + "the Thrift API and Iceberg REST catalog are both disabled, please enable {}", + if *iceberg_rest_catalog_supported { + "at least one" + } else { + "the Thrift API (the Iceberg REST catalog requires Hive 4.2.0 or newer)" + } + ))] + ThriftAndRestDisabled { + iceberg_rest_catalog_supported: bool, + }, + + #[snafu(display( + "the Iceberg REST catalog requires Hive 4.2.0 or newer, but {product_version} is used" + ))] + IcebergRestCatalogUnsupportedProductVersion { product_version: String }, } /// Validated logging configuration for the Hive metastore and (optional) Vector container. @@ -123,7 +140,7 @@ fn validate_logging( } pub fn validate_cluster( - hive: &v1alpha1::HiveCluster, + hive: &v1alpha2::HiveCluster, image_repository: &str, dereferenced_objects: DereferencedObjects, ) -> Result { @@ -208,8 +225,32 @@ pub fn validate_cluster( .to_owned(); let hdfs = hive.spec.cluster_config.hdfs.clone(); + let thrift_api_enabled = hive.spec.cluster_config.thrift_api.enabled; + let iceberg_rest_catalog_enabled = hive.spec.cluster_config.iceberg_rest_catalog.enabled; + let thrift_kerberos_secret_class = hive.thrift_kerberos_secret_class(); + let iceberg_rest_catalog_security = hive + .spec + .cluster_config + .iceberg_rest_catalog + .authentication + .clone(); + + // The Iceberg REST catalog is available since Hive 4.1.0, but only works since 4.2.0. + let product_version = &image.product_version; + let iceberg_rest_catalog_supported = !(product_version.starts_with("3.") + || product_version.starts_with("4.0.") + || product_version.starts_with("4.1.")); - let kerberos_secret_class = hive.kerberos_secret_class(); + ensure!( + thrift_api_enabled || iceberg_rest_catalog_enabled, + ThriftAndRestDisabledSnafu { + iceberg_rest_catalog_supported + } + ); + ensure!( + !iceberg_rest_catalog_enabled || iceberg_rest_catalog_supported, + IcebergRestCatalogUnsupportedProductVersionSnafu { product_version } + ); Ok(ValidatedCluster::new( name, @@ -224,7 +265,12 @@ pub fn validate_cluster( hdfs, s3_connection_spec: dereferenced_objects.s3_connection_spec, hive_opa_config: dereferenced_objects.hive_opa_config, - kerberos_secret_class, + warehouse_dir: hive.spec.cluster_config.warehouse_dir.clone(), + warehouse_external_dir: hive.spec.cluster_config.warehouse_external_dir.clone(), + thrift_api_enabled, + iceberg_rest_catalog_enabled, + thrift_kerberos_secret_class, + iceberg_rest_catalog_security, }, role_group_configs, dereferenced_objects.role_listener, @@ -237,15 +283,15 @@ fn validate_role_group_config( role_group_name: &RoleGroupName, role_group: &crate::crd::HiveRoleGroupType, role: &crate::crd::HiveRoleType, - default_config: &crate::crd::MetaStoreConfigFragment, + default_config: &crate::crd::v1alpha2::MetaStoreConfigFragment, vector_aggregator_config_map_name: &Option, ) -> Result { let merged = with_validated_config::< MetaStoreConfig, JavaCommonConfig, - crate::crd::MetaStoreConfigFragment, + crate::crd::v1alpha2::MetaStoreConfigFragment, HiveMetastoreRoleConfig, - v1alpha1::HiveConfigOverrides, + v1alpha2::HiveConfigOverrides, >(role_group, role, default_config) .with_context(|_| ValidateConfigSnafu { role_group: role_group_name.clone(), @@ -354,7 +400,7 @@ mod tests { assert_eq!(cluster_config.hdfs, None); assert_eq!(cluster_config.s3_connection_spec, None); assert!(cluster_config.hive_opa_config.is_none()); - assert_eq!(cluster_config.kerberos_secret_class, None); + assert_eq!(cluster_config.thrift_kerberos_secret_class, None); // A single metastore role with the single `default` role group. assert_eq!(cluster.role_group_configs.len(), 1); @@ -373,7 +419,7 @@ mod tests { // A copy of `DERBY_YAML` whose role-group name violates the RFC 1123 label rules // (uppercase and underscore). let yaml = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -385,6 +431,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ metastore: roleGroups: Invalid_RG: diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 5ce569cb..48de415c 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -61,7 +61,7 @@ mod tests { #[case(HiveRole::MetaStore)] fn test_affinity_defaults(#[case] role: HiveRole) { let input = r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: simple-hive @@ -73,6 +73,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/warehouse/ authorization: opa: configMapName: simple-opa diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 04cf2411..b8e45fcb 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::{ops::Deref, str::FromStr}; +use std::{iter, ops::Deref, str::FromStr}; use databases::MetadataDatabaseConnection; /// Re-export of the shared product-logging spec data types (test-only). @@ -6,7 +6,7 @@ use databases::MetadataDatabaseConnection; pub use product_logging::spec::{ ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, CustomContainerLogConfig, }; -use security::AuthenticationConfig; +use security::ThriftAuthenticationConfig; use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::{ @@ -32,7 +32,7 @@ use stackable_operator::{ status::condition::{ClusterCondition, HasStatusCondition}, v2::{ config_overrides::KeyValueConfigOverrides, - role_utils::{JavaCommonConfig, Role, RoleGroup}, + role_utils::{CommonConfiguration, JavaCommonConfig, Role, RoleGroup}, types::{ common::Port, kubernetes::{ @@ -41,12 +41,11 @@ use stackable_operator::{ operator::RoleName, }, }, - versioned::versioned, + versioned::{TrackingFrom, jthong_path, versioned}, }; use strum::{Display, EnumIter}; -use v1alpha1::HiveMetastoreRoleConfig; -use crate::crd::affinity::get_affinity; +use crate::crd::{affinity::get_affinity, security::IcebergRestCatalogAuthenticationConfig}; pub mod affinity; pub mod databases; @@ -69,6 +68,8 @@ constant!(pub STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME: VolumeName = "log-config-moun // Default ports pub const HIVE_PORT_NAME: &str = "hive"; pub const HIVE_PORT: Port = Port(9083); +pub const ICEBERG_REST_CATALOG_PORT_NAME: &str = "iceberg-rest"; +pub const ICEBERG_REST_CATALOG_PORT: Port = Port(9001); pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: Port = Port(9084); @@ -90,24 +91,27 @@ const DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_min constant!(METASTORE_ROLE_NAME: RoleName = "metastore"); pub type HiveRoleType = Role< - MetaStoreConfigFragment, - v1alpha1::HiveConfigOverrides, - v1alpha1::HiveMetastoreRoleConfig, + v1alpha2::MetaStoreConfigFragment, + v1alpha2::HiveConfigOverrides, + v1alpha2::HiveMetastoreRoleConfig, JavaCommonConfig, >; pub type HiveRoleGroupType = - RoleGroup; + RoleGroup; #[versioned( version(name = "v1alpha1"), + version(name = "v1alpha2"), + options(k8s(experimental_conversion_tracking)), crates( kube_core = "stackable_operator::kube::core", kube_client = "stackable_operator::kube::client", k8s_openapi = "stackable_operator::k8s_openapi", schemars = "stackable_operator::schemars", versioned = "stackable_operator::versioned" - ) + ), + skip(from) // TODO: Can we only skip between v1alpha1 and v1alpha2 to be future-proof? )] pub mod versioned { /// A Hive cluster stacklet. This resource is managed by the Stackable operator for Apache Hive. @@ -126,7 +130,8 @@ pub mod versioned { pub struct HiveClusterSpec { /// Hive metastore settings that affect all roles and role groups. /// The settings in the `clusterConfig` are cluster wide settings that do not need to be configurable at role or role group level. - pub cluster_config: v1alpha1::HiveClusterConfig, + #[versioned(nested)] + pub cluster_config: HiveClusterConfig, // no doc - docs in ClusterOperation struct. #[serde(default)] @@ -140,7 +145,12 @@ pub mod versioned { pub image: ProductImage, // no doc - docs in Role struct. - pub metastore: HiveRoleType, + pub metastore: Role< + MetaStoreConfigFragment, + HiveConfigOverrides, + HiveMetastoreRoleConfig, + JavaCommonConfig, + >, } // TODO: move generic version to op-rs? @@ -158,8 +168,24 @@ pub mod versioned { #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HiveClusterConfig { + // // FIXME: This should be a #[versioned(removed)], but that's not supported yet + // #[versioned(deprecated( + // in = "v1alpha2", + // previous_docs = "Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/hive/usage-guide/security)." + // ))] + // /// This field is deprecated, please don't use it! This field will always be converted to `null`. + // /// + // /// Configure the Thrift API authentication on `.spec.clusterConfig.thriftApi.authentication` + // /// instead. + // pub deprecated_authentication: Option, + // + // + // /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/hive/usage-guide/security). - pub authentication: Option, + /// + /// Moved to `.spec.clusterConfig.thriftApi.authentication` in v1alpha2. + #[versioned(removed(in = "v1alpha2"))] + pub authentication: Option, /// Authorization options for Hive. /// Learn more in the [Hive authorization usage guide](DOCS_BASE_URL_PLACEHOLDER/hive/usage-guide/security#authorization). @@ -169,6 +195,16 @@ pub mod versioned { /// as schema and tables. pub metadata_database: MetadataDatabaseConnection, + /// Start the Hive metastore Thrift API (enabled by default). + #[versioned(added(in = "v1alpha2"))] + #[serde(default)] + pub thrift_api: ThriftApiConfig, + + /// Experimental support for starting an Iceberg REST catalog server (disabled by default). + #[versioned(added(in = "v1alpha2"))] + #[serde(default)] + pub iceberg_rest_catalog: IcebergRestCatalogConfig, + /// HDFS connection specification. #[serde(default, skip_serializing_if = "Option::is_none")] pub hdfs: Option, @@ -178,6 +214,19 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub s3: Option, + /// The location of default database for the Hive warehouse (e.g. `s3a://my-bucket/my-warehouse`). + /// + /// Maps to the `hive.metastore.warehouse.dir` setting. + #[versioned(added(in = "v1alpha2"))] + pub warehouse_dir: String, + + /// Optional external location of Hive warehouse (e.g. `s3a://my-bucket/my-warehouse/external`). + /// + /// Maps to the `hive.metastore.warehouse.external.dir` setting. If not specified the operator + /// will *not* pass the setting to the Hive metastore and will let it default internally. + #[versioned(added(in = "v1alpha2"))] + pub warehouse_external_dir: Option, + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) @@ -186,6 +235,72 @@ pub mod versioned { pub vector_aggregator_config_map_name: Option, } + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct ThriftApiConfig { + /// Whether the Thrift API is exposed (defaults to `true`). + /// + /// Hive can not switch the Thrift server off, so if disabled it is bound to the loopback + /// interface and not exposed via the Service and Listener. + #[serde(default = "default_thrift_api_enabled")] + enabled: bool, + + /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/hive/usage-guide/security). + /// + /// Only Kerberos is supported. + pub authentication: Option, + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IcebergRestCatalogConfig { + /// Whether the Iceberg REST catalog is started (defaults to `false`). + /// Requires Hive 4.2.0 or newer. + #[serde(default = "default_iceberg_rest_catalog_enabled")] + enabled: bool, + + /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/hive/usage-guide/iceberg-rest-catalog#authentication). + /// + /// If not specified, the Iceberg REST catalog does not authenticate requests at all. + pub authentication: Option, + } + + #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] + #[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") + )] + pub struct MetaStoreConfig { + /// The location of default database for the Hive warehouse. + /// Maps to the `hive.metastore.warehouse.dir` setting. + /// + /// Moved to `.spec.clusterConfig.warehouseDir` in v1alpha2. + #[versioned(removed(in = "v1alpha2"))] + pub warehouse_dir: Option, + + #[fragment_attrs(serde(default))] + pub resources: Resources, + + #[fragment_attrs(serde(default))] + pub logging: Logging, + + #[fragment_attrs(serde(default))] + pub affinity: StackableAffinity, + + /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + #[fragment_attrs(serde(default))] + pub graceful_shutdown_timeout: Option, + } + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HiveConfigOverrides { @@ -197,34 +312,351 @@ pub mod versioned { } } -impl Default for v1alpha1::HiveMetastoreRoleConfig { +fn default_thrift_api_enabled() -> bool { + true +} + +fn default_iceberg_rest_catalog_enabled() -> bool { + false +} + +impl Default for v1alpha2::HiveMetastoreRoleConfig { fn default() -> Self { - v1alpha1::HiveMetastoreRoleConfig { + v1alpha2::HiveMetastoreRoleConfig { listener_class: metastore_default_listener_class(), common: Default::default(), } } } -impl HasStatusCondition for v1alpha1::HiveCluster { +impl Default for v1alpha2::ThriftApiConfig { + fn default() -> Self { + Self { + enabled: default_thrift_api_enabled(), + authentication: None, + } + } +} + +impl Default for v1alpha2::IcebergRestCatalogConfig { + fn default() -> Self { + Self { + enabled: default_iceberg_rest_catalog_enabled(), + authentication: None, + } + } +} + +impl Default for v1alpha1::HiveMetastoreRoleConfig { + fn default() -> Self { + Self { + common: Default::default(), + listener_class: metastore_default_listener_class(), + } + } +} + +impl TrackingFrom + for v1alpha2::HiveClusterSpec +{ + // Note: I let stackable-versioned generate the boilerplate and adopted it to our needs. + // Ideally stackable-versioned eventually supports all changes we need, in which case this can + // be generated by the macro. + fn tracking_from( + old: v1alpha1::HiveClusterSpec, + status: &mut HiveClusterStatusWithChangedValues, + parent: &str, + ) -> Self { + use stackable_operator::versioned::{ExtractIfExt as _, TrackingStatus as _}; + + // We need to model the paths exactly the same way `stackable-versioned` does, so that we + // we can switch to macro magic later. + let __sv_cluster_config_thrift_api_path = jthong_path(parent, "cluster_config.thrift_api"); + let __sv_cluster_config_iceberg_rest_catalog_path = + jthong_path(parent, "cluster_config.iceberg_rest_catalog"); + let __sv_cluster_config_warehouse_dir_path = + jthong_path(parent, "cluster_config.warehouse_dir"); + let __sv_cluster_config_warehouse_external_dir_path = + jthong_path(parent, "cluster_config.warehouse_external_dir"); + + // v1alpha1 was broken in the sense, that users could configure warehouse dirs on a + // roleGroup level. This is bad, as this is a cluster-wide setting. So we try our best to + // migrate it correctly. + let old_role_warehouse_dir = &old.metastore.config.config.warehouse_dir; + let old_role_groups_warehouse_dirs = old + .metastore + .role_groups + .values() + .map(|rg| &rg.config.config.warehouse_dir); + // Yes, this logic is broken, but is impossible to do correctly and should be good enough. + let old_warehouse_dir = old_role_groups_warehouse_dirs + .chain(iter::once(old_role_warehouse_dir)) + .flatten() + .next(); + // Fill with the old (shitty) default + let old_warehouse_dir = old_warehouse_dir + .map(String::as_str) + .unwrap_or_else(|| "/stackable/warehouse"); + + let mut cluster_config = v1alpha2::HiveClusterConfig { + // // We old keep the deprecated property, as stackable-versioned doesn't support removing + // // yet. We restore the value from the new location, so the deprecated value doesn't + // // matter. We set it to null to make this clear. + // #[expect(deprecated)] + // deprecated_authentication: None, + authorization: old.cluster_config.authorization, + metadata_database: old.cluster_config.metadata_database, + thrift_api: v1alpha2::ThriftApiConfig { + enabled: true, + authentication: old.cluster_config.authentication, + }, + iceberg_rest_catalog: v1alpha2::IcebergRestCatalogConfig { + enabled: false, + authentication: None, + }, + hdfs: old.cluster_config.hdfs, + s3: old.cluster_config.s3, + warehouse_dir: old_warehouse_dir.to_owned(), + warehouse_external_dir: None, + vector_aggregator_config_map_name: old.cluster_config.vector_aggregator_config_map_name, + }; + + #[expect(clippy::type_complexity)] + let upgrade_common_config: fn( + CommonConfiguration< + v1alpha1::MetaStoreConfigFragment, + JavaCommonConfig, + v1alpha1::HiveConfigOverrides, + >, + ) -> CommonConfiguration< + v1alpha2::MetaStoreConfigFragment, + JavaCommonConfig, + v1alpha2::HiveConfigOverrides, + > = |old| CommonConfiguration { + config: v1alpha2::MetaStoreConfigFragment { + resources: old.config.resources, + logging: old.config.logging, + affinity: old.config.affinity, + graceful_shutdown_timeout: old.config.graceful_shutdown_timeout, + }, + config_overrides: v1alpha2::HiveConfigOverrides { + hive_site_xml: old.config_overrides.hive_site_xml, + security_properties: old.config_overrides.security_properties, + }, + env_overrides: old.env_overrides, + cli_overrides: old.cli_overrides, + pod_overrides: old.pod_overrides, + product_specific_common_config: old.product_specific_common_config, + }; + + let metastore = Role { + config: upgrade_common_config(old.metastore.config), + role_config: v1alpha2::HiveMetastoreRoleConfig { + common: old.metastore.role_config.common, + listener_class: old.metastore.role_config.listener_class, + }, + role_groups: old + .metastore + .role_groups + .into_iter() + .map(|(rg_name, rg)| { + ( + rg_name, + RoleGroup { + config: upgrade_common_config(rg.config), + replicas: rg.replicas, + }, + ) + }) + .collect(), + }; + + if let Some(upgrades) = status.changes().upgrades.get_mut("v1alpha2") { + if let Some(stackable_operator::versioned::ChangedValue { value, .. }) = + upgrades.extract_first_if(|u| u.json_path == __sv_cluster_config_thrift_api_path) + { + cluster_config.thrift_api = ::serde_json::from_value(value).expect("internal error: {__sv_cluster_config_thrift_api_path} must be serializable for spec.thrift_api"); + } + if let Some(stackable_operator::versioned::ChangedValue { value, .. }) = upgrades + .extract_first_if(|u| u.json_path == __sv_cluster_config_iceberg_rest_catalog_path) + { + cluster_config.iceberg_rest_catalog = ::serde_json::from_value(value).expect("internal error: {__sv_cluster_config_iceberg_rest_catalog_path} must be serializable for spec.iceberg_rest_catalog"); + } + if let Some(stackable_operator::versioned::ChangedValue { value, .. }) = + upgrades.extract_first_if(|u| u.json_path == __sv_cluster_config_warehouse_dir_path) + { + cluster_config.warehouse_dir = ::serde_json::from_value(value).expect("internal error: {__sv_cluster_config_warehouse_dir_path} must be serializable for spec.thrift_api"); + } + if let Some(stackable_operator::versioned::ChangedValue { value, .. }) = upgrades + .extract_first_if(|u| { + u.json_path == __sv_cluster_config_warehouse_external_dir_path + }) + { + cluster_config.warehouse_external_dir = ::serde_json::from_value(value).expect("internal error: {__sv_cluster_config_warehouse_external_dir_path} must be serializable for spec.thrift_api"); + } + if upgrades.is_empty() { + status.changes().upgrades.remove("v1alpha2"); + } + } + + Self { + cluster_config, + cluster_operation: old.cluster_operation, + object_overrides: old.object_overrides, + image: old.image, + metastore, + } + } +} + +impl TrackingFrom + for v1alpha1::HiveClusterSpec +{ + fn tracking_from( + new: v1alpha2::HiveClusterSpec, + status: &mut HiveClusterStatusWithChangedValues, + parent: &str, + ) -> Self { + use stackable_operator::versioned::TrackingStatus as _; + + // We need to model the paths exactly the same way `stackable-versioned` does, so that we + // we can switch to macro magic later. + let __sv_cluster_config_thrift_api_path = jthong_path(parent, "cluster_config.thrift_api"); + let __sv_cluster_config_iceberg_rest_catalog_path = + jthong_path(parent, "cluster_config.iceberg_rest_catalog"); + let __sv_cluster_config_warehouse_dir_path = + jthong_path(parent, "cluster_config.warehouse_dir"); + let __sv_cluster_config_warehouse_external_dir_path = + jthong_path(parent, "cluster_config.warehouse_external_dir"); + + let upgrades = status + .changes() + .upgrades + .entry("v1alpha2".to_owned()) + .or_default(); + upgrades.push(stackable_operator::versioned::ChangedValue { + json_path: __sv_cluster_config_thrift_api_path, + value: ::serde_json::to_value(&new.cluster_config.thrift_api).unwrap(), + downgraded_value: ::core::option::Option::None, + }); + upgrades.push(stackable_operator::versioned::ChangedValue { + json_path: __sv_cluster_config_iceberg_rest_catalog_path, + value: ::serde_json::to_value(&new.cluster_config.iceberg_rest_catalog).unwrap(), + downgraded_value: ::core::option::Option::None, + }); + upgrades.push(stackable_operator::versioned::ChangedValue { + json_path: __sv_cluster_config_warehouse_dir_path, + value: ::serde_json::to_value(&new.cluster_config.warehouse_dir).unwrap(), + downgraded_value: ::core::option::Option::None, + }); + upgrades.push(stackable_operator::versioned::ChangedValue { + json_path: __sv_cluster_config_warehouse_external_dir_path, + value: ::serde_json::to_value(&new.cluster_config.warehouse_external_dir).unwrap(), + downgraded_value: ::core::option::Option::None, + }); + + #[expect(clippy::type_complexity)] + let downgrade_common_config: fn( + CommonConfiguration< + v1alpha2::MetaStoreConfigFragment, + JavaCommonConfig, + v1alpha2::HiveConfigOverrides, + >, + ) -> CommonConfiguration< + v1alpha1::MetaStoreConfigFragment, + JavaCommonConfig, + v1alpha1::HiveConfigOverrides, + > = |old| CommonConfiguration { + config: v1alpha1::MetaStoreConfigFragment { + resources: old.config.resources, + logging: old.config.logging, + affinity: old.config.affinity, + graceful_shutdown_timeout: old.config.graceful_shutdown_timeout, + // We only set the warehouse dir to the role config below explicitly (and not on the + // rolegroup ones). + warehouse_dir: None, + }, + config_overrides: v1alpha1::HiveConfigOverrides { + hive_site_xml: old.config_overrides.hive_site_xml, + security_properties: old.config_overrides.security_properties, + }, + env_overrides: old.env_overrides, + cli_overrides: old.cli_overrides, + pod_overrides: old.pod_overrides, + product_specific_common_config: old.product_specific_common_config, + }; + + let mut metastore = Role { + config: downgrade_common_config(new.metastore.config), + role_config: v1alpha1::HiveMetastoreRoleConfig { + common: new.metastore.role_config.common, + listener_class: new.metastore.role_config.listener_class, + }, + role_groups: new + .metastore + .role_groups + .into_iter() + .map(|(rg_name, rg)| { + ( + rg_name, + RoleGroup { + config: downgrade_common_config(rg.config), + replicas: rg.replicas, + }, + ) + }) + .collect(), + }; + + // The warehouseDir migration is a bit more complicated (see the upgrade path), so we only + // restore the new warehouse path to the role-level config, and don't set anything on the + // roleGroups. + // + // However, to pass the roundtrip tests we only restore a value in case it's not the old + // default of "/stackable/warehouse" we used, so that a roundtrip starting with `null` end + // up with `null` as well. + if new.cluster_config.warehouse_dir != "/stackable/warehouse" { + metastore.config.config.warehouse_dir = Some(new.cluster_config.warehouse_dir); + } + + let cluster_config = v1alpha1::HiveClusterConfig { + authentication: new.cluster_config.thrift_api.authentication, + authorization: new.cluster_config.authorization, + metadata_database: new.cluster_config.metadata_database, + hdfs: new.cluster_config.hdfs, + s3: new.cluster_config.s3, + vector_aggregator_config_map_name: new.cluster_config.vector_aggregator_config_map_name, + }; + Self { + cluster_config, + cluster_operation: new.cluster_operation, + object_overrides: new.object_overrides, + image: new.image, + metastore, + } + } +} + +impl HasStatusCondition for v1alpha2::HiveCluster { fn conditions(&self) -> Vec { match &self.status { - Some(status) => status.conditions.clone(), + Some(status) => status.status.conditions.clone(), None => vec![], } } } -impl v1alpha1::HiveCluster { - pub fn role_config(&self, role: &HiveRole) -> &HiveMetastoreRoleConfig { +impl v1alpha2::HiveCluster { + pub fn role_config(&self, role: &HiveRole) -> &v1alpha2::HiveMetastoreRoleConfig { match role { HiveRole::MetaStore => &self.spec.metastore.role_config, } } - pub fn kerberos_secret_class(&self) -> Option { + pub fn thrift_kerberos_secret_class(&self) -> Option { self.spec .cluster_config + .thrift_api .authentication .as_ref() .map(|a| &a.kerberos) @@ -240,6 +672,37 @@ impl v1alpha1::HiveCluster { } } +impl v1alpha2::MetaStoreConfig { + pub(crate) fn default_config( + cluster_name: &str, + role: &HiveRole, + opa_config: Option<&OpaConfig>, + ) -> v1alpha2::MetaStoreConfigFragment { + v1alpha2::MetaStoreConfigFragment { + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("250m".to_owned())), + max: Some(Quantity("1000m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("768Mi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: MetastoreStorageConfigFragment { + data: PvcConfigFragment { + capacity: Some(Quantity("0Mi".to_owned())), // "0Mi" is a marker for us, so we don't warn unnecessarily + storage_class: None, + selectors: None, + }, + }, + }, + logging: product_logging::spec::default_logging(), + affinity: get_affinity(cluster_name, role, opa_config), + graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT), + } + } +} + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HdfsConnection { @@ -329,71 +792,6 @@ pub struct MetastoreStorageConfig { pub data: PvcConfig, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct MetaStoreConfig { - /// The location of default database for the Hive warehouse. - /// Maps to the `hive.metastore.warehouse.dir` setting. - pub warehouse_dir: Option, - - #[fragment_attrs(serde(default))] - pub resources: Resources, - - #[fragment_attrs(serde(default))] - pub logging: Logging, - - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - - /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, -} - -impl MetaStoreConfig { - pub(crate) fn default_config( - cluster_name: &str, - role: &HiveRole, - opa_config: Option<&OpaConfig>, - ) -> MetaStoreConfigFragment { - MetaStoreConfigFragment { - warehouse_dir: None, - resources: ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("250m".to_owned())), - max: Some(Quantity("1000m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("768Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: MetastoreStorageConfigFragment { - data: PvcConfigFragment { - capacity: Some(Quantity("0Mi".to_owned())), // "0Mi" is a marker for us, so we don't warn unnecessarily - storage_class: None, - selectors: None, - }, - }, - }, - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, opa_config), - graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT), - } - } -} - #[derive(Clone, Default, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HiveClusterStatus { @@ -406,10 +804,11 @@ pub struct HiveClusterStatus { #[cfg(test)] mod tests { - use stackable_operator::versioned::test_utils::RoundtripTestData; + use stackable_operator::versioned::{TrackingInto, test_utils::RoundtripTestData}; use strum::IntoEnumIterator; use super::*; + use crate::crd::v1alpha2::{IcebergRestCatalogConfig, ThriftApiConfig}; #[test] fn test_constants() { @@ -527,4 +926,111 @@ mod tests { .expect("Failed to parse HiveClusterSpec YAML") } } + + impl RoundtripTestData for v1alpha2::HiveClusterSpec { + fn roundtrip_test_data() -> Vec { + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" + - image: + productVersion: 1.2.3 + clusterConfig: + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/warehouse/ + metastore: + roleGroups: + default: + replicas: 1 + - image: + productVersion: 1.2.3 + clusterConfig: + thriftApi: + enabled: true + authentication: + kerberos: + secretClass: my-kerberos + icebergRestCatalog: + enabled: false + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/warehouse/ + metastore: + roleGroups: + default: + replicas: 1 + - image: + productVersion: 1.2.3 + clusterConfig: + thriftApi: + enabled: true + authentication: + kerberos: + secretClass: my-kerberos + icebergRestCatalog: + enabled: true # defaults to false + authentication: + oAuth2: + issuerUrl: https://keycloak.my-namespace.svc.cluster.local:8443/realms/test + audience: hive-metastore + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/warehouse/ + metastore: + roleGroups: + default: + replicas: 1 + "#}) + .expect("Failed to parse HiveClusterSpec YAML") + } + } + + #[test] + fn cluster_config_v1_to_v2_keeps_thrift_authentication() { + let mut status = HiveClusterStatusWithChangedValues::default(); + let v1: v1alpha1::HiveClusterSpec = + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" + image: + productVersion: 1.2.3 + clusterConfig: + authentication: + kerberos: + secretClass: my-kerberos + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: hive-credentials + metastore: + roleGroups: + default: + replicas: 1 + "#}) + .expect("Failed to parse HiveClusterSpec YAML"); + + let v2: v1alpha2::HiveClusterSpec = v1.clone().tracking_into(&mut status, "$"); + // assert_eq!(v2.cluster_config.deprecated_authentication, None); + assert_eq!( + v2.cluster_config.thrift_api, + ThriftApiConfig { + enabled: true, + // Assert that the old authentication (Kerberos) is still active + authentication: v1.cluster_config.authentication + } + ); + assert_eq!( + v2.cluster_config.iceberg_rest_catalog, + IcebergRestCatalogConfig { + enabled: false, + authentication: None + } + ); + } } diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 3bb851cb..bae3eb43 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -4,10 +4,11 @@ use stackable_operator::{ schemars::{self, JsonSchema}, v2::types::kubernetes::SecretClassName, }; +use url::Url; #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AuthenticationConfig { +pub struct ThriftAuthenticationConfig { /// Kerberos configuration. pub kerberos: KerberosConfig, } @@ -23,6 +24,56 @@ pub struct AuthorizationConfig { #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KerberosConfig { - /// Name of the SecretClass providing the keytab for the HBase services. + /// Name of the SecretClass providing the keytab for the Hive metastore. pub secret_class: SecretClassName, } + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum IcebergRestCatalogAuthenticationConfig { + /// Hive Metastore assumes that the value of the `x-actor-username` HTTP header is the + /// authenticated username, without verifying it. You may use it to test authorized access, + /// but this mode is not recommended in a production environment. + Simple {}, + /// OAuth 2 is the industry standard for authenticating Iceberg client usernames. You can + /// integrate Hive Metastore with your Authorization Server, e.g., Keycloak, to protect Iceberg + /// resources. + OAuth2(OAuth2AuthenticationConfig), + /// Configure Hive Metastore so that its Iceberg REST API accepts a JSON Web Token (JWT) as a + /// bearer token in the Authorization header. Hive Metastore derives the username from the `sub` + /// claim of a properly-signed JWT. + /// + /// Only the signature and the expiry of the JWT are checked, not its issuer or audience. + /// Prefer `oAuth2` where possible. + Jwt(JwtAuthenticationConfig), +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OAuth2AuthenticationConfig { + /// URL of the OAuth 2 authorization server (issuer), e.g. a Keycloak realm. + /// + /// The Hive metastore fetches the server metadata from + /// `/.well-known/oauth-authorization-server` on startup, which must include the + /// `jwks_uri` used to validate access tokens. + /// + /// TODO: Should this be a struct instead? + pub issuer_url: Url, + + /// The audience the JWT access tokens must be issued for. + pub audience: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JwtAuthenticationConfig { + /// URLs of the JSON Web Key Set (JWKS) used to validate the signature of the JWTs. + /// + /// Maps to the `metastore.authentication.jwt.jwks.url` setting. Hive only falls back to + /// another URL if fetching the JWKS fails, so all URLs need to serve the same keys. Hive does + /// not preserve the order of the URLs, and the URLs must not contain commas. + /// + /// TODO: Should this be a struct instead? + #[schemars(length(min = 1))] + pub jwks_urls: Vec, +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 5d17acb2..9ad7b727 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -37,7 +37,7 @@ use stackable_operator::{ use crate::{ controller::HIVE_FULL_CONTROLLER_NAME, - crd::{HiveCluster, HiveClusterVersion, v1alpha1}, + crd::{HiveCluster, HiveClusterVersion, v1alpha2}, webhooks::conversion::create_webhook_server, }; @@ -69,7 +69,7 @@ struct Opts { async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { - Command::Crd => HiveCluster::merged_crd(HiveClusterVersion::V1Alpha1)? + Command::Crd => HiveCluster::merged_crd(HiveClusterVersion::V1Alpha2)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?, Command::Run(RunArguments { operator_environment, @@ -112,7 +112,7 @@ async fn main() -> anyhow::Result<()> { let mut readiness_checks = HealthCheckRegistry::new(); let hive_cluster_crd_check = readiness_checks.register(format!( "CRD {crd} established", - crd = v1alpha1::HiveCluster::crd_name() + crd = v1alpha2::HiveCluster::crd_name() )); let webhook_server = create_webhook_server( @@ -136,7 +136,7 @@ async fn main() -> anyhow::Result<()> { )); let hive_controller = Controller::new( - watch_namespace.get_api::>(&client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ); let config_map_store = hive_controller.store(); @@ -226,7 +226,7 @@ async fn main() -> anyhow::Result<()> { .map(anyhow::Ok); let delayed_hive_controller = async { - signal::crd_established(&client, v1alpha1::HiveCluster::crd_name()).await?; + signal::crd_established(&client, v1alpha2::HiveCluster::crd_name()).await?; hive_cluster_crd_check.mark_passed(); hive_controller.await }; @@ -239,7 +239,7 @@ async fn main() -> anyhow::Result<()> { } fn references_config_map( - hive: &DeserializeGuard, + hive: &DeserializeGuard, config_map: &DeserializeGuard, ) -> bool { let Ok(hive) = &hive.0 else { @@ -257,7 +257,7 @@ fn references_config_map( } fn references_s3_connection( - hive: &DeserializeGuard, + hive: &DeserializeGuard, s3_connection: &DeserializeGuard, ) -> bool { let Ok(hive) = &hive.0 else { @@ -283,10 +283,10 @@ mod tests { use super::*; - fn hive_cluster(cluster_config: &str) -> DeserializeGuard { + fn hive_cluster(cluster_config: &str) -> DeserializeGuard { let input = format!( indoc! {r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -297,6 +297,7 @@ mod tests { clusterConfig: metadataDatabase: derby: {{}} + warehouseDir: s3a://my-bucket/warehouse/ {cluster_config} metastore: roleGroups: @@ -367,7 +368,7 @@ mod tests { #[test] fn references_s3_connection_ignores_undeserializable_clusters() { let hive = yaml_serde::from_str(indoc! {r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -405,7 +406,7 @@ mod tests { #[test] fn references_config_map_ignores_undeserializable_clusters() { let hive = yaml_serde::from_str(indoc! {r#" - apiVersion: hive.stackable.tech/v1alpha1 + apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index da289235..d8169397 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -30,7 +30,7 @@ pub async fn create_webhook_server( client: Client, ) -> Result { let crds_and_handlers = vec![( - HiveCluster::merged_crd(HiveClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, + HiveCluster::merged_crd(HiveClusterVersion::V1Alpha2).context(MergeCrdSnafu)?, HiveCluster::try_convert, )]; diff --git a/tests/release.yaml b/tests/release.yaml index 225e77af..87155420 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -20,3 +20,5 @@ releases: operatorVersion: 0.0.0-dev opa: operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 index 25ecb9aa..57198ef4 100644 --- a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive @@ -15,6 +15,7 @@ spec: clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/my-warehouse/ {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 index 37b0476f..c57d1c48 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive diff --git a/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 index ca23843d..1290d42b 100644 --- a/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive diff --git a/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 index ac4b348c..29177a30 100644 --- a/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive diff --git a/tests/templates/kuttl/cluster-operation/50-assert.yaml b/tests/templates/kuttl/cluster-operation/50-assert.yaml index 7bd142e2..b5b138da 100644 --- a/tests/templates/kuttl/cluster-operation/50-assert.yaml +++ b/tests/templates/kuttl/cluster-operation/50-assert.yaml @@ -15,7 +15,7 @@ kind: StatefulSet metadata: name: test-hive-metastore-default ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -28,7 +28,7 @@ kind: ServiceAccount metadata: name: test-hive-serviceaccount ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -38,7 +38,7 @@ kind: RoleBinding metadata: name: test-hive-rolebinding ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -48,7 +48,7 @@ kind: PodDisruptionBudget metadata: name: test-hive-metastore ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -58,7 +58,7 @@ kind: Listener metadata: name: test-hive-metastore ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -68,7 +68,7 @@ kind: ConfigMap metadata: name: test-hive ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -78,7 +78,7 @@ kind: Service metadata: name: test-hive-metastore-default-headless ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive @@ -88,7 +88,7 @@ kind: Service metadata: name: test-hive-metastore-default-metrics ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: test-hive diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml index 842e17e5..6ca9b638 100644 --- a/tests/templates/kuttl/external-access/10-listener-classes.yaml +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - envsubst < listener-classes.yaml | kubectl apply -n "$NAMESPACE" -f - + envsubst < 10_listener-classes.yaml | kubectl apply -n "$NAMESPACE" -f - diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/10_listener-classes.yaml similarity index 100% rename from tests/templates/kuttl/external-access/listener-classes.yaml rename to tests/templates/kuttl/external-access/10_listener-classes.yaml diff --git a/tests/templates/kuttl/external-access/20-install-hive.yaml b/tests/templates/kuttl/external-access/20-install-hive.yaml index 73d580d6..1120d121 100644 --- a/tests/templates/kuttl/external-access/20-install-hive.yaml +++ b/tests/templates/kuttl/external-access/20-install-hive.yaml @@ -4,5 +4,5 @@ kind: TestStep timeout: 600 commands: - script: > - envsubst < install-hive.yaml | + envsubst < 20_hive.yaml | kubectl apply -n "$NAMESPACE" -f - diff --git a/tests/templates/kuttl/external-access/install-hive.yaml.j2 b/tests/templates/kuttl/external-access/20_hive.yaml.j2 similarity index 91% rename from tests/templates/kuttl/external-access/install-hive.yaml.j2 rename to tests/templates/kuttl/external-access/20_hive.yaml.j2 index 2f100403..fe24bd29 100644 --- a/tests/templates/kuttl/external-access/install-hive.yaml.j2 +++ b/tests/templates/kuttl/external-access/20_hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive @@ -15,6 +15,7 @@ spec: clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/my-warehouse/ {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/iceberg-rest-catalog/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/iceberg-rest-catalog/00-limit-range.yaml b/tests/templates/kuttl/iceberg-rest-catalog/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/00-patch-ns.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..7400f1ee --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/iceberg-rest-catalog/10-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/10-assert.yaml new file mode 100644 index 00000000..12951a0f --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/10-assert.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/10-install-garage.yaml b/tests/templates/kuttl/iceberg-rest-catalog/10-install-garage.yaml new file mode 100644 index 00000000..014b08ed --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/10-install-garage.yaml @@ -0,0 +1,174 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage +data: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + # Single node, no redundancy. Test data is disposable (emptyDir). + replication_factor = 1 + + # Throwaway value, this cluster is never joined by another node. + rpc_secret = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + # Garage rejects requests signed for a different region, so this must match + # the region of the S3Connection in 20_hive.yaml. + s3_region = "region-1" + api_bind_addr = "[::]:3900" + + [admin] + api_bind_addr = "[::]:3903" + # Garage terminates no TLS on any endpoint, so nginx does it and forwards to + # Garage on loopback. Trino requires TLS for S3. + # Mounted over /etc/nginx/nginx.conf, hence the full file. + nginx.conf: | + # OpenShift runs the container as an arbitrary UID, which may not write to + # /run. This file replaces the one in the image, so it must be set here. + pid /tmp/nginx.pid; + + events {} + http { + server { + listen 9000 ssl; + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; + + # Don't buffer or size-limit object uploads. + client_max_body_size 0; + + location / { + proxy_pass http://127.0.0.1:3900; + # Must be the original Host, it is part of the SigV4 signature. + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + } + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: garage +spec: + selector: + app: garage + ports: + - name: https + port: 9000 + targetPort: https +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + labels: + app: garage +spec: + replicas: 1 + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + spec: + containers: + - name: garage + image: oci.stackable.tech/stackable/dxflrs/garage:v2.4.1 + imagePullPolicy: IfNotPresent + # The image has no entrypoint. --single-node creates the cluster + # layout, --default-bucket creates the bucket and the access key from + # the GARAGE_DEFAULT_* variables below (implies --default-access-key). + command: + - /garage + - server + - --single-node + - --default-bucket + env: + - name: GARAGE_DEFAULT_BUCKET + value: lakehouse + # Garage requires the GK<24 hex> key id format and a 64 hex char + # secret. These are also set in 20_hive.yaml. + - name: GARAGE_DEFAULT_ACCESS_KEY + value: GK31c0ffee31c0ffee31c0ffee + - name: GARAGE_DEFAULT_SECRET_KEY + value: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + ports: + - name: rpc + containerPort: 3901 + - name: admin + containerPort: 3903 + readinessProbe: + httpGet: + path: /health + port: admin + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: data + mountPath: /var/lib/garage + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "1" + memory: 512Mi + - name: nginx + image: docker.io/library/nginx:1.29-alpine + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 9000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tls + mountPath: /stackable/tls + # nginx creates its temp dirs here on startup even though the proxy + # buffering above is off, and the image owns /var/cache/nginx by the + # image UID, not the arbitrary one OpenShift assigns. + - name: nginx-cache + mountPath: /var/cache/nginx + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 128Mi + volumes: + - name: config + configMap: + name: garage + - name: data + emptyDir: {} + - name: nginx-cache + emptyDir: {} + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=garage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech diff --git a/tests/templates/kuttl/iceberg-rest-catalog/10-install-postgres.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/10-install-postgres.yaml.j2 new file mode 100644 index 00000000..a62ebdf2 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/10-install-postgres.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install postgresql + --version={{ test_scenario['values']['postgres'] }} + --namespace "$NAMESPACE" + -f 10_helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/kerberos-hdfs/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/10_helm-bitnami-postgresql-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/kerberos-hdfs/helm-bitnami-postgresql-values.yaml.j2 rename to tests/templates/kuttl/iceberg-rest-catalog/10_helm-bitnami-postgresql-values.yaml.j2 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/20-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/20-assert.yaml new file mode 100644 index 00000000..fed212dd --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/20-assert.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +commands: + - script: kubectl -n "$NAMESPACE" wait --for=condition=available hiveclusters.hive.stackable.tech/hive --timeout 10m + - script: kubectl -n "$NAMESPACE" wait --for=condition=available trinoclusters.trino.stackable.tech/trino --timeout 10m + - script: kubectl -n "$NAMESPACE" get cm hive -o json | jq -e '.data.HIVE' # Ensure the discovery CM is populated +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: Listener +metadata: + name: hive-metastore +spec: + ports: + - name: hive + port: 9083 + protocol: TCP +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trino-cli +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/20-install-hive-and-trino.yaml b/tests/templates/kuttl/iceberg-rest-catalog/20-install-hive-and-trino.yaml new file mode 100644 index 00000000..6d85208a --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/20-install-hive-and-trino.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < 20_hive.yaml | kubectl apply -n "$NAMESPACE" -f - + - script: kubectl apply -n "$NAMESPACE" -f 20_trino.yaml diff --git a/tests/templates/kuttl/iceberg-rest-catalog/20_hive.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/20_hive.yaml.j2 new file mode 100644 index 00000000..95ad9d14 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/20_hive.yaml.j2 @@ -0,0 +1,86 @@ +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Connection +metadata: + name: garage +spec: + host: garage.$NAMESPACE.svc.cluster.local + port: 9000 + # The region must match the s3_region in 10-install-garage.yaml. + region: + name: region-1 + accessStyle: Path + credentials: + secretClass: s3-credentials-class + tls: + verification: + server: + caCert: + secretClass: tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: s3-credentials-class +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: garage-credentials + labels: + secrets.stackable.tech/class: s3-credentials-class +stringData: + # Must match the GARAGE_DEFAULT_* variables in 10-install-garage.yaml. + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive +--- +# The starting point of the migration: A Hive metastore that only serves the Thrift API. +# The Iceberg REST catalog is enabled in step 30 and the Thrift API disabled in step 40. +apiVersion: hive.stackable.tech/v1alpha2 +kind: HiveCluster +metadata: + name: hive +spec: + image: +{% if test_scenario['values']['hive-rest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive-rest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive-rest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hive-rest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + # The HMS is restarted on every config change in this test, so the metadata needs to be stored + # persistently (Derby would lose all tables). + metadataDatabase: + postgresql: + host: postgresql + database: hive + credentialsSecretName: hive-credentials + s3: + reference: garage + warehouseDir: s3a://lakehouse/warehouse/ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/20_trino.yaml.j2 b/tests/templates/kuttl/iceberg-rest-catalog/20_trino.yaml.j2 new file mode 100644 index 00000000..976316c0 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/20_trino.yaml.j2 @@ -0,0 +1,101 @@ +--- +# Iceberg tables accessed via the Thrift API of the Hive metastore, as existing users have them. +apiVersion: trino.stackable.tech/v1alpha2 +kind: TrinoCatalog +metadata: + name: iceberg-thrift + labels: + trino: trino +spec: + name: + inferred: + replaceHyphensWithUnderscores: true # iceberg_thrift + connector: + iceberg: + catalog: + hiveMetastore: + configMap: hive + s3: + reference: garage +--- +# Plain (non-Iceberg) Hive tables, which sometimes live in the same Hive metastore (although I +# personally don't recommend that) +apiVersion: trino.stackable.tech/v1alpha2 +kind: TrinoCatalog +metadata: + name: hive + labels: + trino: trino +spec: + connector: + hive: + metastore: + configMap: hive + s3: + reference: garage +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: trino +spec: + image: +{% if test_scenario['values']['trino-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trino-cli + labels: + app: trino-cli +spec: + replicas: 1 + selector: + matchLabels: + app: trino-cli + template: + metadata: + labels: + app: trino-cli + spec: + containers: + - name: trino-cli + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'].split(',')[0] }}-stackable0.0.0-dev" + command: + - sleep + - infinity + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 512Mi + cpu: "1" diff --git a/tests/templates/kuttl/iceberg-rest-catalog/21-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/21-assert.yaml new file mode 100644 index 00000000..69be0b76 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/21-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: sh orders.sh iceberg_thrift | diff 21_orders.csv - + - script: test "$(sh trino-query.sh 'SELECT array_agg(revenue ORDER BY day) FROM iceberg_thrift.sales.daily_revenue')" = '[7.99, 8.75]' + - script: test "$(sh trino-query.sh 'SELECT count(*) FROM hive.sales.legacy_orders')" = 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/21-create-tables.yaml b/tests/templates/kuttl/iceberg-rest-catalog/21-create-tables.yaml new file mode 100644 index 00000000..ee429225 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/21-create-tables.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: sh trino-query.sh < 21_create-tables.sql diff --git a/tests/templates/kuttl/iceberg-rest-catalog/21_create-tables.sql b/tests/templates/kuttl/iceberg-rest-catalog/21_create-tables.sql new file mode 100644 index 00000000..4d6b964f --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/21_create-tables.sql @@ -0,0 +1,81 @@ +-- Create what existing users have: Iceberg tables, views and plain Hive tables, all created via the +-- Thrift API of the Hive metastore. + +CREATE SCHEMA IF NOT EXISTS iceberg_thrift.sales WITH (location = 's3a://lakehouse/sales/'); + +CREATE TABLE IF NOT EXISTS iceberg_thrift.sales.orders ( + id bigint, + order_uuid uuid, + customer ROW(name varchar, email varchar, address ROW(city varchar, zip varchar)), + items ARRAY(ROW(sku varchar, quantity integer, price decimal(10, 2))), + tags MAP(varchar, varchar), + total decimal(12, 2), + express boolean, + weight_kg double, + ordered_at timestamp(6) with time zone, + updated_at timestamp(6), + delivery_date date +) +WITH (partitioning = ARRAY['day(ordered_at)']); + +-- Two separate inserts, so that the table has multiple snapshots (for time travel). Note that the +-- CREATE TABLE already created an (empty) snapshot. +INSERT INTO iceberg_thrift.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '1', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a01', + ROW('Alice', 'alice@example.com', ROW('Berlin', '10115')), + ARRAY[ROW('apple', 3, DECIMAL '0.50'), ROW('banana', 6, DECIMAL '0.25')], + MAP(ARRAY['channel', 'coupon'], ARRAY['web', 'WELCOME10']), + DECIMAL '3.00', + false, + 1.2E0, + TIMESTAMP '2026-11-01 09:15:00.123456 UTC', + TIMESTAMP '2026-11-01 09:15:00.123456', + DATE '2026-11-03' + ), + ( + BIGINT '2', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a02', + ROW('Bob', 'bob@example.com', ROW('Hamburg', '20095')), + ARRAY[ROW('cherry', 1, DECIMAL '4.99')], + MAP(ARRAY['channel'], ARRAY['app']), + DECIMAL '4.99', + true, + 0.3E0, + TIMESTAMP '2026-11-01 17:42:13.000001 Europe/Berlin', + TIMESTAMP '2026-11-01 16:42:13.000001', + DATE '2026-11-02' + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date) +WHERE id NOT IN (SELECT id FROM iceberg_thrift.sales.orders); + +INSERT INTO iceberg_thrift.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '3', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a03', + ROW('Carla', 'carla@example.com', ROW('Munich', '80331')), + ARRAY[ROW('date', 2, DECIMAL '2.50'), ROW('elderberry', 1, DECIMAL '3.75')], + MAP(ARRAY['channel'], ARRAY['store']), + DECIMAL '8.75', + false, + CAST(NULL AS double), + TIMESTAMP '2026-11-02 08:00:00.000000 UTC', + TIMESTAMP '2026-11-02 08:00:00.000000', + CAST(NULL AS date) + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date) +WHERE id NOT IN (SELECT id FROM iceberg_thrift.sales.orders); + +-- A Trino view, which Trino stores as Hive view in the Hive metastore +CREATE OR REPLACE VIEW iceberg_thrift.sales.daily_revenue AS +SELECT date(ordered_at AT TIME ZONE 'UTC') AS day, sum(total) AS revenue +FROM iceberg_thrift.sales.orders +GROUP BY 1; + +CREATE TABLE IF NOT EXISTS hive.sales.legacy_orders AS +SELECT * FROM (VALUES (1, 'apple')) AS t (id, item); diff --git a/tests/templates/kuttl/iceberg-rest-catalog/21_orders.csv b/tests/templates/kuttl/iceberg-rest-catalog/21_orders.csv new file mode 100644 index 00000000..a116ba66 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/21_orders.csv @@ -0,0 +1,3 @@ +1,Alice,Berlin,2,apple,web,3.00,false,1.2,2026-11-01 09:15:00.123456 UTC,2026-11-01 09:15:00.123456,2026-11-03 +2,Bob,Hamburg,1,cherry,app,4.99,true,0.3,2026-11-01 16:42:13.000001 UTC,2026-11-01 16:42:13.000001,2026-11-02 +3,Carla,Munich,2,date,store,8.75,false,,2026-11-02 08:00:00.000000 UTC,2026-11-02 08:00:00.000000, diff --git a/tests/templates/kuttl/iceberg-rest-catalog/30-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/30-assert.yaml new file mode 100644 index 00000000..5954b7f0 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/30-assert.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + # The Thrift API is still available + - script: kubectl -n "$NAMESPACE" get cm hive -o json | jq -e '.data.HIVE' + # The existing Iceberg table shows up in the REST catalog. The plain Hive table and the Trino view + # (stored as Hive view) do not. + - script: test "$(sh trino-query.sh 'SHOW TABLES FROM iceberg_rest.sales')" = orders + # The content, including complex types, timestamps and NULLs, is the same via both APIs + - script: sh orders.sh iceberg_rest | diff 21_orders.csv - + - script: sh compare-catalogs.sh + # The partitioning and the table history are kept. The second snapshot is the first INSERT (the + # first one is the CREATE TABLE). + - script: test "$(sh trino-query.sh 'SELECT count(*) FROM iceberg_rest.sales."orders$partitions"')" = 2 + - script: | + SNAPSHOT=$(sh trino-query.sh 'SELECT snapshot_id FROM iceberg_rest.sales."orders$snapshots" ORDER BY committed_at OFFSET 1 LIMIT 1') + test "$(sh trino-query.sh "SELECT count(*) FROM iceberg_rest.sales.orders FOR VERSION AS OF $SNAPSHOT")" = 2 +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: Listener +metadata: + name: hive-metastore +spec: + ports: + - name: hive + port: 9083 + protocol: TCP + - name: iceberg-rest + port: 9001 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: hive-metastore-default-headless +spec: + ports: + - name: hive + port: 9083 + protocol: TCP + targetPort: 9083 + - name: iceberg-rest + port: 9001 + protocol: TCP + targetPort: 9001 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/30-enable-rest-catalog.yaml b/tests/templates/kuttl/iceberg-rest-catalog/30-enable-rest-catalog.yaml new file mode 100644 index 00000000..5f237bd5 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/30-enable-rest-catalog.yaml @@ -0,0 +1,24 @@ +--- +# Add a Trino catalog using the REST catalog +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < 30_trino-catalog-rest.yaml | kubectl apply -n "$NAMESPACE" -f - +--- +# Enable the Iceberg REST catalog next to the Thrift API. +apiVersion: hive.stackable.tech/v1alpha2 +kind: HiveCluster +metadata: + name: hive +spec: + clusterConfig: + # The REST catalog can not create any namespace (Trino schema) without warehouseExternalDir, not + # even one with an explicit location. Error message: + # ("java.lang.NullPointerException: Warehouse location is not set: hive.metastore.warehouse.external.dir=null"). + # + # There are ideas to fix this in Hive itself (https://github.com/stackabletech/docker-images/pull/1722), + # but I'm not sure if and when we will merge that. Because of that, I also did *not* add this as + # an error to hive-operator, as future hive images might not require this. + warehouseExternalDir: s3a://lakehouse/warehouse/external/ + icebergRestCatalog: + enabled: true diff --git a/tests/templates/kuttl/iceberg-rest-catalog/30_trino-catalog-rest.yaml b/tests/templates/kuttl/iceberg-rest-catalog/30_trino-catalog-rest.yaml new file mode 100644 index 00000000..d2d0d4b6 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/30_trino-catalog-rest.yaml @@ -0,0 +1,22 @@ +--- +# The same Iceberg tables, accessed via the Iceberg REST catalog of the Hive metastore. +apiVersion: trino.stackable.tech/v1alpha2 +kind: TrinoCatalog +metadata: + name: iceberg-rest + labels: + trino: trino +spec: + name: + inferred: + replaceHyphensWithUnderscores: true # iceberg_rest + connector: + iceberg: + catalog: + rest: + # The address of the Listener for the default cluster-internal ListenerClass + hostname: hive-metastore.$NAMESPACE.svc.cluster.local + port: 9001 + path: /iceberg + s3: + reference: garage diff --git a/tests/templates/kuttl/iceberg-rest-catalog/31-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/31-assert.yaml new file mode 100644 index 00000000..5f5938ab --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/31-assert.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + # Writes, schema evolution and row-level changes via either API are visible via the other one + - script: sh orders.sh iceberg_rest | diff 31_orders.csv - + - script: sh compare-catalogs.sh + - script: test "$(sh trino-query.sh 'SELECT id, discount FROM iceberg_thrift.sales.orders WHERE discount IS NOT NULL')" = 4,1.00 + # Views are not shared: The Iceberg view created via the REST catalog is listed, but can not be + # read by Trino via the Thrift API + - script: test "$(sh trino-query.sh 'SELECT array_agg(id ORDER BY id) FROM iceberg_rest.sales.express_orders')" = '[3, 4]' + - script: sh trino-query.sh 'SELECT * FROM iceberg_thrift.sales.express_orders' 2>&1 | grep -q 'Not an Iceberg table' + # Namespaces created via the REST catalog without a location are placed in warehouseExternalDir + - script: test "$(sh trino-query.sh 'SELECT count(*) FROM iceberg_thrift.marketing.campaigns')" = 2 + - script: sh trino-query.sh 'SELECT DISTINCT "$path" FROM iceberg_rest.marketing.campaigns' | grep -q '^s3a://lakehouse/warehouse/external/marketing.db/' + # The plain Hive table is still accessible via the Thrift API + - script: test "$(sh trino-query.sh 'SELECT count(*) FROM hive.sales.legacy_orders')" = 1 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/31-use-both-apis.yaml b/tests/templates/kuttl/iceberg-rest-catalog/31-use-both-apis.yaml new file mode 100644 index 00000000..16cdeb4a --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/31-use-both-apis.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: sh trino-query.sh < 31_use-both-apis.sql diff --git a/tests/templates/kuttl/iceberg-rest-catalog/31_orders.csv b/tests/templates/kuttl/iceberg-rest-catalog/31_orders.csv new file mode 100644 index 00000000..89449a1b --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/31_orders.csv @@ -0,0 +1,4 @@ +1,Alice,Berlin,2,apple,web,3.00,false,1.2,2026-11-01 09:15:00.123456 UTC,2026-11-01 09:15:00.123456,2026-11-03 +3,Carla,Munich,2,date,store,8.75,true,,2026-11-02 08:00:00.000000 UTC,2026-11-02 10:00:00.000000, +4,Dave,Cologne,1,fig,web,7.00,true,0.5,2026-11-02 12:30:00.000000 UTC,2026-11-02 12:30:00.000000,2026-11-03 +5,Eve,Leipzig,2,grape,app,5.50,false,2.0,2026-11-03 20:00:00.000000 UTC,2026-11-03 20:00:00.000000,2026-11-05 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/31_use-both-apis.sql b/tests/templates/kuttl/iceberg-rest-catalog/31_use-both-apis.sql new file mode 100644 index 00000000..bcc3d507 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/31_use-both-apis.sql @@ -0,0 +1,62 @@ +-- Use the Thrift API and the REST catalog side by side, e.g. while migrating clients one by one. + +-- Schema evolution via the REST catalog, the new column is then used via the Thrift API +ALTER TABLE iceberg_rest.sales.orders ADD COLUMN IF NOT EXISTS discount decimal(5, 2); + +INSERT INTO iceberg_thrift.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '4', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a04', + ROW('Dave', 'dave@example.com', ROW('Cologne', '50667')), + ARRAY[ROW('fig', 10, DECIMAL '0.80')], + MAP(ARRAY['channel', 'coupon'], ARRAY['web', 'FIGS']), + DECIMAL '7.00', + true, + 0.5E0, + TIMESTAMP '2026-11-02 12:30:00.000000 UTC', + TIMESTAMP '2026-11-02 12:30:00.000000', + DATE '2026-11-03', + DECIMAL '1.00' + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date, discount) +WHERE id NOT IN (SELECT id FROM iceberg_thrift.sales.orders); + +-- New partition via the REST catalog +INSERT INTO iceberg_rest.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '5', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a05', + ROW('Eve', 'eve@example.com', ROW('Leipzig', '04109')), + ARRAY[ROW('grape', 1, DECIMAL '2.20'), ROW('honeydew', 1, DECIMAL '3.30')], + MAP(ARRAY['channel'], ARRAY['app']), + DECIMAL '5.50', + false, + 2.0E0, + TIMESTAMP '2026-11-03 20:00:00.000000 UTC', + TIMESTAMP '2026-11-03 20:00:00.000000', + DATE '2026-11-05', + CAST(NULL AS decimal(5, 2)) + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date, discount) +WHERE id NOT IN (SELECT id FROM iceberg_rest.sales.orders); + +-- Row-level changes via both APIs, which (for format version 2) write delete files +DELETE FROM iceberg_rest.sales.orders WHERE id = 2; +UPDATE iceberg_thrift.sales.orders +SET express = true, updated_at = TIMESTAMP '2026-11-02 10:00:00.000000' +WHERE id = 3; + +-- An Iceberg view, created via the REST catalog +CREATE OR REPLACE VIEW iceberg_rest.sales.express_orders AS +SELECT id, customer.name AS customer, total +FROM iceberg_rest.sales.orders +WHERE express; + +-- A new namespace via the REST catalog, without an explicit location +CREATE SCHEMA IF NOT EXISTS iceberg_rest.marketing; +CREATE TABLE IF NOT EXISTS iceberg_rest.marketing.campaigns AS +SELECT * FROM (VALUES (1, 'launch'), (2, 'sale')) AS t (id, name); diff --git a/tests/templates/kuttl/iceberg-rest-catalog/32-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/32-assert.yaml new file mode 100644 index 00000000..0bc100c2 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/32-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + # Via the REST catalog: One data file per partition (3 partitions) and no delete files left + - script: test "$(sh trino-query.sh 'SELECT count(*) FILTER (WHERE content = 0), count(DISTINCT partition), count(*) FILTER (WHERE content <> 0) FROM iceberg_rest.sales."orders$files"')" = 3,3,0 + - script: test "$(sh trino-query.sh 'SELECT operation FROM iceberg_rest.sales."orders$snapshots" ORDER BY committed_at DESC LIMIT 1')" = replace + # The content is unchanged + - script: sh orders.sh iceberg_rest | diff 31_orders.csv - + - script: sh compare-catalogs.sh diff --git a/tests/templates/kuttl/iceberg-rest-catalog/32-compact-via-thrift.yaml b/tests/templates/kuttl/iceberg-rest-catalog/32-compact-via-thrift.yaml new file mode 100644 index 00000000..62bde2ee --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/32-compact-via-thrift.yaml @@ -0,0 +1,7 @@ +--- +# Compaction via the Thrift API, which rewrites the data files and applies the delete files written +# by the DELETE and UPDATE in step 31 +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: sh trino-query.sh 'ALTER TABLE iceberg_thrift.sales.orders EXECUTE optimize' diff --git a/tests/templates/kuttl/iceberg-rest-catalog/33-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/33-assert.yaml new file mode 100644 index 00000000..34a71d74 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/33-assert.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + # Via the Thrift API: One data file per partition (3 partitions) and no delete files left + - script: test "$(sh trino-query.sh 'SELECT count(*) FILTER (WHERE content = 0), count(DISTINCT partition), count(*) FILTER (WHERE content <> 0) FROM iceberg_thrift.sales."orders$files"')" = 3,3,0 + - script: test "$(sh trino-query.sh 'SELECT operation FROM iceberg_thrift.sales."orders$snapshots" ORDER BY committed_at DESC LIMIT 1')" = replace + - script: sh orders.sh iceberg_thrift | diff 33_orders.csv - + - script: sh compare-catalogs.sh diff --git a/tests/templates/kuttl/iceberg-rest-catalog/33-compact-via-rest.yaml b/tests/templates/kuttl/iceberg-rest-catalog/33-compact-via-rest.yaml new file mode 100644 index 00000000..ee566576 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/33-compact-via-rest.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: sh trino-query.sh < 33_compact-via-rest.sql diff --git a/tests/templates/kuttl/iceberg-rest-catalog/33_compact-via-rest.sql b/tests/templates/kuttl/iceberg-rest-catalog/33_compact-via-rest.sql new file mode 100644 index 00000000..23b030c9 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/33_compact-via-rest.sql @@ -0,0 +1,23 @@ +-- Add another data file to an existing partition via the Thrift API ... +INSERT INTO iceberg_thrift.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '6', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a06', + ROW('Frank', 'frank@example.com', ROW('Dresden', '01067')), + ARRAY[ROW('kiwi', 4, DECIMAL '0.60')], + MAP(ARRAY['channel'], ARRAY['web']), + DECIMAL '2.40', + false, + 0.4E0, + TIMESTAMP '2026-11-01 11:11:11.111111 UTC', + TIMESTAMP '2026-11-01 11:11:11.111111', + DATE '2026-11-04', + CAST(NULL AS decimal(5, 2)) + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date, discount) +WHERE id NOT IN (SELECT id FROM iceberg_thrift.sales.orders); + +-- ... and compact it via the REST catalog +ALTER TABLE iceberg_rest.sales.orders EXECUTE optimize; diff --git a/tests/templates/kuttl/iceberg-rest-catalog/33_orders.csv b/tests/templates/kuttl/iceberg-rest-catalog/33_orders.csv new file mode 100644 index 00000000..652fa288 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/33_orders.csv @@ -0,0 +1,5 @@ +1,Alice,Berlin,2,apple,web,3.00,false,1.2,2026-11-01 09:15:00.123456 UTC,2026-11-01 09:15:00.123456,2026-11-03 +3,Carla,Munich,2,date,store,8.75,true,,2026-11-02 08:00:00.000000 UTC,2026-11-02 10:00:00.000000, +4,Dave,Cologne,1,fig,web,7.00,true,0.5,2026-11-02 12:30:00.000000 UTC,2026-11-02 12:30:00.000000,2026-11-03 +5,Eve,Leipzig,2,grape,app,5.50,false,2.0,2026-11-03 20:00:00.000000 UTC,2026-11-03 20:00:00.000000,2026-11-05 +6,Frank,Dresden,1,kiwi,web,2.40,false,0.4,2026-11-01 11:11:11.111111 UTC,2026-11-01 11:11:11.111111,2026-11-04 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/40-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/40-assert.yaml new file mode 100644 index 00000000..87b81111 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/40-assert.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: kubectl -n "$NAMESPACE" get cm hive -o json | jq -e '.data.HIVE == null' + # The Thrift server is bound to the loopback interface, so it refuses connections to the Pod IP. + # Other errors, such as the Pod name not resolving during the restart, don't count. + - script: >- + kubectl -n "$NAMESPACE" exec trino-cli-0 -- + timeout 5 bash -c "&1 + | grep -q 'Connection refused' + # All tables, including the ones created via the Thrift API, are still accessible + - script: sh orders.sh iceberg_rest | diff 33_orders.csv - + - script: test "$(sh trino-query.sh 'SELECT count(*) FROM iceberg_rest.marketing.campaigns')" = 2 +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: Listener +metadata: + name: hive-metastore +spec: + ports: + - name: iceberg-rest + port: 9001 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: hive-metastore-default-headless +spec: + ports: + - name: iceberg-rest + port: 9001 + protocol: TCP + targetPort: 9001 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/40-disable-thrift-api.yaml b/tests/templates/kuttl/iceberg-rest-catalog/40-disable-thrift-api.yaml new file mode 100644 index 00000000..128e6533 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/40-disable-thrift-api.yaml @@ -0,0 +1,20 @@ +--- +# Finish the migration: All clients use the REST catalog, so the Thrift API can be disabled. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: trino.stackable.tech/v1alpha2 + kind: TrinoCatalog + name: iceberg-thrift + - apiVersion: trino.stackable.tech/v1alpha2 + kind: TrinoCatalog + name: hive +--- +apiVersion: hive.stackable.tech/v1alpha2 +kind: HiveCluster +metadata: + name: hive +spec: + clusterConfig: + thriftApi: + enabled: false diff --git a/tests/templates/kuttl/iceberg-rest-catalog/41-assert.yaml b/tests/templates/kuttl/iceberg-rest-catalog/41-assert.yaml new file mode 100644 index 00000000..8e329ece --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/41-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: sh orders.sh iceberg_rest | diff 41_orders.csv - + - script: test "$(sh trino-query.sh 'SELECT count(*) FILTER (WHERE content = 0), count(DISTINCT partition), count(*) FILTER (WHERE content <> 0) FROM iceberg_rest.sales."orders$files"')" = 3,3,0 + - script: test "$(sh trino-query.sh 'SELECT array_agg(id ORDER BY id) FROM iceberg_rest.sales.express_orders')" = '[3, 4, 7]' diff --git a/tests/templates/kuttl/iceberg-rest-catalog/41-use-rest-only.yaml b/tests/templates/kuttl/iceberg-rest-catalog/41-use-rest-only.yaml new file mode 100644 index 00000000..906d81cb --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/41-use-rest-only.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: sh trino-query.sh < 41_use-rest-only.sql diff --git a/tests/templates/kuttl/iceberg-rest-catalog/41_orders.csv b/tests/templates/kuttl/iceberg-rest-catalog/41_orders.csv new file mode 100644 index 00000000..7e7036e1 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/41_orders.csv @@ -0,0 +1,6 @@ +1,Alice,Berlin,2,apple,web,3.00,false,1.2,2026-11-01 09:15:00.123456 UTC,2026-11-01 09:15:00.123456,2026-11-03 +3,Carla,Munich,2,date,store,8.75,true,,2026-11-02 08:00:00.000000 UTC,2026-11-02 10:00:00.000000, +4,Dave,Cologne,1,fig,web,7.00,true,0.5,2026-11-02 12:30:00.000000 UTC,2026-11-02 12:30:00.000000,2026-11-03 +5,Eve,Leipzig,2,grape,app,5.50,false,2.0,2026-11-03 20:00:00.000000 UTC,2026-11-03 20:00:00.000000,2026-11-05 +6,Frank,Dresden,1,kiwi,web,2.40,false,0.4,2026-11-01 11:11:11.111111 UTC,2026-11-01 11:11:11.111111,2026-11-04 +7,Grace,Bremen,1,lemon,store,0.90,true,0.2,2026-11-03 07:00:00.000000 UTC,2026-11-03 07:00:00.000000,2026-11-04 diff --git a/tests/templates/kuttl/iceberg-rest-catalog/41_use-rest-only.sql b/tests/templates/kuttl/iceberg-rest-catalog/41_use-rest-only.sql new file mode 100644 index 00000000..ea596ea8 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/41_use-rest-only.sql @@ -0,0 +1,23 @@ +-- Write and compact via the REST catalog while the Thrift API is disabled. Commits take locks in the +-- Hive metastore, which the REST catalog does via the Thrift server bound to the loopback interface. +INSERT INTO iceberg_rest.sales.orders +SELECT * FROM ( + VALUES + ( + BIGINT '7', + UUID '7e1f1c3a-1d9b-4c55-9c39-2a8f3b6e0a07', + ROW('Grace', 'grace@example.com', ROW('Bremen', '28195')), + ARRAY[ROW('lemon', 2, DECIMAL '0.45')], + MAP(ARRAY['channel'], ARRAY['store']), + DECIMAL '0.90', + true, + 0.2E0, + TIMESTAMP '2026-11-03 07:00:00.000000 UTC', + TIMESTAMP '2026-11-03 07:00:00.000000', + DATE '2026-11-04', + CAST(NULL AS decimal(5, 2)) + ) +) AS new_orders (id, order_uuid, customer, items, tags, total, express, weight_kg, ordered_at, updated_at, delivery_date, discount) +WHERE id NOT IN (SELECT id FROM iceberg_rest.sales.orders); + +ALTER TABLE iceberg_rest.sales.orders EXECUTE optimize; diff --git a/tests/templates/kuttl/iceberg-rest-catalog/compare-catalogs.sh b/tests/templates/kuttl/iceberg-rest-catalog/compare-catalogs.sh new file mode 100644 index 00000000..2e534de4 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/compare-catalogs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env sh +# Fails unless the orders table has the same content via the Thrift API and via the REST catalog. +# The content is compared using the row count and an order-independent checksum over all columns. +# Usage: sh compare-catalogs.sh +set -eu + +checksum() { + COLUMN_LIST=$(sh trino-query.sh " + SELECT array_join(array_agg(column_name ORDER BY ordinal_position), ', ') + FROM $1.information_schema.columns + WHERE table_schema = 'sales' AND table_name = 'orders' + ") + sh trino-query.sh "SELECT count(*), checksum(ROW($COLUMN_LIST)) FROM $1.sales.orders" +} + +THRIFT=$(checksum iceberg_thrift) +REST=$(checksum iceberg_rest) +echo "Thrift API: $THRIFT" +echo "REST catalog: $REST" +test "$THRIFT" = "$REST" diff --git a/tests/templates/kuttl/iceberg-rest-catalog/orders.sh b/tests/templates/kuttl/iceberg-rest-catalog/orders.sh new file mode 100644 index 00000000..e0461b4d --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/orders.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh +# Prints selected columns of the orders table as seen via the given Trino catalog as CSV, so that it +# can be compared with the expected content, e.g. `sh orders.sh iceberg_rest | diff 21_orders.csv -`. +# Usage: sh orders.sh iceberg_rest +set -eu + +sh trino-query.sh " + SELECT + id, + customer.name, + customer.address.city, + cardinality(items), + element_at(items, 1).sku, + element_at(tags, 'channel'), + total, + express, + weight_kg, + ordered_at, + updated_at, + delivery_date + FROM $1.sales.orders + ORDER BY id +" diff --git a/tests/templates/kuttl/iceberg-rest-catalog/trino-query.sh b/tests/templates/kuttl/iceberg-rest-catalog/trino-query.sh new file mode 100644 index 00000000..0bd5b52a --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest-catalog/trino-query.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh +# Executes SQL in Trino and prints the results as unquoted CSV without header. +# Usage: sh trino-query.sh "SELECT 1" +# sh trino-query.sh < statements.sql +set -eu + +TRINO_CLI="./trino-cli --server https://trino-coordinator:8443 --insecure --user admin --output-format CSV_UNQUOTED" + +if [ $# -gt 0 ]; then + # shellcheck disable=SC2086 # Word splitting of TRINO_CLI is intended + kubectl exec -n "$NAMESPACE" trino-cli-0 -- $TRINO_CLI --execute "$1" +else + # shellcheck disable=SC2086 # Word splitting of TRINO_CLI is intended + kubectl exec -i -n "$NAMESPACE" trino-cli-0 -- $TRINO_CLI +fi diff --git a/tests/templates/kuttl/kerberos-hdfs/35-access-hdfs.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/35-access-hdfs.yaml.j2 index 150d1128..9a2704ea 100644 --- a/tests/templates/kuttl/kerberos-hdfs/35-access-hdfs.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/35-access-hdfs.yaml.j2 @@ -33,8 +33,6 @@ commands: kinit -kt /stackable/kerberos/keytab testuser/access-hdfs.$NAMESPACE.svc.cluster.local klist - bin/hdfs dfs -mkdir /stackable - bin/hdfs dfs -chown -R access-hive /stackable bin/hdfs dfs -mkdir /access-hive bin/hdfs dfs -chown -R access-hive /access-hive bin/hdfs dfs -ls / diff --git a/tests/templates/kuttl/kerberos-hdfs/40-install-postgres.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/40-install-postgres.yaml.j2 index aa7e4108..0717fadb 100644 --- a/tests/templates/kuttl/kerberos-hdfs/40-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/40-install-postgres.yaml.j2 @@ -6,5 +6,5 @@ commands: helm install postgresql --version={{ test_scenario['values']['postgres'] }} --namespace "$NAMESPACE" - -f helm-bitnami-postgresql-values.yaml + -f 40_helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/kerberos-s3/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/40_helm-bitnami-postgresql-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/kerberos-s3/helm-bitnami-postgresql-values.yaml.j2 rename to tests/templates/kuttl/kerberos-hdfs/40_helm-bitnami-postgresql-values.yaml.j2 diff --git a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 index 49ec2332..bbaea9c4 100644 --- a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 @@ -5,7 +5,7 @@ commands: - script: | kubectl apply -n "$NAMESPACE" -f - < kubectl create configmap prepared-logs - --from-file=prepared-logs.log4j2.xml + --from-file=03_prepared-logs.log4j2.xml --namespace="$NAMESPACE" diff --git a/tests/templates/kuttl/logging/prepared-logs.log4j2.xml b/tests/templates/kuttl/logging/03_prepared-logs.log4j2.xml similarity index 100% rename from tests/templates/kuttl/logging/prepared-logs.log4j2.xml rename to tests/templates/kuttl/logging/03_prepared-logs.log4j2.xml diff --git a/tests/templates/kuttl/logging/04-install-hive.yaml.j2 b/tests/templates/kuttl/logging/04-install-hive.yaml.j2 index 16e30f49..51f301d3 100644 --- a/tests/templates/kuttl/logging/04-install-hive.yaml.j2 +++ b/tests/templates/kuttl/logging/04-install-hive.yaml.j2 @@ -33,7 +33,7 @@ data: rootLogger.appenderRef.CONSOLE.ref = CONSOLE rootLogger.appenderRef.FILE.ref = FILE --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive @@ -52,6 +52,7 @@ spec: host: hive-postgresql database: hive credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/my-warehouse/ vectorAggregatorConfigMapName: hive-vector-aggregator-discovery metastore: roleGroups: diff --git a/tests/templates/kuttl/logging/06-assert.yaml b/tests/templates/kuttl/logging/06-assert.yaml index d7adc692..2c6c55bb 100644 --- a/tests/templates/kuttl/logging/06-assert.yaml +++ b/tests/templates/kuttl/logging/06-assert.yaml @@ -4,4 +4,4 @@ kind: TestAssert commands: - script: >- kubectl exec --namespace="$NAMESPACE" hive-test-runner-0 -- - python /tmp/test_log_aggregation.py -n "$NAMESPACE" + python /tmp/06_test_log_aggregation.py -n "$NAMESPACE" diff --git a/tests/templates/kuttl/logging/06-test-log-aggregation.yaml b/tests/templates/kuttl/logging/06-test-log-aggregation.yaml index fb124ab1..ce929a88 100644 --- a/tests/templates/kuttl/logging/06-test-log-aggregation.yaml +++ b/tests/templates/kuttl/logging/06-test-log-aggregation.yaml @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - kubectl cp ./test_log_aggregation.py "$NAMESPACE/hive-test-runner-0:/tmp" + kubectl cp ./06_test_log_aggregation.py "$NAMESPACE/hive-test-runner-0:/tmp" diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/06_test_log_aggregation.py similarity index 100% rename from tests/templates/kuttl/logging/test_log_aggregation.py rename to tests/templates/kuttl/logging/06_test_log_aggregation.py diff --git a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 index de25c096..bb3bd525 100644 --- a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive @@ -15,6 +15,7 @@ spec: clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/my-warehouse/ {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 b/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 index ddc0eda0..3123fcae 100644 --- a/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive diff --git a/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml b/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml index 712baac9..c42e419c 100644 --- a/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml +++ b/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: test-hive diff --git a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 index 7c847240..34748ca4 100644 --- a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -15,6 +15,7 @@ spec: clusterConfig: metadataDatabase: derby: {} + warehouseDir: s3a://my-bucket/my-warehouse/ {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 b/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 index d5070277..7f44aa20 100644 --- a/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 @@ -9,6 +9,6 @@ commands: helm install minio --namespace "$NAMESPACE" --version 12.6.4 - -f helm-bitnami-minio-values.yaml + -f 30_helm-bitnami-minio-values.yaml --repo https://charts.bitnami.com/bitnami minio timeout: 240 diff --git a/tests/templates/kuttl/smoke/helm-bitnami-minio-values.yaml.j2 b/tests/templates/kuttl/smoke/30_helm-bitnami-minio-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/smoke/helm-bitnami-minio-values.yaml.j2 rename to tests/templates/kuttl/smoke/30_helm-bitnami-minio-values.yaml.j2 diff --git a/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 b/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 index aa7e4108..0717fadb 100644 --- a/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 @@ -6,5 +6,5 @@ commands: helm install postgresql --version={{ test_scenario['values']['postgres'] }} --namespace "$NAMESPACE" - -f helm-bitnami-postgresql-values.yaml + -f 40_helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/upgrade/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/smoke/40_helm-bitnami-postgresql-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/upgrade/helm-bitnami-postgresql-values.yaml.j2 rename to tests/templates/kuttl/smoke/40_helm-bitnami-postgresql-values.yaml.j2 diff --git a/tests/templates/kuttl/smoke/60-assert.yaml.j2 b/tests/templates/kuttl/smoke/60-assert.yaml.j2 index 85d5708b..cbd5e131 100644 --- a/tests/templates/kuttl/smoke/60-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/60-assert.yaml.j2 @@ -39,7 +39,7 @@ metadata: stackable.tech/vendor: Stackable name: hive-metastore-default-headless ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -74,7 +74,7 @@ metadata: stackable.tech/vendor: Stackable name: hive-metastore-default-metrics ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -106,7 +106,7 @@ metadata: stackable.tech/vendor: Stackable name: hive-metastore-default ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -466,7 +466,7 @@ metadata: stackable.tech/vendor: Stackable name: hive ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -483,7 +483,7 @@ metadata: stackable.tech/vendor: Stackable name: hive-metastore-default ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -497,7 +497,7 @@ metadata: app.kubernetes.io/name: hive name: hive-serviceaccount ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -511,7 +511,7 @@ metadata: app.kubernetes.io/name: hive name: hive-rolebinding ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -534,7 +534,7 @@ metadata: app.kubernetes.io/name: hive name: hive-metastore ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive @@ -562,7 +562,7 @@ metadata: stackable.tech/vendor: Stackable name: hive-metastore ownerReferences: - - apiVersion: hive.stackable.tech/v1alpha1 + - apiVersion: hive.stackable.tech/v1alpha2 controller: true kind: HiveCluster name: hive diff --git a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 index 2b1ef1c3..3d5af530 100644 --- a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 @@ -46,7 +46,7 @@ stringData: username: hive password: hive --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -71,6 +71,7 @@ spec: credentialsSecretName: hive-credentials s3: reference: minio + warehouseDir: s3a://hive {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -83,7 +84,7 @@ spec: ROLE_VAR: role-value # only defined here at role level configOverrides: hive-site.xml: - hive.metastore.warehouse.dir: "/stackable/warehouse/override" # Also set by the operator + hive.metastore.warehouse.dir: "s3a://hive/override" # Override what the operator set common-var: role-value # Overridden by role group below role-var: role-value # Only defined here at role level roleGroups: diff --git a/tests/templates/kuttl/smoke/61-assert.yaml b/tests/templates/kuttl/smoke/61-assert.yaml index c35e9a5f..11a1c984 100644 --- a/tests/templates/kuttl/smoke/61-assert.yaml +++ b/tests/templates/kuttl/smoke/61-assert.yaml @@ -14,7 +14,7 @@ commands: # Test configOverrides # - script: | - kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "hive.metastore.warehouse.dir") | .value' | grep -qx "/stackable/warehouse/override" + kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "hive.metastore.warehouse.dir") | .value' | grep -qx "s3a://hive/override" kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "role-var") | .value' | grep -qx "role-value" kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "group-var") | .value' | grep -qx "group-value" kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "common-var") | .value' | grep -qx "group-value" diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index ed78faa4..334588bb 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,6 +2,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -d test_metastore -m hive-metastore.$NAMESPACE.svc.cluster.local - - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -d test_metastore -m hive-metastore-default-headless.$NAMESPACE.svc.cluster.local - - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore_opa.py -d db_not_allowed -m hive-metastore.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/80_test_metastore.py -d test_metastore -m hive-metastore.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/80_test_metastore.py -d test_metastore -m hive-metastore-default-headless.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/80_test_metastore_opa.py -d db_not_allowed -m hive-metastore.$NAMESPACE.svc.cluster.local diff --git a/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml b/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml index 3704aaba..6a6d164e 100644 --- a/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml +++ b/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml @@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl cp -n "$NAMESPACE" ./test_metastore.py test-metastore-0:/tmp - - script: kubectl cp -n "$NAMESPACE" ./test_metastore_opa.py test-metastore-0:/tmp + - script: kubectl cp -n "$NAMESPACE" ./80_test_metastore.py test-metastore-0:/tmp + - script: kubectl cp -n "$NAMESPACE" ./80_test_metastore_opa.py test-metastore-0:/tmp diff --git a/tests/templates/kuttl/smoke/test_metastore.py b/tests/templates/kuttl/smoke/80_test_metastore.py similarity index 97% rename from tests/templates/kuttl/smoke/test_metastore.py rename to tests/templates/kuttl/smoke/80_test_metastore.py index 8cc33bbd..6cb558b1 100755 --- a/tests/templates/kuttl/smoke/test_metastore.py +++ b/tests/templates/kuttl/smoke/80_test_metastore.py @@ -13,7 +13,7 @@ GetTableRequest, ) import argparse - +import sys def table(db_name, table_name, location): columns = [ColumnBuilder("id", "string", "col comment").build()] @@ -57,7 +57,7 @@ def check_table(hive_client, db_name, table_name, location, label): print( f"[ERROR]: Received {label} schema {schema} - expected schema: {expected}" ) - exit(-1) + sys.exit(-1) return hive_client.get_table_req( GetTableRequest(dbName=db_name, tblName=table_name) @@ -76,7 +76,7 @@ def check_legacy_get_table(hive_client, db_name, table_name): print( f"[ERROR]: Legacy get_table returned table {legacy.tableName} - expected {table_name}" ) - exit(-1) + sys.exit(-1) legacy_tables = hive_client.get_table_objects_by_name( dbname=db_name, tbl_names=[table_name] @@ -86,7 +86,7 @@ def check_legacy_get_table(hive_client, db_name, table_name): f"[ERROR]: Legacy get_table_objects_by_name returned " f"{[t.tableName for t in legacy_tables]} - expected {[table_name]}" ) - exit(-1) + sys.exit(-1) print("[INFO]: Metastore serves the legacy get_table Thrift methods") @@ -155,7 +155,7 @@ def check_legacy_get_table(hive_client, db_name, table_name): f"[ERROR]: Expected the stored location to keep the s3:// scheme, " f"got {s3_scheme_location}" ) - exit(-1) + sys.exit(-1) # Removed test, because it failed against Hive 3.1.3. We do not know if the behavior of the Hive metastore changed or we made a mistake. We improved the Trino tests to do more stuff with S3 (e.g. writing tables) which passed, # so we are confident that the removal of this test is ok @@ -165,13 +165,13 @@ def check_legacy_get_table(hive_client, db_name, table_name): # wrong_location = "s3a://wrongbucket/" # hive_client.create_table(table(database_name, s3_test_table_name_wrong_bucket, wrong_location)) # print(f"[ERROR]: Hive metastore created table {s3_test_table_name_wrong_bucket} in wrong location {wrong_location} which should have not been possible because the bucket didn't exist") - # exit(-1) + # sys.exit(-1) # except MetaException as ex: # if ex.message == 'Got exception: java.io.FileNotFoundException Bucket wrongbucket does not exist': # print(f"[SUCCESS]: Could not read from wrong bucket: {ex}") # else: # print(f"[ERROR]: Got error during creating table pointing to wrong bucket: {ex}") - # exit(-1) + # sys.exit(-1) print("[SUCCESS] Test finished successfully!") exit(0) diff --git a/tests/templates/kuttl/smoke/test_metastore_opa.py b/tests/templates/kuttl/smoke/80_test_metastore_opa.py similarity index 98% rename from tests/templates/kuttl/smoke/test_metastore_opa.py rename to tests/templates/kuttl/smoke/80_test_metastore_opa.py index bde8b6db..ec20b4ee 100755 --- a/tests/templates/kuttl/smoke/test_metastore_opa.py +++ b/tests/templates/kuttl/smoke/80_test_metastore_opa.py @@ -8,7 +8,7 @@ TableBuilder, ) import argparse - +import sys def table(db_name, table_name, location): columns = [ColumnBuilder("id", "string", "col comment").build()] @@ -72,4 +72,4 @@ def table(db_name, table_name, location): print( f"[ERROR] Test hive-metastore-opa-authorizer failed. Could access database '{database_name}'!" ) - exit(-1) + sys.exit(-1) diff --git a/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 b/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 index aa7e4108..c371caa0 100644 --- a/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 @@ -6,5 +6,5 @@ commands: helm install postgresql --version={{ test_scenario['values']['postgres'] }} --namespace "$NAMESPACE" - -f helm-bitnami-postgresql-values.yaml + -f 20_helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/upgrade/20_helm-bitnami-postgresql-values.yaml.j2 similarity index 57% rename from tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 rename to tests/templates/kuttl/upgrade/20_helm-bitnami-postgresql-values.yaml.j2 index 9e821599..de75603e 100644 --- a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 +++ b/tests/templates/kuttl/upgrade/20_helm-bitnami-postgresql-values.yaml.j2 @@ -1,10 +1,25 @@ --- +global: + security: + allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images + +image: + repository: bitnamilegacy/postgresql + volumePermissions: enabled: false + image: + repository: bitnamilegacy/os-shell securityContext: runAsUser: auto +metrics: + image: + repository: bitnamilegacy/postgres-exporter + primary: + extendedConfiguration: | + password_encryption=md5 podSecurityContext: {% if test_scenario['values']['openshift'] == 'true' %} enabled: false @@ -21,10 +36,6 @@ primary: memory: "128Mi" cpu: "1" -shmVolume: - chmod: - enabled: false - auth: username: hive password: hive diff --git a/tests/templates/kuttl/upgrade/30-install-hive.yaml.j2 b/tests/templates/kuttl/upgrade/30-install-hive.yaml.j2 index 1418b147..52154e15 100644 --- a/tests/templates/kuttl/upgrade/30-install-hive.yaml.j2 +++ b/tests/templates/kuttl/upgrade/30-install-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive @@ -18,6 +18,7 @@ spec: host: postgresql database: hive credentialsSecretName: hive-credentials + warehouseDir: s3a://my-bucket/my-warehouse/ {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/upgrade/31-upgrade-hive.yaml.j2 b/tests/templates/kuttl/upgrade/31-upgrade-hive.yaml.j2 index a3601aa3..d65b0ef9 100644 --- a/tests/templates/kuttl/upgrade/31-upgrade-hive.yaml.j2 +++ b/tests/templates/kuttl/upgrade/31-upgrade-hive.yaml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: hive.stackable.tech/v1alpha1 +apiVersion: hive.stackable.tech/v1alpha2 kind: HiveCluster metadata: name: hive diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 90f31a83..b6dc3b91 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -38,6 +38,12 @@ dimensions: - name: hdfs-latest values: - 3.5.0 + - name: trino-latest + values: + - "483" + # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version + # as in the example below. + # - 483,oci.stackable.tech/sdp/trino:483-stackable0.0.0-dev - name: zookeeper-latest values: - 3.9.6 @@ -119,6 +125,12 @@ tests: dimensions: - hive - openshift + - name: iceberg-rest-catalog + dimensions: + - postgres + - hive-latest + - trino-latest + - openshift suites: - name: nightly patch: