From a42e43ea1f25fa513f4aeccee8571d91085a75c0 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 17 Nov 2026 11:35:38 +0100 Subject: [PATCH 1/7] fix: Time out the ZooKeeper connect in the znode controller --- CHANGELOG.md | 5 ++++ rust/operator-binary/src/znode_controller.rs | 30 ++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f676f12f..f1e641d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,10 @@ All notable changes to this project will be documented in this file. `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is rotated hourly and at most two files are kept, because rotation is opt-in ([#1106]). +- Deleting a Namespace is no longer delayed by the znode finalizer for the length of the kernel's + TCP SYN retry budget. The ZooKeeper connection used to clean up a znode had no timeout, so once + the ensemble's Service was removed the connect blocked for over two minutes; it now gives up + after 10 seconds and is retried ([#1109]). [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 @@ -76,6 +80,7 @@ All notable changes to this project will be documented in this file. [#1102]: https://github.com/stackabletech/zookeeper-operator/pull/1102 [#1105]: https://github.com/stackabletech/zookeeper-operator/pull/1105 [#1106]: https://github.com/stackabletech/zookeeper-operator/pull/1106 +[#1109]: https://github.com/stackabletech/zookeeper-operator/pull/1109 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index decaf06a..61e9ac86 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -441,10 +441,10 @@ mod tests { } mod znode_mgmt { - use std::{collections::VecDeque, net::SocketAddr}; + use std::{collections::VecDeque, net::SocketAddr, time::Duration}; use snafu::{OptionExt, ResultExt, Snafu}; - use tokio::net::lookup_host; + use tokio::{net::lookup_host, time::timeout}; use tokio_zookeeper::{Acl, Permission, ZooKeeper}; #[derive(Snafu, Debug)] @@ -461,6 +461,12 @@ mod znode_mgmt { source: tokio_zookeeper::error::Error, addr: SocketAddr, }, + #[snafu(display("failed to connect to {addr:?} within {timeout:?}"))] + ConnectTimeout { + source: tokio::time::error::Elapsed, + addr: SocketAddr, + timeout: Duration, + }, #[snafu(display("protocol error creating znode {path:?}"))] CreateZnodeProtocol { source: tokio_zookeeper::error::Error, @@ -488,6 +494,20 @@ mod znode_mgmt { }, } + /// How long to wait for a connection to the ZooKeeper ensemble before giving up. + /// + /// [`ZooKeeper::connect`] applies no deadline of its own, so without this the TCP connect falls + /// back to the kernel's SYN retry budget, which is over two minutes on Linux. That matters most + /// while a Namespace is being deleted: its contents are removed concurrently and in no + /// particular order, so the ensemble's Service and Pods are typically gone before the + /// ZookeeperCluster is. The znode cleanup then dials a ClusterIP that no longer routes + /// anywhere and silently drops traffic, and the znode finalizer blocks Namespace termination + /// for the whole retry budget. + /// + /// Timing out instead lets the reconcile fail fast and be retried by [`error_policy`][super::error_policy], by + /// which point the ZookeeperCluster is usually gone too and the cleanup is skipped entirely. + const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + async fn connect(addr: &str) -> Result { tracing::debug!(addr, "Connecting to ZooKeeper"); // TODO: Happy eyeballs? @@ -496,8 +516,12 @@ mod znode_mgmt { .context(InvalidAddrSnafu { addr })? .next() .context(AddrResolutionSnafu { addr })?; - let (zk, _) = ZooKeeper::connect(&addr) + let (zk, _) = timeout(CONNECT_TIMEOUT, ZooKeeper::connect(&addr)) .await + .context(ConnectTimeoutSnafu { + addr, + timeout: CONNECT_TIMEOUT, + })? .context(ConnectSnafu { addr })?; tracing::debug!("Connected to ZooKeeper"); Ok(zk) From cd48edd1d18aa4c284cf662675073908a69a0d39 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Nov 2026 09:51:35 +0100 Subject: [PATCH 2/7] fix: Time out the ZooKeeper requests in the znode controller --- CHANGELOG.md | 8 ++- rust/operator-binary/src/znode_controller.rs | 75 ++++++++++++++++---- 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e641d4..2fc7414c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,11 @@ All notable changes to this project will be documented in this file. moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is rotated hourly and at most two files are kept, because rotation is opt-in ([#1106]). - Deleting a Namespace is no longer delayed by the znode finalizer for the length of the kernel's - TCP SYN retry budget. The ZooKeeper connection used to clean up a znode had no timeout, so once - the ensemble's Service was removed the connect blocked for over two minutes; it now gives up - after 10 seconds and is retried ([#1109]). + TCP retransmission budget. Neither the ZooKeeper connection used to clean up a znode nor the + requests sent over it had a timeout, so the cleanup blocked for over two minutes when the + ensemble's Service was already removed, and for around 15 minutes when the ensemble's Pods + disappeared while the znode subtree was being walked. Both now give up after 10 seconds and are + retried ([#1109]). [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 61e9ac86..85e07cf3 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -441,7 +441,7 @@ mod tests { } mod znode_mgmt { - use std::{collections::VecDeque, net::SocketAddr, time::Duration}; + use std::{collections::VecDeque, future::Future, net::SocketAddr, time::Duration}; use snafu::{OptionExt, ResultExt, Snafu}; use tokio::{net::lookup_host, time::timeout}; @@ -467,6 +467,15 @@ mod znode_mgmt { addr: SocketAddr, timeout: Duration, }, + #[snafu(display( + "{request} request for znode {path:?} was not answered within {timeout:?}" + ))] + RequestTimeout { + source: tokio::time::error::Elapsed, + request: &'static str, + path: String, + timeout: Duration, + }, #[snafu(display("protocol error creating znode {path:?}"))] CreateZnodeProtocol { source: tokio_zookeeper::error::Error, @@ -508,6 +517,41 @@ mod znode_mgmt { /// which point the ZookeeperCluster is usually gone too and the cleanup is skipped entirely. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + /// How long to wait for a single request to an established ZooKeeper connection before giving + /// up. + /// + /// A successful connect is no guarantee that the ensemble is still reachable for the rest of + /// the operation, and the requests of [`ZooKeeper`] apply no deadline of their own either. + /// Deleting a znode walks its subtree one request at a time, so the ensemble's Pods can + /// disappear midway through. Because they go away without closing the TCP connection, the + /// in-flight request is neither answered nor failed until the kernel exhausts its + /// retransmission budget, which takes roughly 15 minutes with the default `tcp_retries2` on + /// Linux. A Namespace deletion was observed to be held up for 957 seconds by exactly this, + /// well past the point where the znode had become unreachable for good. + /// + /// The deletion is resumable, so timing out costs nothing but a retry: it is retried by + /// [`error_policy`][super::error_policy], and the retry either makes further progress or finds + /// the ZookeeperCluster gone and skips the cleanup entirely. + const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + + /// Applies [`REQUEST_TIMEOUT`] to a single ZooKeeper request. + /// + /// The request's own result is passed through untouched so that callers keep attaching their + /// own error context to it. + async fn with_request_timeout( + request: &'static str, + path: &str, + zk_request: impl Future, + ) -> Result { + timeout(REQUEST_TIMEOUT, zk_request) + .await + .context(RequestTimeoutSnafu { + request, + path, + timeout: REQUEST_TIMEOUT, + }) + } + async fn connect(addr: &str) -> Result { tracing::debug!(addr, "Connecting to ZooKeeper"); // TODO: Happy eyeballs? @@ -532,8 +576,10 @@ mod znode_mgmt { pub async fn ensure_znode_exists(addr: &str, path: &str) -> Result<(), Error> { tracing::info!(znode = path, "Creating ZNode"); let zk = connect(addr).await?; - let create_res = zk - .create( + let create_res = with_request_timeout( + "create", + path, + zk.create( path, vec![], vec![Acl { @@ -542,9 +588,10 @@ mod znode_mgmt { id: "anyone".to_string(), }], tokio_zookeeper::CreateMode::Persistent, - ) - .await - .context(CreateZnodeProtocolSnafu { path })?; + ), + ) + .await? + .context(CreateZnodeProtocolSnafu { path })?; match create_res { Ok(_) => { tracing::info!(znode = "Created ZNode"); @@ -573,10 +620,10 @@ mod znode_mgmt { ?queue, "Deleting ZNode from queue" ); - let children = zk - .get_children(&curr_path) - .await - .context(DeleteZnodeFindChildrenProtocolSnafu { path: &curr_path })?; + let children = + with_request_timeout("get_children", &curr_path, zk.get_children(&curr_path)) + .await? + .context(DeleteZnodeFindChildrenProtocolSnafu { path: &curr_path })?; match children { None => { tracing::warn!( @@ -589,10 +636,10 @@ mod znode_mgmt { znode = curr_path.as_str(), "ZNode has no children, deleting..." ); - let delete_res = zk - .delete(&curr_path, None) - .await - .context(DeleteZnodeProtocolSnafu { path: &curr_path })?; + let delete_res = + with_request_timeout("delete", &curr_path, zk.delete(&curr_path, None)) + .await? + .context(DeleteZnodeProtocolSnafu { path: &curr_path })?; match delete_res { Ok(_) => tracing::info!(znode = curr_path.as_str(), "Deleted ZNode"), Err(tokio_zookeeper::error::Delete::NoNode) => tracing::info!( From 2db3a1711056e9d55f7ed04f57440ea2a110719c Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Nov 2026 12:46:15 +0100 Subject: [PATCH 3/7] fix: Time out address resolution and back off after ZooKeeper timeouts --- CHANGELOG.md | 6 +- rust/operator-binary/src/znode_controller.rs | 142 ++++++++++++++----- 2 files changed, 110 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fc7414c..fbba1e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,10 @@ All notable changes to this project will be documented in this file. TCP retransmission budget. Neither the ZooKeeper connection used to clean up a znode nor the requests sent over it had a timeout, so the cleanup blocked for over two minutes when the ensemble's Service was already removed, and for around 15 minutes when the ensemble's Pods - disappeared while the znode subtree was being walked. Both now give up after 10 seconds and are - retried ([#1109]). + disappeared while the znode subtree was being walked. Address resolution, the connection and + every request now give up after 10 seconds and are retried, on the znode creation path as well as + the cleanup path. Because a timed-out connection lingers until the operating system reaps it, + such a reconcile is retried after 60 seconds rather than the usual 5 ([#1109]). [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 85e07cf3..d9a54fd7 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -99,6 +99,21 @@ pub enum Error { type Result = std::result::Result; impl Error { + /// Whether the reconcile gave up waiting for the ZooKeeper ensemble to answer. + /// + /// Both the apply and the cleanup step talk to the ensemble, so either error can carry a + /// timeout. + fn ensemble_timed_out(&self) -> bool { + let zookeeper_error = match self { + Error::EnsureZnodeMissing { source, .. } => source, + Error::ApplyResources { + source: apply::Error::EnsureZnode { source, .. }, + } => source, + _ => return false, + }; + zookeeper_error.timed_out() + } + fn extract_finalizer_err(err: finalizer::Error) -> Self { match err { finalizer::Error::ApplyFailed(source) => source, @@ -327,12 +342,31 @@ fn zk_mgmt_addr( ) } +/// How long to wait before retrying a reconcile that gave up waiting for the ZooKeeper ensemble. +/// +/// A timeout unblocks the reconcile but leaves the connection behind until the kernel reaps it, up +/// to ~15 minutes later (see `REQUEST_TIMEOUT` in [`znode_mgmt`]). On the usual +/// [`DEFAULT_REQUEUE`] interval those abandoned sessions stack up against an ensemble that accepts +/// connections but cannot answer in time - around 60 per ZookeeperZnode, which is also ZooKeeper's +/// default `maxClientCnxns` per client IP, past which it refuses every further connection from the +/// operator. A minute keeps a handful in flight instead, while still recovering promptly from a +/// transient blip such as a leader election. +const UNRESPONSIVE_ENSEMBLE_REQUEUE: Duration = Duration::from_secs(60); + +/// How long to wait before retrying a reconcile that failed for any other reason. +const DEFAULT_REQUEUE: Duration = Duration::from_secs(5); + pub fn error_policy( _obj: Arc>, - _error: &Error, + error: &Error, _ctx: Arc, ) -> controller::Action { - controller::Action::requeue(*Duration::from_secs(5)) + let requeue_after = if error.ensemble_timed_out() { + UNRESPONSIVE_ENSEMBLE_REQUEUE + } else { + DEFAULT_REQUEUE + }; + controller::Action::requeue(*requeue_after) } /// Shared helpers for building validated test znodes from minimal YAML fixtures. @@ -456,6 +490,12 @@ mod znode_mgmt { }, #[snafu(display("address {addr:?} did not resolve to any socket addresses"))] AddrResolution { addr: String }, + #[snafu(display("failed to resolve {addr:?} within {timeout:?}"))] + ResolveTimeout { + source: tokio::time::error::Elapsed, + addr: String, + timeout: Duration, + }, #[snafu(display("failed to connect to {addr:?}"))] Connect { source: tokio_zookeeper::error::Error, @@ -503,41 +543,70 @@ mod znode_mgmt { }, } + impl Error { + /// Whether the ensemble ran out of time to answer, rather than answering with an error. + /// + /// These leave a blocking resolver call or a ZooKeeper session behind, so they must be + /// retried more patiently than a prompt refusal; see [`REQUEST_TIMEOUT`]. + pub fn timed_out(&self) -> bool { + matches!( + self, + Error::ResolveTimeout { .. } + | Error::ConnectTimeout { .. } + | Error::RequestTimeout { .. } + ) + } + } + + /// How long to wait for the ensemble's address to resolve before giving up. + /// + /// [`lookup_host`] applies no deadline of its own, so an unresponsive CoreDNS costs + /// `timeout × attempts × search domains` before the resolver gives up. A deleted Service is not + /// the cause - that answers NXDOMAIN immediately - an unreachable or overloaded one is. + /// + /// [`lookup_host`] resolves on a blocking thread, so timing out unblocks the reconcile without + /// aborting the resolver call itself. + const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10); + /// How long to wait for a connection to the ZooKeeper ensemble before giving up. /// /// [`ZooKeeper::connect`] applies no deadline of its own, so without this the TCP connect falls - /// back to the kernel's SYN retry budget, which is over two minutes on Linux. That matters most - /// while a Namespace is being deleted: its contents are removed concurrently and in no - /// particular order, so the ensemble's Service and Pods are typically gone before the - /// ZookeeperCluster is. The znode cleanup then dials a ClusterIP that no longer routes - /// anywhere and silently drops traffic, and the znode finalizer blocks Namespace termination - /// for the whole retry budget. + /// back to the kernel's SYN retry budget, over two minutes on Linux. That matters most while a + /// Namespace is being deleted: its contents go in no particular order, so the ensemble's + /// Service and Pods are typically gone before the ZookeeperCluster is. The cleanup then dials a + /// ClusterIP that no longer routes anywhere, so the SYNs are dropped rather than refused and + /// the znode finalizer blocks Namespace termination for the whole budget. /// - /// Timing out instead lets the reconcile fail fast and be retried by [`error_policy`][super::error_policy], by - /// which point the ZookeeperCluster is usually gone too and the cleanup is skipped entirely. + /// Timing out instead lets the reconcile fail fast and be retried by + /// [`error_policy`][super::error_policy], by which point the ZookeeperCluster is usually gone + /// too and the cleanup is skipped entirely. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); - /// How long to wait for a single request to an established ZooKeeper connection before giving - /// up. + /// How long to wait for one request on an established ZooKeeper connection before giving up. /// /// A successful connect is no guarantee that the ensemble is still reachable for the rest of /// the operation, and the requests of [`ZooKeeper`] apply no deadline of their own either. /// Deleting a znode walks its subtree one request at a time, so the ensemble's Pods can /// disappear midway through. Because they go away without closing the TCP connection, the /// in-flight request is neither answered nor failed until the kernel exhausts its - /// retransmission budget, which takes roughly 15 minutes with the default `tcp_retries2` on - /// Linux. A Namespace deletion was observed to be held up for 957 seconds by exactly this, - /// well past the point where the znode had become unreachable for good. + /// retransmission budget, roughly 15 minutes with the default `tcp_retries2` on Linux. A + /// Namespace deletion was observed to be held up for 957 seconds by exactly this, well past + /// the point where the znode had become unreachable for good. /// - /// The deletion is resumable, so timing out costs nothing but a retry: it is retried by - /// [`error_policy`][super::error_policy], and the retry either makes further progress or finds - /// the ZookeeperCluster gone and skips the cleanup entirely. + /// The deletion is resumable, so the retry from [`error_policy`][super::error_policy] either + /// makes further progress or finds the ZookeeperCluster gone and skips the cleanup entirely. + /// + /// The timeout only unblocks the reconcile, not the connection underneath it: [`ZooKeeper`] is + /// a handle to a task that `tokio-zookeeper` spawns, and dropping it makes that task queue a + /// `CloseSession` and wait for a close that costs the same retransmission budget again. + /// Retries must therefore be spaced out - see + /// [`UNRESPONSIVE_ENSEMBLE_REQUEUE`][super::UNRESPONSIVE_ENSEMBLE_REQUEUE]. const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); /// Applies [`REQUEST_TIMEOUT`] to a single ZooKeeper request. /// - /// The request's own result is passed through untouched so that callers keep attaching their - /// own error context to it. + /// The request's own result passes through untouched, so callers keep attaching their own + /// error context to it. async fn with_request_timeout( request: &'static str, path: &str, @@ -555,8 +624,12 @@ mod znode_mgmt { async fn connect(addr: &str) -> Result { tracing::debug!(addr, "Connecting to ZooKeeper"); // TODO: Happy eyeballs? - let addr = lookup_host(addr) + let addr = timeout(RESOLVE_TIMEOUT, lookup_host(addr)) .await + .context(ResolveTimeoutSnafu { + addr, + timeout: RESOLVE_TIMEOUT, + })? .context(InvalidAddrSnafu { addr })? .next() .context(AddrResolutionSnafu { addr })?; @@ -576,22 +649,19 @@ mod znode_mgmt { pub async fn ensure_znode_exists(addr: &str, path: &str) -> Result<(), Error> { tracing::info!(znode = path, "Creating ZNode"); let zk = connect(addr).await?; - let create_res = with_request_timeout( - "create", + let create_request = zk.create( path, - zk.create( - path, - vec![], - vec![Acl { - perms: Permission::ALL, - scheme: "world".to_string(), - id: "anyone".to_string(), - }], - tokio_zookeeper::CreateMode::Persistent, - ), - ) - .await? - .context(CreateZnodeProtocolSnafu { path })?; + vec![], + vec![Acl { + perms: Permission::ALL, + scheme: "world".to_string(), + id: "anyone".to_string(), + }], + tokio_zookeeper::CreateMode::Persistent, + ); + let create_res = with_request_timeout("create", path, create_request) + .await? + .context(CreateZnodeProtocolSnafu { path })?; match create_res { Ok(_) => { tracing::info!(znode = "Created ZNode"); From 1598da58ad23490b6bd893829054a0a3784723d9 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Nov 2026 12:58:06 +0100 Subject: [PATCH 4/7] fix: Back off only after request timeouts, not connect timeouts --- CHANGELOG.md | 4 +- rust/operator-binary/src/znode_controller.rs | 43 ++++++++++---------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbba1e3c..93124bb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,8 +61,8 @@ All notable changes to this project will be documented in this file. ensemble's Service was already removed, and for around 15 minutes when the ensemble's Pods disappeared while the znode subtree was being walked. Address resolution, the connection and every request now give up after 10 seconds and are retried, on the znode creation path as well as - the cleanup path. Because a timed-out connection lingers until the operating system reaps it, - such a reconcile is retried after 60 seconds rather than the usual 5 ([#1109]). + the cleanup path. Because a timed-out request leaves its ZooKeeper session to be reaped by the + operating system, that one case is retried after 60 seconds rather than the usual 5 ([#1109]). [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index d9a54fd7..db4ad48b 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -99,11 +99,10 @@ pub enum Error { type Result = std::result::Result; impl Error { - /// Whether the reconcile gave up waiting for the ZooKeeper ensemble to answer. + /// Whether the reconcile left an abandoned ZooKeeper session behind. /// - /// Both the apply and the cleanup step talk to the ensemble, so either error can carry a - /// timeout. - fn ensemble_timed_out(&self) -> bool { + /// Both the apply and the cleanup step talk to the ensemble, so either error can carry one. + fn abandoned_zookeeper_session(&self) -> bool { let zookeeper_error = match self { Error::EnsureZnodeMissing { source, .. } => source, Error::ApplyResources { @@ -111,7 +110,7 @@ impl Error { } => source, _ => return false, }; - zookeeper_error.timed_out() + zookeeper_error.abandoned_session() } fn extract_finalizer_err(err: finalizer::Error) -> Self { @@ -342,16 +341,19 @@ fn zk_mgmt_addr( ) } -/// How long to wait before retrying a reconcile that gave up waiting for the ZooKeeper ensemble. +/// How long to wait before retrying a reconcile that left a ZooKeeper session behind. /// -/// A timeout unblocks the reconcile but leaves the connection behind until the kernel reaps it, up -/// to ~15 minutes later (see `REQUEST_TIMEOUT` in [`znode_mgmt`]). On the usual +/// Timing out a request unblocks the reconcile but leaves the session behind until the kernel +/// reaps it, up to ~15 minutes later (see `REQUEST_TIMEOUT` in [`znode_mgmt`]). On the usual /// [`DEFAULT_REQUEUE`] interval those abandoned sessions stack up against an ensemble that accepts /// connections but cannot answer in time - around 60 per ZookeeperZnode, which is also ZooKeeper's /// default `maxClientCnxns` per client IP, past which it refuses every further connection from the /// operator. A minute keeps a handful in flight instead, while still recovering promptly from a /// transient blip such as a leader election. -const UNRESPONSIVE_ENSEMBLE_REQUEUE: Duration = Duration::from_secs(60); +/// +/// Only requests qualify. A connect that times out usually leaves nothing behind, and waiting a +/// minute on it would slow down the Namespace deletion this is all meant to unblock. +const ABANDONED_SESSION_REQUEUE: Duration = Duration::from_secs(60); /// How long to wait before retrying a reconcile that failed for any other reason. const DEFAULT_REQUEUE: Duration = Duration::from_secs(5); @@ -361,8 +363,8 @@ pub fn error_policy( error: &Error, _ctx: Arc, ) -> controller::Action { - let requeue_after = if error.ensemble_timed_out() { - UNRESPONSIVE_ENSEMBLE_REQUEUE + let requeue_after = if error.abandoned_zookeeper_session() { + ABANDONED_SESSION_REQUEUE } else { DEFAULT_REQUEUE }; @@ -544,17 +546,14 @@ mod znode_mgmt { } impl Error { - /// Whether the ensemble ran out of time to answer, rather than answering with an error. + /// Whether giving up left an established ZooKeeper session behind to be reaped. /// - /// These leave a blocking resolver call or a ZooKeeper session behind, so they must be - /// retried more patiently than a prompt refusal; see [`REQUEST_TIMEOUT`]. - pub fn timed_out(&self) -> bool { - matches!( - self, - Error::ResolveTimeout { .. } - | Error::ConnectTimeout { .. } - | Error::RequestTimeout { .. } - ) + /// Only a request can: the session is established by then, and abandoning it costs the + /// full retransmission budget (see [`REQUEST_TIMEOUT`]). A connect that times out is + /// nearly always still in [`ZooKeeper::connect`]'s `TcpStream::connect`, which has spawned + /// nothing yet, and resolving never reaches the ensemble at all. + pub fn abandoned_session(&self) -> bool { + matches!(self, Error::RequestTimeout { .. }) } } @@ -600,7 +599,7 @@ mod znode_mgmt { /// a handle to a task that `tokio-zookeeper` spawns, and dropping it makes that task queue a /// `CloseSession` and wait for a close that costs the same retransmission budget again. /// Retries must therefore be spaced out - see - /// [`UNRESPONSIVE_ENSEMBLE_REQUEUE`][super::UNRESPONSIVE_ENSEMBLE_REQUEUE]. + /// [`ABANDONED_SESSION_REQUEUE`][super::ABANDONED_SESSION_REQUEUE]. const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); /// Applies [`REQUEST_TIMEOUT`] to a single ZooKeeper request. From 8cbc7d34867314fbff297a4054f2b684e9f5da9a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Nov 2026 14:10:04 +0100 Subject: [PATCH 5/7] refactor: Fold the protocol error into the request helper --- CHANGELOG.md | 13 +-- rust/operator-binary/src/znode_controller.rs | 109 ++++++------------- 2 files changed, 38 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93124bb0..07fd80f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,14 +55,11 @@ All notable changes to this project will be documented in this file. `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is rotated hourly and at most two files are kept, because rotation is opt-in ([#1106]). -- Deleting a Namespace is no longer delayed by the znode finalizer for the length of the kernel's - TCP retransmission budget. Neither the ZooKeeper connection used to clean up a znode nor the - requests sent over it had a timeout, so the cleanup blocked for over two minutes when the - ensemble's Service was already removed, and for around 15 minutes when the ensemble's Pods - disappeared while the znode subtree was being walked. Address resolution, the connection and - every request now give up after 10 seconds and are retried, on the znode creation path as well as - the cleanup path. Because a timed-out request leaves its ZooKeeper session to be reaped by the - operating system, that one case is retried after 60 seconds rather than the usual 5 ([#1109]). +- Deleting a Namespace is no longer held up by the znode finalizer, which could block it for up to + 15 minutes when the ZooKeeper ensemble was already gone. Resolving the ensemble's address, + connecting to it and each request now give up after 10 seconds and are retried, instead of + waiting for the operating system to notice that the connection is dead. Creating a znode is + bounded the same way and can now fail with a timeout error ([#1109]). [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060 diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index db4ad48b..d5e3aa5f 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -343,16 +343,10 @@ fn zk_mgmt_addr( /// How long to wait before retrying a reconcile that left a ZooKeeper session behind. /// -/// Timing out a request unblocks the reconcile but leaves the session behind until the kernel -/// reaps it, up to ~15 minutes later (see `REQUEST_TIMEOUT` in [`znode_mgmt`]). On the usual -/// [`DEFAULT_REQUEUE`] interval those abandoned sessions stack up against an ensemble that accepts -/// connections but cannot answer in time - around 60 per ZookeeperZnode, which is also ZooKeeper's -/// default `maxClientCnxns` per client IP, past which it refuses every further connection from the -/// operator. A minute keeps a handful in flight instead, while still recovering promptly from a -/// transient blip such as a leader election. -/// -/// Only requests qualify. A connect that times out usually leaves nothing behind, and waiting a -/// minute on it would slow down the Namespace deletion this is all meant to unblock. +/// A request that times out leaves its connection open for another 15 minutes or so, because +/// closing it cleanly would need an answer from the ensemble that just stopped answering. Retrying +/// every 5 seconds would pile up around 60 of these per znode and hit ZooKeeper's per-client +/// connection limit (`maxClientCnxns`, 60 by default), so these retries wait a minute instead. const ABANDONED_SESSION_REQUEUE: Duration = Duration::from_secs(60); /// How long to wait before retrying a reconcile that failed for any other reason. @@ -518,9 +512,10 @@ mod znode_mgmt { path: String, timeout: Duration, }, - #[snafu(display("protocol error creating znode {path:?}"))] - CreateZnodeProtocol { + #[snafu(display("{request} request for znode {path:?} failed"))] + RequestProtocol { source: tokio_zookeeper::error::Error, + request: &'static str, path: String, }, #[snafu(display("failed to create znode {path:?}"))] @@ -528,30 +523,20 @@ mod znode_mgmt { source: tokio_zookeeper::error::Create, path: String, }, - #[snafu(display("protocol error deleting znode {path:?}"))] - DeleteZnodeProtocol { - source: tokio_zookeeper::error::Error, - path: String, - }, #[snafu(display("failed to delete znode {path:?}"))] DeleteZnode { source: tokio_zookeeper::error::Delete, path: String, }, - #[snafu(display("failed to find children to delete of {path:?}"))] - DeleteZnodeFindChildrenProtocol { - source: tokio_zookeeper::error::Error, - path: String, - }, } impl Error { - /// Whether giving up left an established ZooKeeper session behind to be reaped. + /// Whether giving up left a live ZooKeeper session behind. /// - /// Only a request can: the session is established by then, and abandoning it costs the - /// full retransmission budget (see [`REQUEST_TIMEOUT`]). A connect that times out is - /// nearly always still in [`ZooKeeper::connect`]'s `TcpStream::connect`, which has spawned - /// nothing yet, and resolving never reaches the ensemble at all. + /// Only a timed-out request can leave a session behind, because a session exists only + /// once the connection is established (see [`REQUEST_TIMEOUT`]). A connect that times out + /// usually has not opened a session yet, and a failed address lookup never reached the + /// ensemble at all. pub fn abandoned_session(&self) -> bool { matches!(self, Error::RequestTimeout { .. }) } @@ -559,57 +544,35 @@ mod znode_mgmt { /// How long to wait for the ensemble's address to resolve before giving up. /// - /// [`lookup_host`] applies no deadline of its own, so an unresponsive CoreDNS costs - /// `timeout × attempts × search domains` before the resolver gives up. A deleted Service is not - /// the cause - that answers NXDOMAIN immediately - an unreachable or overloaded one is. - /// - /// [`lookup_host`] resolves on a blocking thread, so timing out unblocks the reconcile without - /// aborting the resolver call itself. + /// DNS lookups have no deadline of their own and can hang for a long time when CoreDNS is + /// unreachable. The lookup itself keeps running in the background; the timeout only frees up + /// the reconcile. const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10); /// How long to wait for a connection to the ZooKeeper ensemble before giving up. /// - /// [`ZooKeeper::connect`] applies no deadline of its own, so without this the TCP connect falls - /// back to the kernel's SYN retry budget, over two minutes on Linux. That matters most while a - /// Namespace is being deleted: its contents go in no particular order, so the ensemble's - /// Service and Pods are typically gone before the ZookeeperCluster is. The cleanup then dials a - /// ClusterIP that no longer routes anywhere, so the SYNs are dropped rather than refused and - /// the znode finalizer blocks Namespace termination for the whole budget. - /// - /// Timing out instead lets the reconcile fail fast and be retried by - /// [`error_policy`][super::error_policy], by which point the ZookeeperCluster is usually gone - /// too and the cleanup is skipped entirely. + /// Connecting has no deadline of its own, and when the ensemble's Service is already gone the + /// attempt is silently dropped rather than refused, so the kernel keeps retrying for over two + /// minutes. That is long enough to hold up the deletion of an entire Namespace. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// How long to wait for one request on an established ZooKeeper connection before giving up. /// - /// A successful connect is no guarantee that the ensemble is still reachable for the rest of - /// the operation, and the requests of [`ZooKeeper`] apply no deadline of their own either. - /// Deleting a znode walks its subtree one request at a time, so the ensemble's Pods can - /// disappear midway through. Because they go away without closing the TCP connection, the - /// in-flight request is neither answered nor failed until the kernel exhausts its - /// retransmission budget, roughly 15 minutes with the default `tcp_retries2` on Linux. A - /// Namespace deletion was observed to be held up for 957 seconds by exactly this, well past - /// the point where the znode had become unreachable for good. - /// - /// The deletion is resumable, so the retry from [`error_policy`][super::error_policy] either - /// makes further progress or finds the ZookeeperCluster gone and skips the cleanup entirely. - /// - /// The timeout only unblocks the reconcile, not the connection underneath it: [`ZooKeeper`] is - /// a handle to a task that `tokio-zookeeper` spawns, and dropping it makes that task queue a - /// `CloseSession` and wait for a close that costs the same retransmission budget again. - /// Retries must therefore be spaced out - see - /// [`ABANDONED_SESSION_REQUEUE`][super::ABANDONED_SESSION_REQUEUE]. + /// Requests have no deadline of their own either, so if the ensemble's Pods vanish while a + /// znode is being deleted, the request simply never comes back and the kernel only gives up on + /// the dead connection after about 15 minutes. One Namespace deletion was stuck like this for + /// 957 seconds. const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); - /// Applies [`REQUEST_TIMEOUT`] to a single ZooKeeper request. + /// Sends a single ZooKeeper request, bounded by [`REQUEST_TIMEOUT`]. /// - /// The request's own result passes through untouched, so callers keep attaching their own - /// error context to it. - async fn with_request_timeout( + /// Both ways the connection itself can fail the request - the timeout and a protocol error - + /// are turned into an [`Error`] here, so callers are left with only ZooKeeper's own answer to + /// match on. + async fn request( request: &'static str, path: &str, - zk_request: impl Future, + zk_request: impl Future>, ) -> Result { timeout(REQUEST_TIMEOUT, zk_request) .await @@ -617,7 +580,8 @@ mod znode_mgmt { request, path, timeout: REQUEST_TIMEOUT, - }) + })? + .context(RequestProtocolSnafu { request, path }) } async fn connect(addr: &str) -> Result { @@ -658,9 +622,7 @@ mod znode_mgmt { }], tokio_zookeeper::CreateMode::Persistent, ); - let create_res = with_request_timeout("create", path, create_request) - .await? - .context(CreateZnodeProtocolSnafu { path })?; + let create_res = request("create", path, create_request).await?; match create_res { Ok(_) => { tracing::info!(znode = "Created ZNode"); @@ -689,10 +651,7 @@ mod znode_mgmt { ?queue, "Deleting ZNode from queue" ); - let children = - with_request_timeout("get_children", &curr_path, zk.get_children(&curr_path)) - .await? - .context(DeleteZnodeFindChildrenProtocolSnafu { path: &curr_path })?; + let children = request("get_children", &curr_path, zk.get_children(&curr_path)).await?; match children { None => { tracing::warn!( @@ -706,9 +665,7 @@ mod znode_mgmt { "ZNode has no children, deleting..." ); let delete_res = - with_request_timeout("delete", &curr_path, zk.delete(&curr_path, None)) - .await? - .context(DeleteZnodeProtocolSnafu { path: &curr_path })?; + request("delete", &curr_path, zk.delete(&curr_path, None)).await?; match delete_res { Ok(_) => tracing::info!(znode = curr_path.as_str(), "Deleted ZNode"), Err(tokio_zookeeper::error::Delete::NoNode) => tracing::info!( From c3e1b82253a457f7670ab627fdd3270061ca9227 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Nov 2026 15:49:21 +0100 Subject: [PATCH 6/7] chore: Improve the comment of ABANDONED_SESSION_REQUEUE --- rust/operator-binary/src/znode_controller.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index d5e3aa5f..6777e300 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -343,10 +343,11 @@ fn zk_mgmt_addr( /// How long to wait before retrying a reconcile that left a ZooKeeper session behind. /// -/// A request that times out leaves its connection open for another 15 minutes or so, because -/// closing it cleanly would need an answer from the ensemble that just stopped answering. Retrying -/// every 5 seconds would pile up around 60 of these per znode and hit ZooKeeper's per-client -/// connection limit (`maxClientCnxns`, 60 by default), so these retries wait a minute instead. +/// A request that times out does not close its connection, because closing it cleanly would need +/// an answer from the ensemble that just stopped answering. The connection therefore lingers, +/// until either the ensemble gets round to the close or the operating system gives up on a dead +/// one. Retrying every 5 seconds would keep stacking these up on an ensemble that is already +/// struggling, so these retries wait a minute instead. const ABANDONED_SESSION_REQUEUE: Duration = Duration::from_secs(60); /// How long to wait before retrying a reconcile that failed for any other reason. From a373d005f896be7fadc24dcbf9770a1aa6fd17c5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 19 Nov 2026 09:57:06 +0100 Subject: [PATCH 7/7] chore: Tighten a comment --- rust/operator-binary/src/znode_controller.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 6777e300..2026e3c8 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -561,8 +561,7 @@ mod znode_mgmt { /// /// Requests have no deadline of their own either, so if the ensemble's Pods vanish while a /// znode is being deleted, the request simply never comes back and the kernel only gives up on - /// the dead connection after about 15 minutes. One Namespace deletion was stuck like this for - /// 957 seconds. + /// the dead connection after about 15 minutes. const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); /// Sends a single ZooKeeper request, bounded by [`REQUEST_TIMEOUT`].