From 1911c22f71fcb7e1a07e719c9b3ece6b43db6490 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 09:19:05 -0800 Subject: [PATCH 1/8] feat: label replicas --- pgdog-config/src/database.rs | 4 + .../src/backend/passthrough_auth/check_db.rs | 4 +- pgdog/src/backend/pool/address.rs | 4 + pgdog/src/backend/pool/cluster.rs | 8 +- .../pool/connection/cluster_connection.rs | 10 +-- .../src/backend/pool/connection/mirror/mod.rs | 4 +- pgdog/src/backend/pool/connection/mod.rs | 19 ++-- .../pool/connection/multi_shard/upgrade.rs | 6 +- pgdog/src/backend/pool/connection/test/mod.rs | 14 +-- pgdog/src/backend/pool/lb/mod.rs | 12 +-- pgdog/src/backend/pool/lb/request.rs | 47 ++++++++++ pgdog/src/backend/pool/lb/test/mod.rs | 86 ++++++++++--------- pgdog/src/backend/pool/mod.rs | 2 +- pgdog/src/backend/pool/request.rs | 16 +--- pgdog/src/backend/pool/shard/mod.rs | 16 ++-- pgdog/src/backend/pool/shard/oids.rs | 4 +- .../replication/logical/resharding_state.rs | 8 +- .../replication/logical/schema_sync.rs | 11 ++- .../replication/logical/subscriber/copy.rs | 6 +- .../replication/logical/tables_sync.rs | 4 +- pgdog/src/backend/replication/tests.rs | 34 ++++---- .../backend/replication/tests/validation.rs | 8 +- pgdog/src/backend/schema/mod.rs | 4 +- pgdog/src/backend/schema/sync/pg_dump.rs | 4 +- pgdog/src/frontend/client/mod.rs | 4 +- .../frontend/client/query_engine/connect.rs | 8 +- pgdog/src/frontend/client/query_engine/mod.rs | 2 +- .../query_engine/test/lazy_connections.rs | 4 +- .../test/regression_auto_two_pc.rs | 4 +- .../test/regression_bind_transaction_end.rs | 4 +- .../regression_prepare_transaction_end.rs | 4 +- .../client/query_engine/two_pc/manager.rs | 4 +- .../client/query_engine/two_pc/test.rs | 10 +-- .../query_engine/two_pc/wal/tests/client.rs | 6 +- pgdog/src/frontend/router/comment.rs | 2 + .../src/frontend/router/parser/comment/mod.rs | 1 + .../router/parser/query/test/test_comments.rs | 2 +- 37 files changed, 233 insertions(+), 157 deletions(-) create mode 100644 pgdog/src/backend/pool/lb/request.rs diff --git a/pgdog-config/src/database.rs b/pgdog-config/src/database.rs index 9473f438f..b6918bb01 100644 --- a/pgdog-config/src/database.rs +++ b/pgdog-config/src/database.rs @@ -230,6 +230,10 @@ pub struct Database { /// Role-specific settings. #[serde(flatten, default)] pub role_config: RoleConfig, + + /// Labels. + #[serde(default)] + pub labels: Vec, } impl Database { diff --git a/pgdog/src/backend/passthrough_auth/check_db.rs b/pgdog/src/backend/passthrough_auth/check_db.rs index 15bcee653..f247da1e7 100644 --- a/pgdog/src/backend/passthrough_auth/check_db.rs +++ b/pgdog/src/backend/passthrough_auth/check_db.rs @@ -11,7 +11,7 @@ use tokio::sync::OnceCell; use tracing::error; use crate::backend::Cluster; -use crate::backend::pool::Request; +use crate::backend::pool::LbRequest; use crate::backend::schema::SchemaCache; use crate::config::config; @@ -151,7 +151,7 @@ async fn check_password(user: &User, config: &Config) -> Result { .iter() .next() .expect("cluster to have at least one shard") - .primary_or_replica(&Request::default()), + .primary_or_replica(&LbRequest::default()), ) .await { diff --git a/pgdog/src/backend/pool/address.rs b/pgdog/src/backend/pool/address.rs index c753a9652..877427deb 100644 --- a/pgdog/src/backend/pool/address.rs +++ b/pgdog/src/backend/pool/address.rs @@ -49,6 +49,8 @@ pub(crate) struct Address { /// back to the `[general]` settings. #[serde(default)] pub(crate) tls: ServerTls, + /// Labels. + pub(crate) labels: Vec, } impl Address { @@ -92,6 +94,7 @@ impl Address { database_number, configured_role: database.role, tls: database.tls.clone(), + labels: database.labels.clone(), } } @@ -224,6 +227,7 @@ impl Address { database_number: 0, configured_role: Role::Primary, tls: ServerTls::default(), + labels: vec![], } } } diff --git a/pgdog/src/backend/pool/cluster.rs b/pgdog/src/backend/pool/cluster.rs index 25eb2b9fc..16f48e2b6 100644 --- a/pgdog/src/backend/pool/cluster.rs +++ b/pgdog/src/backend/pool/cluster.rs @@ -30,7 +30,7 @@ use crate::{ use super::{ Address, CanonicalOids, ClusterFailoverSignalWatcher, ClusterMetrics, Config, Error, Guard, - Request, Shard, ShardConfig, + LbRequest, Shard, ShardConfig, }; use crate::config::LoadBalancingStrategy; use launch::Readiness; @@ -456,7 +456,7 @@ impl Cluster { } /// Get a connection to a primary of the given shard. - pub(crate) async fn primary(&self, shard: usize, request: &Request) -> Result { + pub(crate) async fn primary(&self, shard: usize, request: &LbRequest) -> Result { #[allow(clippy::unnecessary_lazy_evaluations)] let shard = self .shards @@ -466,7 +466,7 @@ impl Cluster { } /// Get a connection to a replica of the given shard. - pub(crate) async fn replica(&self, shard: usize, request: &Request) -> Result { + pub(crate) async fn replica(&self, shard: usize, request: &LbRequest) -> Result { #[allow(clippy::unnecessary_lazy_evaluations)] let shard = self .shards @@ -777,7 +777,7 @@ impl Cluster { .shards .get(round_robin::next(self.shards.len().max(1))) .ok_or_else(|| crate::backend::pool::Error::NoDatabases)?; - let mut server = shard.primary_or_replica(&Request::default()).await?; + let mut server = shard.primary_or_replica(&LbRequest::default()).await?; server .fetch_all(ServerRequest::parameterized(query, params)) diff --git a/pgdog/src/backend/pool/connection/cluster_connection.rs b/pgdog/src/backend/pool/connection/cluster_connection.rs index 8c292229c..c9eed971e 100644 --- a/pgdog/src/backend/pool/connection/cluster_connection.rs +++ b/pgdog/src/backend/pool/connection/cluster_connection.rs @@ -3,7 +3,7 @@ use crate::{ backend::{ Cluster, databases, passthrough_auth, pool::{ - Error as PoolError, Guard, Request, + Error as PoolError, Guard, LbRequest, connection::mirror::{Mirror, MirrorHandler}, }, reload_notify, @@ -94,7 +94,7 @@ impl ClusterConnection { /// connect to replicas. pub(crate) async fn get_conns_for_shards( &mut self, - request: &Request, + request: &LbRequest, shards: &[usize], is_read: bool, ) -> Result, Error> { @@ -120,7 +120,7 @@ impl ClusterConnection { /// Get all connections necessary to serve the request. pub(super) async fn get_conns( &mut self, - request: &Request, + request: &LbRequest, shard: &Shard, is_read: bool, ) -> Result<(Vec, Vec), Error> { @@ -145,7 +145,7 @@ impl ClusterConnection { /// Get a connection from the cluster for the given `shard`. pub(super) async fn get_conn( &mut self, - request: &Request, + request: &LbRequest, shard: usize, is_read: bool, ) -> Result { @@ -171,7 +171,7 @@ impl ClusterConnection { async fn get_conn_internal( &self, - request: &Request, + request: &LbRequest, shard: usize, is_read: bool, ) -> Result { diff --git a/pgdog/src/backend/pool/connection/mirror/mod.rs b/pgdog/src/backend/pool/connection/mirror/mod.rs index e4f885544..8c9e6778e 100644 --- a/pgdog/src/backend/pool/connection/mirror/mod.rs +++ b/pgdog/src/backend/pool/connection/mirror/mod.rs @@ -172,7 +172,7 @@ mod test { use pgdog_config::MirrorConfig; use crate::{ - backend::{Error as BackendError, pool::Request}, + backend::{Error as BackendError, pool::LbRequest}, config::{self, PoolerMode, PreparedStatementsLevel}, net::{Parameter, Parameters, Query}, util::safe_timeout, @@ -241,7 +241,7 @@ mod test { let cluster = Cluster::new_test(&config()); cluster.launch(); let mut mirror = Mirror::spawn("pgdog", &cluster, None).unwrap(); - let mut conn = cluster.primary(0, &Request::default()).await.unwrap(); + let mut conn = cluster.primary(0, &LbRequest::default()).await.unwrap(); conn.execute("DROP TABLE IF EXISTS pgdog.test_mirror") .await .expect("clear the mirror fixture"); diff --git a/pgdog/src/backend/pool/connection/mod.rs b/pgdog/src/backend/pool/connection/mod.rs index b4f0d3468..3f8926f71 100644 --- a/pgdog/src/backend/pool/connection/mod.rs +++ b/pgdog/src/backend/pool/connection/mod.rs @@ -19,7 +19,7 @@ use crate::{ use super::{ super::{Error, pool::Guard}, - Address, Cluster, Request, + Address, Cluster, LbRequest, }; use std::{ @@ -152,7 +152,11 @@ impl Connection { } /// Create a server connection if one doesn't exist already. - pub(crate) async fn connect(&mut self, request: &Request, route: &Route) -> Result<(), Error> { + pub(crate) async fn connect( + &mut self, + request: &LbRequest, + route: &Route, + ) -> Result<(), Error> { self.ensure_connected(request, route).await?; Ok(()) @@ -173,7 +177,7 @@ impl Connection { } /// Make sure we have all required shard connections to serve the request. - async fn ensure_connected(&mut self, request: &Request, route: &Route) -> Result<(), Error> { + async fn ensure_connected(&mut self, request: &LbRequest, route: &Route) -> Result<(), Error> { if matches!( self.binding, Binding::NotConnected | Binding::Transaction(_) @@ -209,7 +213,7 @@ impl Connection { } /// Try to get a connection for the given route. - async fn connect_internal(&mut self, request: &Request, route: &Route) -> Result<(), Error> { + async fn connect_internal(&mut self, request: &LbRequest, route: &Route) -> Result<(), Error> { // Start a transaction on the server(s) if client started one. let (is_read, transaction_stmt) = if let Binding::Transaction(ref mut transaction) = self.binding { @@ -271,7 +275,10 @@ impl Connection { } /// Get server parameters. - pub(crate) async fn parameters(&mut self, request: &Request) -> Result, Error> { + pub(crate) async fn parameters( + &mut self, + request: &LbRequest, + ) -> Result, Error> { if matches!(self.binding, Binding::Admin(_)) { return Ok(ADMIN_PARAMETERS.clone()); } @@ -289,7 +296,7 @@ impl Connection { } } - async fn try_parameters(&mut self, request: &Request) -> Result, Error> { + async fn try_parameters(&mut self, request: &LbRequest) -> Result, Error> { // Get params from the first database that answers. // Parameters are cached on the pool. for shard in self.cluster()?.shards() { diff --git a/pgdog/src/backend/pool/connection/multi_shard/upgrade.rs b/pgdog/src/backend/pool/connection/multi_shard/upgrade.rs index d242686ca..8352d56d2 100644 --- a/pgdog/src/backend/pool/connection/multi_shard/upgrade.rs +++ b/pgdog/src/backend/pool/connection/multi_shard/upgrade.rs @@ -24,7 +24,11 @@ impl<'a> MultiShardUpgrade<'a> { } /// Change the binding by connecting to required shards to serve the request. - pub(crate) async fn upgrade(&mut self, request: &Request, route: &Route) -> Result<(), Error> { + pub(crate) async fn upgrade( + &mut self, + request: &LbRequest, + route: &Route, + ) -> Result<(), Error> { // Don't switch to a replica or to a primary // between shard connection upgrades. let is_read = match self.connection.binding { diff --git a/pgdog/src/backend/pool/connection/test/mod.rs b/pgdog/src/backend/pool/connection/test/mod.rs index 52bec1c58..75c77c036 100644 --- a/pgdog/src/backend/pool/connection/test/mod.rs +++ b/pgdog/src/backend/pool/connection/test/mod.rs @@ -26,7 +26,7 @@ async fn two_pc_subset_commit_counts() { let pid = FrontendPid::new(); for shard in [1, 2] { connection - .connect(&Request::default(), &route(Shard::Direct(shard), false)) + .connect(&LbRequest::default(), &route(Shard::Direct(shard), false)) .await .unwrap(); connection @@ -58,7 +58,7 @@ async fn two_pc_missing_participants_not_counted() { let mut connection = test_connection(); connection - .connect(&Request::default(), &route(Shard::All, false)) + .connect(&LbRequest::default(), &route(Shard::All, false)) .await .unwrap(); connection @@ -90,7 +90,7 @@ async fn two_pc_recovery_subset() { let pid = FrontendPid::new(); for shard in [1, 2] { connection - .connect(&Request::default(), &route(Shard::Direct(shard), false)) + .connect(&LbRequest::default(), &route(Shard::Direct(shard), false)) .await .unwrap(); connection @@ -111,7 +111,7 @@ async fn two_pc_recovery_subset() { ) .unwrap(); recovery - .connect(&Request::default(), &route(Shard::All, false)) + .connect(&LbRequest::default(), &route(Shard::All, false)) .await .unwrap(); let recovered = recovery @@ -142,7 +142,7 @@ async fn test_connection_upgrade() { let params = ParameterStore::new(new_test("test_connection_connect_upgrade")); connection - .connect(&Request::default(), &route(Shard::Direct(0), true)) + .connect(&LbRequest::default(), &route(Shard::Direct(0), true)) .await .unwrap(); @@ -161,7 +161,7 @@ async fn test_connection_upgrade() { } connection - .connect(&Request::default(), &route(Shard::Direct(1), true)) + .connect(&LbRequest::default(), &route(Shard::Direct(1), true)) .await .unwrap(); assert_eq!(2, connection.connected_servers()); @@ -179,7 +179,7 @@ async fn test_connection_upgrade() { } connection - .connect(&Request::default(), &route(Shard::All, false)) + .connect(&LbRequest::default(), &route(Shard::All, false)) .await .unwrap(); assert_eq!(3, connection.connected_servers()); diff --git a/pgdog/src/backend/pool/lb/mod.rs b/pgdog/src/backend/pool/lb/mod.rs index 7e377ba38..01103f894 100644 --- a/pgdog/src/backend/pool/lb/mod.rs +++ b/pgdog/src/backend/pool/lb/mod.rs @@ -20,16 +20,18 @@ use crate::{ net::Parameters, }; -use super::{Error, Guard, Oids, Pool, PoolConfig, PoolRole, Request}; +use super::{Error, Guard, Oids, Pool, PoolConfig, PoolRole}; use crate::util::safe_timeout; pub(crate) mod ban; pub(crate) mod monitor; +pub(crate) mod request; pub(crate) mod target_health; use ban::Ban; pub(crate) use ban::UnbanReason; use monitor::*; +pub(crate) use request::LbRequest; pub(crate) use target_health::*; #[cfg(test)] @@ -296,12 +298,12 @@ impl LoadBalancer { } /// Get a live connection from the pool. - pub(crate) async fn get(&self, request: &Request) -> Result { + pub(crate) async fn get(&self, request: &LbRequest) -> Result { self.get_internal(request).await } /// Get parameters from first non-banned connection pool. - pub(crate) async fn params(&self, request: &Request) -> Result<&Arc, Error> { + pub(crate) async fn params(&self, request: &LbRequest) -> Result<&Arc, Error> { if let Some(target) = self.targets.iter().find(|target| !target.ban.banned()) { return target.pool.params(request).await; } @@ -407,7 +409,7 @@ impl LoadBalancer { /// /// In automatic mode, the caller waits for an election. Static /// replica-only configurations fail immediately with [`Error::NoPrimary`]. - pub(super) async fn get_primary(&self, request: &Request) -> Result { + pub(super) async fn get_primary(&self, request: &LbRequest) -> Result { if let Some(pool) = self.primary() { return pool.get(request).await; } @@ -420,7 +422,7 @@ impl LoadBalancer { } /// Check out a read connection using the configured load balancing strategy. - async fn get_internal(&self, request: &Request) -> Result { + async fn get_internal(&self, request: &LbRequest) -> Result { use LoadBalancingStrategy::*; use smallvec::SmallVec; diff --git a/pgdog/src/backend/pool/lb/request.rs b/pgdog/src/backend/pool/lb/request.rs new file mode 100644 index 000000000..ff0cf1f79 --- /dev/null +++ b/pgdog/src/backend/pool/lb/request.rs @@ -0,0 +1,47 @@ +use std::ops::Deref; + +use crate::{backend::pool::Request as PoolRequest, frontend::router::Route, net::FrontendPid}; +use tokio::time::Instant; + +/// Pool checkout request with load balancing options. +#[derive(Debug, Clone, Default)] +pub(crate) struct LbRequest { + request: PoolRequest, + /// Whether the client is configured read-only. + pub(super) read_only: bool, + pub(super) labels: Vec, +} + +impl LbRequest { + /// Create new load balancer request. + pub(crate) fn new( + id: FrontendPid, + route: &Route, + labels: Vec, + read_only: bool, + created_at: Instant, + ) -> Self { + Self { + request: PoolRequest::new(id, route, created_at), + labels, + read_only, + } + } + + /// Create a request for a client before a query route is available. + pub(crate) fn unrouted(id: FrontendPid) -> Self { + Self { + request: PoolRequest::unrouted(id), + read_only: false, + labels: Vec::new(), + } + } +} + +impl Deref for LbRequest { + type Target = PoolRequest; + + fn deref(&self) -> &Self::Target { + &self.request + } +} diff --git a/pgdog/src/backend/pool/lb/test/mod.rs b/pgdog/src/backend/pool/lb/test/mod.rs index 9b467a416..d037cbeb1 100644 --- a/pgdog/src/backend/pool/lb/test/mod.rs +++ b/pgdog/src/backend/pool/lb/test/mod.rs @@ -3,7 +3,7 @@ use std::time::Duration; use tokio::task::yield_now; use tokio::time::{Instant, sleep, timeout}; -use crate::backend::pool::{Address, Config, Error, PoolConfig, Request}; +use crate::backend::pool::{Address, Config, Error, LbRequest, PoolConfig}; use crate::backend::replication::publisher::Lsn; use crate::config::{LoadBalancingStrategy, Role}; use crate::frontend::router::Route; @@ -95,7 +95,7 @@ impl TestLbBuilder<'_, S> { } async fn checkout_shares(lb: &LoadBalancer, checkouts: usize) -> Vec { - let request = Request::default(); + let request = LbRequest::default(); let mut shares = vec![0; lb.targets.len()]; for _ in 0..checkouts { let conn = lb.get(&request).await.unwrap(); @@ -144,7 +144,7 @@ async fn test_include_primary_if_replica_banned_only_primary() { .rw_split(ReadWriteSplit::IncludePrimaryIfReplicaBanned) .launch(); - let conn = lb.get(&Request::default()).await.unwrap(); + let conn = lb.get(&LbRequest::default()).await.unwrap(); drop(conn); lb.shutdown(); @@ -353,7 +353,7 @@ async fn test_replica_ban_clears_idle_connections() { let replicas = setup_test_replicas(); // Get a connection and return it to create idle connections - let request = Request::default(); + let request = LbRequest::default(); let conn = replicas.pools()[0] .get(&request) .await @@ -437,7 +437,7 @@ async fn test_read_write_split_exclude_primary() { .rw_split(ReadWriteSplit::ExcludePrimary) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Try getting connections multiple times and verify primary is never used let mut replica_ids = HashSet::new(); @@ -474,7 +474,7 @@ async fn test_lsn_filtering() { let mut route = Route::read(ShardWithPriority::default()); route.lsn = Some(caught_up); - let request = Request::new(Default::default(), &route, false, Instant::now()); + let request = LbRequest::new(Default::default(), &route, vec![], false, Instant::now()); // A replica exactly at the requested LSN is eligible; the lagging one is not. for _ in 0..4 { @@ -484,7 +484,7 @@ async fn test_lsn_filtering() { // Fall back to primary when every replica is behind the requested LSN. route.lsn = Some(ahead); - let request = Request::new(Default::default(), &route, false, Instant::now()); + let request = LbRequest::new(Default::default(), &route, vec![], false, Instant::now()); let conn = replicas.get(&request).await.expect("primary fallback"); assert_eq!(conn.pool.id(), primary_id); } @@ -509,7 +509,13 @@ async fn test_include_primary_if_replica_lag() { let mut route = Route::read(ShardWithPriority::default()); for lsn in [None, Some(200), Some(201)] { route.lsn = lsn; - let request = Request::new(Default::default(), &route, read_only, Instant::now()); + let request = LbRequest::new( + Default::default(), + &route, + vec![], + read_only, + Instant::now(), + ); // Round robin must never select the primary while an eligible replica exists. for _ in 0..4 { @@ -526,7 +532,7 @@ async fn test_include_primary_if_replica_lag() { replicas.shutdown(); } -/// Sets the `Request.read_only` (normally derived from `User.read_only`) to true, +/// Sets the `LbRequest.read_only` (normally derived from `User.read_only`) to true, /// then ensures that the primary is not chosen for reads. We have replicas available. #[tokio::test] async fn test_user_read_only_include_primary() { @@ -545,9 +551,10 @@ async fn test_user_read_only_include_primary() { .launch(); // Notice that we `read` to true and `read_only` to true. - let request = Request::new( + let request = LbRequest::new( Default::default(), &Route::read(ShardWithPriority::default()), + vec![], true, Instant::now(), ); @@ -570,7 +577,7 @@ async fn test_user_read_only_include_primary() { replicas.shutdown(); } -/// Sets the `Request.read_only` (normally derived from `User.read_only`) to true, +/// Sets the `LbRequest.read_only` (normally derived from `User.read_only`) to true, /// then ensures that the primary IS chosen for reads. We have no usable replicas. #[tokio::test] async fn test_user_read_only_include_primary_no_replicas() { @@ -591,9 +598,10 @@ async fn test_user_read_only_include_primary_no_replicas() { // `read_only` is set to true. This would usually override `IncludePrimary`, // but we have no replicas available. - let request = Request::new( + let request = LbRequest::new( Default::default(), &Route::read(ShardWithPriority::default()), + vec![], true, Instant::now(), ); @@ -629,7 +637,7 @@ async fn test_read_write_split_include_primary() { .configs(&replica_configs) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Try getting connections multiple times and verify both primary and replica can be used let mut used_pool_ids = HashSet::new(); @@ -681,7 +689,7 @@ async fn test_prefer_primary_optin_read_honors_read_write_split() { .rw_split(split) .launch(); - let request = Request::default(); + let request = LbRequest::default(); let mut ids = HashSet::new(); for _ in 0..50 { let conn = lb.get(&request).await.unwrap(); @@ -725,7 +733,7 @@ async fn test_read_write_split_exclude_primary_no_replicas() { .rw_split(ReadWriteSplit::ExcludePrimary) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Try getting connections multiple times and we have primary in the set let mut used_pool_ids = HashSet::new(); @@ -758,7 +766,7 @@ async fn test_read_write_split_exclude_primary_no_primary() { .rw_split(ReadWriteSplit::ExcludePrimary) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Should work normally with just replicas let mut replica_ids = HashSet::new(); @@ -793,7 +801,7 @@ async fn test_read_write_split_exclude_primary_all_replicas_banned() { .ban .ban(Error::ServerError, Duration::from_secs(60)); - let result = replicas.get(&Request::default()).await; + let result = replicas.get(&LbRequest::default()).await; assert!( matches!(result, Err(Error::AllReplicasDown)), "exclude_primary must not send reads to the primary when all replicas are banned" @@ -821,7 +829,7 @@ async fn test_read_write_split_exclude_primary_one_replica_banned() { .ban(Error::ServerError, Duration::from_secs(60)); let healthy_id = replicas.targets[1].pool.id(); - let request = Request::default(); + let request = LbRequest::default(); for _ in 0..20 { let conn = replicas.get(&request).await.unwrap(); assert_eq!(conn.pool.id(), healthy_id); @@ -840,7 +848,7 @@ async fn test_read_write_split_include_primary_no_primary() { let replicas = test_lb().configs(&replica_configs).launch(); - let request = Request::default(); + let request = LbRequest::default(); // Should work normally with just replicas let mut replica_ids = HashSet::new(); @@ -872,7 +880,7 @@ async fn test_read_write_split_with_banned_primary() { let primary_ban = &replicas.targets.last().unwrap().ban; primary_ban.ban(Error::ServerError, Duration::from_millis(1000)); - let request = Request::default(); + let request = LbRequest::default(); // Should only use replica even though primary inclusion is enabled let mut used_pool_ids = HashSet::new(); @@ -909,7 +917,7 @@ async fn test_read_write_split_with_banned_replicas() { let replica_ban = &replicas.targets[0].ban; replica_ban.ban(Error::ServerError, Duration::from_millis(1000)); - let request = Request::default(); + let request = LbRequest::default(); // Should only use primary since replica is banned let mut used_pool_ids = HashSet::new(); @@ -946,7 +954,7 @@ async fn test_prefer_primary_with_banned_replicas_falls_back_to_primary() { let replica_ban = &replicas.targets[0].ban; replica_ban.ban(Error::ServerError, Duration::from_millis(1000)); - let request = Request::default(); + let request = LbRequest::default(); let mut used_pool_ids = HashSet::new(); for _ in 0..10 { @@ -980,7 +988,7 @@ async fn test_read_write_split_exclude_primary_with_round_robin() { .rw_split(ReadWriteSplit::ExcludePrimary) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Collect pool IDs from multiple requests to verify round-robin behavior let mut pool_sequence = Vec::new(); @@ -1186,7 +1194,7 @@ async fn test_include_primary_if_replica_banned_no_bans() { .rw_split(ReadWriteSplit::IncludePrimaryIfReplicaBanned) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // When no replicas are banned, primary should NOT be used let mut used_pool_ids = HashSet::new(); @@ -1224,7 +1232,7 @@ async fn test_include_primary_if_replica_banned_with_ban() { let replica_ban = &replicas.targets[0].ban; replica_ban.ban(Error::ServerError, Duration::from_millis(1000)); - let request = Request::default(); + let request = LbRequest::default(); // When replica is banned, primary SHOULD be used let mut used_pool_ids = HashSet::new(); @@ -1513,7 +1521,7 @@ async fn test_auto_mode_waits_for_primary_election() { let lb = test_lb().configs(&[config]).build(); assert!(matches!( - lb.get_primary(&Request::default()).await, + lb.get_primary(&LbRequest::default()).await, Err(Error::CheckoutTimeout) )); } @@ -1531,7 +1539,7 @@ async fn test_auto_mode_primary_election_releases_writes() { assert!(election.redetect_roles()); }); - let primary = lb.get_primary(&Request::default()).await.unwrap(); + let primary = lb.get_primary(&LbRequest::default()).await.unwrap(); assert_eq!(primary.pool.addr().host, "127.0.0.1"); drop(primary); election.await.unwrap(); @@ -1553,7 +1561,7 @@ async fn test_auto_mode_waits_for_each_new_primary() { let spawned = { let lb = lb.clone(); - tokio::spawn(async move { lb.get_primary(&Request::default()).await }) + tokio::spawn(async move { lb.get_primary(&LbRequest::default()).await }) }; // assert that the background wait is in progress @@ -1579,7 +1587,7 @@ async fn test_auto_mode_waits_for_each_new_primary() { let spawned = { let lb = lb.clone(); - tokio::spawn(async move { lb.get_primary(&Request::default()).await }) + tokio::spawn(async move { lb.get_primary(&LbRequest::default()).await }) }; yield_now().await; @@ -1628,7 +1636,7 @@ async fn test_static_replica_only_does_not_wait_for_primary() { let started = Instant::now(); assert!(matches!( - lb.get_primary(&Request::default()).await, + lb.get_primary(&LbRequest::default()).await, Err(Error::NoPrimary) )); assert!(started.elapsed() < Duration::from_millis(100)); @@ -1709,7 +1717,7 @@ async fn test_weighted_round_robin_smooth_distribution() { .strategy(LoadBalancingStrategy::WeightedRoundRobin) .launch(); - let request = Request::default(); + let request = LbRequest::default(); let pool_a = lb.targets[0].pool.id(); let pool_b = lb.targets[1].pool.id(); @@ -1737,7 +1745,7 @@ async fn test_weighted_round_robin_equal_weights() { .strategy(LoadBalancingStrategy::WeightedRoundRobin) .launch(); - let request = Request::default(); + let request = LbRequest::default(); let pool_a = lb.targets[0].pool.id(); let pool_b = lb.targets[1].pool.id(); @@ -1843,7 +1851,7 @@ async fn test_get_returns_all_replicas_down_when_all_targets_banned() { .bans(&[0, 1]) .launch(); - let result = lb.get(&Request::default()).await; + let result = lb.get(&LbRequest::default()).await; assert!( matches!(result, Err(Error::AllReplicasDown)), @@ -1880,7 +1888,7 @@ async fn test_least_active_connections_prefers_pool_with_fewer_checked_out() { .strategy(LoadBalancingStrategy::LeastActiveConnections) .launch(); - let request = Request::default(); + let request = LbRequest::default(); // Get first connection and hold it let conn1 = replicas.get(&request).await.unwrap(); @@ -2406,7 +2414,7 @@ fn test_ban_check_default_threshold_clears_expired_ban_despite_high_lag() { async fn test_params_returns_params_from_non_banned_target() { let replicas = setup_test_replicas(); - let request = Request::default(); + let request = LbRequest::default(); let result = replicas.params(&request).await; assert!(result.is_ok(), "params() should succeed when targets exist"); @@ -2423,7 +2431,7 @@ async fn test_params_returns_all_replicas_down_when_all_banned() { target.ban.ban(Error::ServerError, Duration::from_secs(60)); } - let request = Request::default(); + let request = LbRequest::default(); let result = replicas.params(&request).await; assert!( @@ -2443,7 +2451,7 @@ async fn test_params_skips_banned_targets() { .ban .ban(Error::ServerError, Duration::from_secs(60)); - let request = Request::default(); + let request = LbRequest::default(); let result = replicas.params(&request).await; assert!( @@ -2458,7 +2466,7 @@ async fn test_params_skips_banned_targets() { async fn test_params_returns_all_replicas_down_when_empty() { let replicas = test_lb().configs(&[]).build(); - let request = Request::default(); + let request = LbRequest::default(); let result = replicas.params(&request).await; assert!( @@ -2572,7 +2580,7 @@ async fn test_get_returns_offline_when_all_targets_banned_and_offline() { .build(); replicas.shutdown(); - let result = replicas.get(&Request::default()).await; + let result = replicas.get(&LbRequest::default()).await; assert!(matches!(result, Err(Error::Offline))); } diff --git a/pgdog/src/backend/pool/mod.rs b/pgdog/src/backend/pool/mod.rs index e9dcf13c5..2668fe22b 100644 --- a/pgdog/src/backend/pool/mod.rs +++ b/pgdog/src/backend/pool/mod.rs @@ -38,7 +38,7 @@ pub(crate) use error::Error; pub(super) use failover_signal::ClusterFailoverSignalWatcher; pub(crate) use guard::Guard; pub(crate) use healthcheck::Healtcheck; -pub(crate) use lb::LoadBalancer; +pub(crate) use lb::{LbRequest, LoadBalancer}; pub(crate) use lsn_monitor::LsnStats; pub(crate) use monitor::Monitor; pub(crate) use password::Password; diff --git a/pgdog/src/backend/pool/request.rs b/pgdog/src/backend/pool/request.rs index 39e7c5e64..9ae0ac1d6 100644 --- a/pgdog/src/backend/pool/request.rs +++ b/pgdog/src/backend/pool/request.rs @@ -2,9 +2,7 @@ use tokio::time::Instant; use crate::{frontend::router::Route, net::messages::FrontendPid}; -/// Request for connection(s) from connection pool(s). -/// Contains all the necessary information for the load balancer / connection pool -/// to give the right connection to the client. +/// Request for a connection from a pool. #[derive(Clone, Debug, Copy)] pub(crate) struct Request { /// The id of the client. We use this to track @@ -16,9 +14,6 @@ pub(crate) struct Request { /// Is the query a read or a write? This is the query router decision, i.e., /// a read would typically be a SELECT. pub(crate) read: bool, - /// Load balancer uses this to determine if primary should be allowed to read. - /// Propagated from `User.read_only` setting. - pub(crate) read_only: bool, /// Minimum replica LSN the client wants the replica to have /// before allowing reads on it, in its 64-bit representation. pub(crate) lsn: Option, @@ -26,18 +21,12 @@ pub(crate) struct Request { impl Request { /// Create new request for client given a query route. - pub(crate) fn new( - id: FrontendPid, - route: &Route, - read_only: bool, - created_at: Instant, - ) -> Self { + pub(crate) fn new(id: FrontendPid, route: &Route, created_at: Instant) -> Self { Self { id, created_at, read: route.is_read(), lsn: route.lsn, - read_only, } } @@ -46,7 +35,6 @@ impl Request { id, created_at: Instant::now(), read: false, - read_only: false, lsn: None, } } diff --git a/pgdog/src/backend/pool/shard/mod.rs b/pgdog/src/backend/pool/shard/mod.rs index f093707cd..84924cd95 100644 --- a/pgdog/src/backend/pool/shard/mod.rs +++ b/pgdog/src/backend/pool/shard/mod.rs @@ -20,7 +20,7 @@ use crate::config::{LoadBalancingStrategy, ReadWriteSplit, Role}; use crate::net::Parameters; use crate::net::messages::FrontendPid; -use super::{Error, Guard, LoadBalancer, Pool, PoolConfig, Request}; +use super::{Error, Guard, LbRequest, LoadBalancer, Pool, PoolConfig}; pub(crate) mod failover_signal; pub(crate) mod monitor; @@ -79,18 +79,18 @@ impl Shard { } /// Get connection to the primary database. - pub(crate) async fn primary(&self, request: &Request) -> Result { + pub(crate) async fn primary(&self, request: &LbRequest) -> Result { self.lb.get_primary(request).await } /// Get connection to one of the replica databases, using the configured /// load balancing algorithm. - pub(crate) async fn replica(&self, request: &Request) -> Result { + pub(crate) async fn replica(&self, request: &LbRequest) -> Result { self.lb.get(request).await } /// Get connection to primary if configured, otherwise replica. - pub(crate) async fn primary_or_replica(&self, request: &Request) -> Result { + pub(crate) async fn primary_or_replica(&self, request: &LbRequest) -> Result { match self.primary(request).await { Ok(primary) => Ok(primary), _ => self.replica(request).await, @@ -145,7 +145,7 @@ impl Shard { /// Fetch schema from the shard. This does not use the /// cache and returns the freshest schema available. pub(crate) async fn fetch_schema(&self) -> Result { - let mut server = self.primary_or_replica(&Request::default()).await?; + let mut server = self.primary_or_replica(&LbRequest::default()).await?; let schema = Schema::load(&mut server).await?; info!( @@ -284,7 +284,7 @@ impl Shard { } /// Get parameters from first available connection pool. - pub(crate) async fn params(&self, request: &Request) -> Result<&Arc, Error> { + pub(crate) async fn params(&self, request: &LbRequest) -> Result<&Arc, Error> { self.lb.params(request).await } @@ -432,7 +432,7 @@ mod test { for _ in 0..25 { let replica_id = shard.lb.targets[0].pool.id(); - let conn = shard.replica(&Request::default()).await.unwrap(); + let conn = shard.replica(&LbRequest::default()).await.unwrap(); assert_eq!(conn.pool.id(), replica_id); } @@ -468,7 +468,7 @@ mod test { let mut ids = BTreeSet::new(); for _ in 0..25 { - let conn = shard.replica(&Request::default()).await.unwrap(); + let conn = shard.replica(&LbRequest::default()).await.unwrap(); ids.insert(conn.pool.id()); } diff --git a/pgdog/src/backend/pool/shard/oids.rs b/pgdog/src/backend/pool/shard/oids.rs index ad916dddc..040eb16a6 100644 --- a/pgdog/src/backend/pool/shard/oids.rs +++ b/pgdog/src/backend/pool/shard/oids.rs @@ -1,4 +1,4 @@ -use super::{Request, Shard}; +use super::{LbRequest, Shard}; use crate::{ backend::{Error, Server}, net::DataRow, @@ -26,7 +26,7 @@ impl Oids { pub(crate) async fn load(&self, shard: &Shard) -> Result<&OidMappings, Error> { self.mappings .get_or_try_init(|| async { - let mut server = shard.primary_or_replica(&Request::default()).await?; + let mut server = shard.primary_or_replica(&LbRequest::default()).await?; let oids = load_oids(&mut server).await?; let server_addr = server.addr().clone(); drop(server); diff --git a/pgdog/src/backend/replication/logical/resharding_state.rs b/pgdog/src/backend/replication/logical/resharding_state.rs index d32c559c9..bafd7682d 100644 --- a/pgdog/src/backend/replication/logical/resharding_state.rs +++ b/pgdog/src/backend/replication/logical/resharding_state.rs @@ -11,7 +11,7 @@ use super::publisher::{Permanent, ReplicationSlot, Table}; use super::subscriber::replication_origin::ReplicationOrigin; use super::tables_sync::tables_sync; use super::*; -use crate::backend::pool::Request; +use crate::backend::pool::LbRequest; use crate::tasks; use crate::util::safe_timeout; use crate::{backend::Cluster, util::random_string}; @@ -243,7 +243,7 @@ impl ReshardingState { let slot_name = slot_name.clone(); async move { - let addr = shard.primary(&Request::default()).await?.addr().clone(); + let addr = shard.primary(&LbRequest::default()).await?.addr().clone(); let slot = ReplicationSlot::new_permanent(&publication, &addr, Some(slot_name), number); @@ -268,7 +268,7 @@ impl ReshardingState { let mut addresses = vec![]; for shard in self.destination.shards() { - addresses.push(shard.primary(&Request::default()).await?.addr().clone()); + addresses.push(shard.primary(&LbRequest::default()).await?.addr().clone()); } for (number, slot_name) in slots { @@ -432,7 +432,7 @@ mod test { for (number, shard) in source.shards().iter().enumerate() { let addr = shard - .primary(&Request::default()) + .primary(&LbRequest::default()) .await .unwrap() .addr() diff --git a/pgdog/src/backend/replication/logical/schema_sync.rs b/pgdog/src/backend/replication/logical/schema_sync.rs index 929b751a0..476efb2cd 100644 --- a/pgdog/src/backend/replication/logical/schema_sync.rs +++ b/pgdog/src/backend/replication/logical/schema_sync.rs @@ -9,7 +9,7 @@ use tracing::{info, warn}; use crate::backend::{ self, Cluster, Schema, databases::{databases, reload_from_existing}, - pool::{Guard, Request}, + pool::{Guard, LbRequest}, schema::sync::{PgDump, SchemaSyncError, Statement, pg_dump::PgDumpOutput}, }; @@ -61,7 +61,7 @@ impl SchemaSync { |shard| async move { let mut primary = self .destination - .primary(shard, &Request::default()) + .primary(shard, &LbRequest::default()) .await .ok()?; server_version(&mut primary).await.ok().flatten() @@ -85,7 +85,10 @@ impl SchemaSync { /// Take a connection to one destination shard, to apply statements through. pub(crate) async fn shard(&self, shard: usize) -> Result { - let primary = self.destination.primary(shard, &Request::default()).await?; + let primary = self + .destination + .primary(shard, &LbRequest::default()) + .await?; info!( "syncing schema into shard {} [{}, {}]", @@ -206,7 +209,7 @@ mod tests { cluster.launch(); ShardRestore { - primary: cluster.primary(0, &Request::default()).await.unwrap(), + primary: cluster.primary(0, &LbRequest::default()).await.unwrap(), } } diff --git a/pgdog/src/backend/replication/logical/subscriber/copy.rs b/pgdog/src/backend/replication/logical/subscriber/copy.rs index 623f982d6..7889314a6 100644 --- a/pgdog/src/backend/replication/logical/subscriber/copy.rs +++ b/pgdog/src/backend/replication/logical/subscriber/copy.rs @@ -458,7 +458,7 @@ mod test { use crate::{ backend::{ - pool::Request, replication::publisher::PublicationTable, server::test::test_server, + pool::LbRequest, replication::publisher::PublicationTable, server::test::test_server, }, config::config, frontend::router::parser::binary::{Data, header::Header, tuple::Tuple}, @@ -473,7 +473,7 @@ mod test { ) -> Result<(), crate::backend::Error> { let query: Query = query.into(); for shard in 0..cluster.shards().len() { - let mut server = cluster.primary(shard, &Request::default()).await?; + let mut server = cluster.primary(shard, &LbRequest::default()).await?; server.execute(query.clone()).await?; } @@ -535,7 +535,7 @@ mod test { .copy_done(&CancellationToken::new()) .await .unwrap(); - let mut server = cluster.primary(0, &Request::default()).await.unwrap(); + let mut server = cluster.primary(0, &LbRequest::default()).await.unwrap(); let count = server .fetch_all::("SELECT COUNT(*)::BIGINT FROM pgdog.sharded") .await diff --git a/pgdog/src/backend/replication/logical/tables_sync.rs b/pgdog/src/backend/replication/logical/tables_sync.rs index 2425d3ee2..11d387db2 100644 --- a/pgdog/src/backend/replication/logical/tables_sync.rs +++ b/pgdog/src/backend/replication/logical/tables_sync.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use super::Error; use super::publisher::Table; -use crate::backend::pool::Request; +use crate::backend::pool::LbRequest; use crate::backend::{Cluster, ShardedTables}; /// Fetch the info about tables on shard and distribute the set of tables @@ -26,7 +26,7 @@ pub(crate) async fn tables_sync( let mut omnisharded = HashSet::new(); for shard in source.shards() { - let mut primary = shard.primary(&Request::default()).await?; + let mut primary = shard.primary(&LbRequest::default()).await?; let tables = Table::load(publication, &mut primary).await?; for table in tables { diff --git a/pgdog/src/backend/replication/tests.rs b/pgdog/src/backend/replication/tests.rs index 1150107fe..5b814c76f 100644 --- a/pgdog/src/backend/replication/tests.rs +++ b/pgdog/src/backend/replication/tests.rs @@ -23,7 +23,7 @@ use crate::{ }, backend::{ Cluster, ConnectReason, Error as BackendError, Server, ServerOptions, databases, - pool::{Address, Guard, Request}, + pool::{Address, Guard, LbRequest}, schema::sync::SchemaSyncError, server::test::test_server, }, @@ -113,7 +113,7 @@ async fn replicate_until_caught_up( state: &ReshardingState, slot_name: &str, ) -> Result<(), Box> { - let mut server = state.source.primary(0, &Request::default()).await?; + let mut server = state.source.primary(0, &LbRequest::default()).await?; let (task, stop) = start_replication(state); let caught_up = wait_for_slot(&mut server, &format!("{slot_name}_0")).await; @@ -192,7 +192,7 @@ async fn create_tables( ) -> Result<(Guard, Guard), Box> { let source = databases::databases().schema_owner(schema)?; let dest = databases::databases().schema_owner(destination)?; - let mut source_server = source.primary(0, &Request::default()).await?; + let mut source_server = source.primary(0, &LbRequest::default()).await?; source_server .execute_checked(format!( "CREATE SCHEMA {schema}; \ @@ -205,7 +205,7 @@ async fn create_tables( CREATE PUBLICATION {schema} FOR TABLE {schema}.main, {schema}.small" )) .await?; - let mut dest_server = dest.primary(0, &Request::default()).await?; + let mut dest_server = dest.primary(0, &LbRequest::default()).await?; dest_server .execute_checked(format!( "CREATE SCHEMA {schema}; \ @@ -278,7 +278,7 @@ async fn wait_for_replication_finishes_with_unrelated_writes() setup_replication_test(&mut admin, schema, destination).await?; let source = databases::databases().schema_owner(schema)?; let dest = databases::databases().schema_owner(destination)?; - let mut server = source.primary(0, &Request::default()).await?; + let mut server = source.primary(0, &LbRequest::default()).await?; server .execute_checked(format!( "CREATE SCHEMA {schema}; \ @@ -287,7 +287,7 @@ async fn wait_for_replication_finishes_with_unrelated_writes() CREATE PUBLICATION {schema} FOR TABLE {schema}.main" )) .await?; - let mut dest_server = dest.primary(0, &Request::default()).await?; + let mut dest_server = dest.primary(0, &LbRequest::default()).await?; dest_server .execute_checked(format!( "CREATE SCHEMA {schema}; \ @@ -354,7 +354,7 @@ async fn test_replication_fk_conflicts_after_delete_during_copy() let result = async { setup_replication_test(&mut admin, &schema, &destination).await?; let source = databases::databases().schema_owner(&schema)?; - let mut source_server = source.primary(0, &Request::default()).await?; + let mut source_server = source.primary(0, &LbRequest::default()).await?; // create the tables tied by fk and put some data there source_server @@ -454,7 +454,7 @@ async fn test_replication_fk_conflicts_after_delete_during_copy() let validation = async { let dest = result?; - let mut server = dest.primary(0, &Request::default()).await?; + let mut server = dest.primary(0, &LbRequest::default()).await?; let parents: Vec = server .fetch_all(format!("SELECT id FROM {parent} ORDER BY id")) .await?; @@ -491,7 +491,7 @@ async fn test_replication_fk_constraints_after_copy_child_before_parent() let result = async { setup_replication_test(&mut admin, schema, destination).await?; let source = databases::databases().schema_owner(schema)?; - let mut server = source.primary(0, &Request::default()).await?; + let mut server = source.primary(0, &LbRequest::default()).await?; // create the tables tied by fk and put some data there server @@ -569,7 +569,7 @@ async fn test_replication_fk_constraints_after_copy_child_before_parent() // check that both tables have the copied rows and the fk still points to the parent let validation = async { let dest = result?; - let mut server = dest.primary(0, &Request::default()).await?; + let mut server = dest.primary(0, &LbRequest::default()).await?; let parents: Vec = server .fetch_all(format!( "SELECT id || ':' || tenant_id FROM {schema}.parents ORDER BY id" @@ -604,7 +604,7 @@ async fn test_replication_copy_custom_parent_trigger() -> Result<(), Box Result<(), Box = server .fetch_all(format!("SELECT id FROM {schema}.parents ORDER BY id")) .await?; @@ -911,7 +911,7 @@ async fn post_data_fails_when_index_rejects_copied_rows() -> Result<(), Box Result<(), Box Result<(), Box = server .fetch_all(format!("SELECT id FROM {schema}.items ORDER BY id")) .await?; @@ -988,7 +988,7 @@ async fn tables_only_copies_and_replicates_one_table() -> Result<(), Box Result<(), Box TestResult { ); let schema = &fixture.schema; - let mut server = fixture.dest.primary(0, &Request::default()).await?; + let mut server = fixture.dest.primary(0, &LbRequest::default()).await?; let parents: Vec = server .fetch_all(format!("SELECT id FROM {schema}.parents ORDER BY id")) .await?; @@ -361,7 +361,7 @@ async fn post_data_validation_during_replication_gates_cutover() -> TestResult { PostDataValidationStage::DuringReplication, false, async |fixture| { - let mut lock = fixture.dest.primary(0, &Request::default()).await?; + let mut lock = fixture.dest.primary(0, &LbRequest::default()).await?; lock.execute_checked(format!( "BEGIN; LOCK TABLE {}.children IN SHARE UPDATE EXCLUSIVE MODE", fixture.schema diff --git a/pgdog/src/backend/schema/mod.rs b/pgdog/src/backend/schema/mod.rs index f9cee8c84..5ee2da795 100644 --- a/pgdog/src/backend/schema/mod.rs +++ b/pgdog/src/backend/schema/mod.rs @@ -15,7 +15,7 @@ use tracing::info; pub(crate) use cache::SchemaCache; pub(crate) use relation::Relation; -use super::{Cluster, Error, Server, pool::Request}; +use super::{Cluster, Error, Server, pool::LbRequest}; use crate::frontend::router::parser::Table; use crate::net::parameter::ParameterValue; use sync::ShardConfig; @@ -185,7 +185,7 @@ impl Schema { ShardConfig::sync_all(cluster).await?; for shard in shards { - let mut server = shard.primary(&Request::default()).await?; + let mut server = shard.primary(&LbRequest::default()).await?; Self::install_server(&mut server).await?; } diff --git a/pgdog/src/backend/schema/sync/pg_dump.rs b/pgdog/src/backend/schema/sync/pg_dump.rs index 9e3a1fbf5..16a12f64b 100644 --- a/pgdog/src/backend/schema/sync/pg_dump.rs +++ b/pgdog/src/backend/schema/sync/pg_dump.rs @@ -20,7 +20,7 @@ use tracing::{info, trace, warn}; use super::SchemaSyncError; use crate::{ - backend::{self, Cluster, pool::Request, replication::publisher::PublicationTable}, + backend::{self, Cluster, pool::LbRequest, replication::publisher::PublicationTable}, config::config, frontend::router::parser::{Column, Sequence, Table}, }; @@ -236,7 +236,7 @@ impl PgDump { ); for (num, shard) in self.source.shards().iter().enumerate() { - let mut server = shard.primary_or_replica(&Request::default()).await?; + let mut server = shard.primary_or_replica(&LbRequest::default()).await?; let tables = PublicationTable::load(&self.publication, &mut server).await?; if comparison.is_empty() { comparison.extend(tables); diff --git a/pgdog/src/frontend/client/mod.rs b/pgdog/src/frontend/client/mod.rs index fd10dd884..74f17be56 100644 --- a/pgdog/src/frontend/client/mod.rs +++ b/pgdog/src/frontend/client/mod.rs @@ -19,7 +19,7 @@ use crate::auth::{AUTH_TOKEN_CACHE, AuthResult, md5, scram::Server}; use crate::backend::pool::stats::MemoryStats; use crate::backend::{ databases, - pool::{Connection, Request}, + pool::{Connection, LbRequest}, }; use crate::backend::{maintenance_mode, passthrough_auth}; use crate::config::convert::user_from_params; @@ -398,7 +398,7 @@ impl Client { // Get connection parameters. These will be most likely cached, // unless the pool was just created. - let server_params = match conn.parameters(&Request::unrouted(id)).await { + let server_params = match conn.parameters(&LbRequest::unrouted(id)).await { Ok(params) => params, Err(err) => { if err.no_server() { diff --git a/pgdog/src/frontend/client/query_engine/connect.rs b/pgdog/src/frontend/client/query_engine/connect.rs index 079ce77c0..9f56bccc5 100644 --- a/pgdog/src/frontend/client/query_engine/connect.rs +++ b/pgdog/src/frontend/client/query_engine/connect.rs @@ -34,7 +34,13 @@ impl QueryEngine { // Pass through the cluster's `read_only` flag (propagated from `User.read_only`) // to determine if we should exclude the primary from being allowed to read. let read_only = self.backend.cluster()?.read_only(); - let request = Request::new(context.id, connect_route, read_only, context.query_start); + let request = LbRequest::new( + context.id, + connect_route, + vec![], + read_only, + context.query_start, + ); self.stats.waiting(request.created_at); self.comms.update_stats(self.stats); diff --git a/pgdog/src/frontend/client/query_engine/mod.rs b/pgdog/src/frontend/client/query_engine/mod.rs index 607aa563a..1174a36c5 100644 --- a/pgdog/src/frontend/client/query_engine/mod.rs +++ b/pgdog/src/frontend/client/query_engine/mod.rs @@ -1,5 +1,5 @@ use crate::{ - backend::pool::{Connection, Request}, + backend::pool::{Connection, LbRequest}, config::config, frontend::{ BufferedQuery, Client, ClientComms, ClientRequest, Command, DiscardTarget, Error, Router, diff --git a/pgdog/src/frontend/client/query_engine/test/lazy_connections.rs b/pgdog/src/frontend/client/query_engine/test/lazy_connections.rs index c2e72ada5..9f8feb221 100644 --- a/pgdog/src/frontend/client/query_engine/test/lazy_connections.rs +++ b/pgdog/src/frontend/client/query_engine/test/lazy_connections.rs @@ -137,7 +137,7 @@ async fn recovery_setup() -> (SpawnedClient, Guard) { use crate::{ backend::{ databases::{databases, reload_from_existing}, - pool::Request, + pool::LbRequest, }, config::{config, load_test_sharded, set}, }; @@ -151,7 +151,7 @@ async fn recovery_setup() -> (SpawnedClient, Guard) { reload_from_existing().unwrap(); let client = SpawnedClient::new(Parameters::default()).await; let cluster = databases().cluster(("pgdog", "pgdog")).unwrap(); - let held = cluster.primary(1, &Request::default()).await.unwrap(); + let held = cluster.primary(1, &LbRequest::default()).await.unwrap(); (client, held) } diff --git a/pgdog/src/frontend/client/query_engine/test/regression_auto_two_pc.rs b/pgdog/src/frontend/client/query_engine/test/regression_auto_two_pc.rs index dde5fb725..318788df8 100644 --- a/pgdog/src/frontend/client/query_engine/test/regression_auto_two_pc.rs +++ b/pgdog/src/frontend/client/query_engine/test/regression_auto_two_pc.rs @@ -1,6 +1,6 @@ use super::prelude::*; use crate::{ - backend::{Cluster, pool::Request}, + backend::{Cluster, pool::LbRequest}, expect_message, net::{CommandComplete, DataRow, ErrorResponse, ReadyForQuery}, }; @@ -17,7 +17,7 @@ async fn two_pc_client(auto: bool) -> (TestClient, Cluster) { } async fn execute_on_shard(cluster: &Cluster, shard: usize, query: String) -> String { - let mut server = cluster.primary(shard, &Request::default()).await.unwrap(); + let mut server = cluster.primary(shard, &LbRequest::default()).await.unwrap(); let messages = server.execute(query).await.unwrap(); let complete = messages .iter() diff --git a/pgdog/src/frontend/client/query_engine/test/regression_bind_transaction_end.rs b/pgdog/src/frontend/client/query_engine/test/regression_bind_transaction_end.rs index 6787af016..1c6093072 100644 --- a/pgdog/src/frontend/client/query_engine/test/regression_bind_transaction_end.rs +++ b/pgdog/src/frontend/client/query_engine/test/regression_bind_transaction_end.rs @@ -1,6 +1,6 @@ use super::prelude::*; use crate::{ - backend::pool::Request, + backend::pool::LbRequest, expect_message, net::{BindComplete, CommandComplete, ParseComplete, ReadyForQuery}, }; @@ -41,7 +41,7 @@ async fn bind_transaction_end(sql: &str) { client.send_simple(Query::new("ROLLBACK")).await; client.read_until('Z').await.unwrap(); - let mut server = cluster.primary(0, &Request::default()).await.unwrap(); + let mut server = cluster.primary(0, &LbRequest::default()).await.unwrap(); let cleanup = server .execute(format!("DELETE FROM sharded WHERE id = {id}")) .await diff --git a/pgdog/src/frontend/client/query_engine/test/regression_prepare_transaction_end.rs b/pgdog/src/frontend/client/query_engine/test/regression_prepare_transaction_end.rs index c751132e1..a0c560ee3 100644 --- a/pgdog/src/frontend/client/query_engine/test/regression_prepare_transaction_end.rs +++ b/pgdog/src/frontend/client/query_engine/test/regression_prepare_transaction_end.rs @@ -1,6 +1,6 @@ use super::prelude::*; use crate::{ - backend::pool::Request, + backend::pool::LbRequest, expect_message, net::{CommandComplete, DataRow, NoData, ParameterDescription, ParseComplete, ReadyForQuery}, }; @@ -68,7 +68,7 @@ async fn prepare_transaction_end(sql: &str) { client.read_until('Z').await.unwrap(); // Verify the effect on PostgreSQL directly, independently of client state. - let mut server = cluster.primary(0, &Request::default()).await.unwrap(); + let mut server = cluster.primary(0, &LbRequest::default()).await.unwrap(); let rows: Vec = server .fetch_all(format!("SELECT id FROM sharded WHERE id = {id}")) .await diff --git a/pgdog/src/frontend/client/query_engine/two_pc/manager.rs b/pgdog/src/frontend/client/query_engine/two_pc/manager.rs index 5e76c8c4a..40f3e8294 100644 --- a/pgdog/src/frontend/client/query_engine/two_pc/manager.rs +++ b/pgdog/src/frontend/client/query_engine/two_pc/manager.rs @@ -20,7 +20,7 @@ use tracing::{debug, error, info, warn}; use crate::{ backend::{ databases::User, - pool::{Connection, Request}, + pool::{Connection, LbRequest}, }, frontend::router::{ Route, @@ -378,7 +378,7 @@ impl Manager { connection .connect( - &Request::default(), + &LbRequest::default(), &Route::write(ShardWithPriority::new_override_cross_shard(Shard::All)), ) .await?; diff --git a/pgdog/src/frontend/client/query_engine/two_pc/test.rs b/pgdog/src/frontend/client/query_engine/two_pc/test.rs index 8a15e0db0..084fa38c1 100644 --- a/pgdog/src/frontend/client/query_engine/two_pc/test.rs +++ b/pgdog/src/frontend/client/query_engine/two_pc/test.rs @@ -1,7 +1,7 @@ use crate::{ backend::{ databases::databases, - pool::{Connection, Request}, + pool::{Connection, LbRequest}, }, config, frontend::router::{ @@ -24,7 +24,7 @@ async fn test_cleanup_transaction_phase_one() { let mut conn = Connection::new(cluster.user(), cluster.name(), false).unwrap(); conn.connect( - &Request::default(), + &LbRequest::default(), &Route::write(ShardWithPriority::new_default_unset(Shard::All)), ) .await @@ -60,7 +60,7 @@ async fn test_cleanup_transaction_phase_one() { assert!(transactions.is_empty()); conn.connect( - &Request::default(), + &LbRequest::default(), &Route::write(ShardWithPriority::new_default_unset(Shard::All)), ) .await @@ -96,7 +96,7 @@ async fn test_cleanup_transaction_phase_two() { let mut conn = Connection::new(cluster.user(), cluster.name(), false).unwrap(); conn.connect( - &Request::default(), + &LbRequest::default(), &Route::write(ShardWithPriority::new_default_unset(Shard::All)), ) .await @@ -137,7 +137,7 @@ async fn test_cleanup_transaction_phase_two() { assert!(transactions.is_empty()); conn.connect( - &Request::default(), + &LbRequest::default(), &Route::write(ShardWithPriority::new_default_unset(Shard::All)), ) .await diff --git a/pgdog/src/frontend/client/query_engine/two_pc/wal/tests/client.rs b/pgdog/src/frontend/client/query_engine/two_pc/wal/tests/client.rs index e96a16399..48a1bf753 100644 --- a/pgdog/src/frontend/client/query_engine/two_pc/wal/tests/client.rs +++ b/pgdog/src/frontend/client/query_engine/two_pc/wal/tests/client.rs @@ -3,7 +3,7 @@ use std::time::Duration; use tempfile::TempDir; -use crate::backend::pool::Request; +use crate::backend::pool::LbRequest; use crate::backend::{Cluster, databases::databases}; use crate::config; use crate::frontend::client::query_engine::TwoPcPhase; @@ -29,7 +29,7 @@ impl TwoPcTestClient { let mut conns = vec![]; for shard in self.cluster.shards() { - conns.push(shard.primary(&Request::default()).await.unwrap()); + conns.push(shard.primary(&LbRequest::default()).await.unwrap()); } let txn = TwoPcTransaction::new(); @@ -117,7 +117,7 @@ impl TwoPcTestClient { let mut transactions = HashSet::new(); for shard in self.cluster.shards() { - let mut server = shard.primary(&Request::default()).await.unwrap(); + let mut server = shard.primary(&LbRequest::default()).await.unwrap(); let records: Vec = server .fetch_all("SELECT gid FROM pg_prepared_xacts") .await diff --git a/pgdog/src/frontend/router/comment.rs b/pgdog/src/frontend/router/comment.rs index f160f4556..eeddbf715 100644 --- a/pgdog/src/frontend/router/comment.rs +++ b/pgdog/src/frontend/router/comment.rs @@ -11,6 +11,7 @@ pub(in crate::frontend) struct RoutingComment { pub(super) role: Option, pub(super) sharding_key: Option, pub(super) lsn: Option, + pub(super) label: Option, } impl RoutingComment { @@ -21,6 +22,7 @@ impl RoutingComment { role: None, sharding_key: None, lsn: None, + label: None, } } } diff --git a/pgdog/src/frontend/router/parser/comment/mod.rs b/pgdog/src/frontend/router/parser/comment/mod.rs index 82bfc112a..030b32757 100644 --- a/pgdog/src/frontend/router/parser/comment/mod.rs +++ b/pgdog/src/frontend/router/parser/comment/mod.rs @@ -129,6 +129,7 @@ pub(crate) fn parse_edge_comment<'a>( shard: directive.shard_or_lookup, role: directive.role, lsn: directive.lsn, + label: None, }, spans, }) diff --git a/pgdog/src/frontend/router/parser/query/test/test_comments.rs b/pgdog/src/frontend/router/parser/query/test/test_comments.rs index 7da49818d..88c8b6461 100644 --- a/pgdog/src/frontend/router/parser/query/test/test_comments.rs +++ b/pgdog/src/frontend/router/parser/query/test/test_comments.rs @@ -21,7 +21,7 @@ fn test_comment_pgdog_lsn() { let command = test.execute(vec![Query::new(sql).into()]); let route = command.route(); assert_eq!(route.lsn, expected, "{sql}"); - let request = Request::new(Default::default(), route, false, Instant::now()); + let request = Request::new(Default::default(), route, Instant::now()); assert_eq!(request.lsn, expected, "{sql}"); } } From 5c4f15c954bb7774cc555dda3479b4128848e08a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 10:23:48 -0800 Subject: [PATCH 2/8] labels --- pgdog/src/backend/pool/lb/mod.rs | 13 +++-- pgdog/src/backend/pool/lb/request.rs | 22 +++++-- pgdog/src/backend/pool/lb/test/label.rs | 57 +++++++++++++++++++ pgdog/src/backend/pool/lb/test/mod.rs | 15 ++--- .../frontend/client/query_engine/connect.rs | 8 +-- .../src/frontend/client/query_engine/query.rs | 6 +- .../router/parser/comment/directive.rs | 14 ++++- .../src/frontend/router/parser/comment/mod.rs | 6 +- .../frontend/router/parser/comment/tests.rs | 52 +++++++++++++++-- pgdog/src/frontend/router/parser/error.rs | 3 - pgdog/src/frontend/router/parser/query/mod.rs | 5 ++ .../router/parser/query/test/test_comments.rs | 19 +++++++ pgdog/src/frontend/router/parser/route.rs | 2 + 13 files changed, 183 insertions(+), 39 deletions(-) create mode 100644 pgdog/src/backend/pool/lb/test/label.rs diff --git a/pgdog/src/backend/pool/lb/mod.rs b/pgdog/src/backend/pool/lb/mod.rs index 01103f894..00c3ea199 100644 --- a/pgdog/src/backend/pool/lb/mod.rs +++ b/pgdog/src/backend/pool/lb/mod.rs @@ -426,6 +426,8 @@ impl LoadBalancer { use LoadBalancingStrategy::*; use smallvec::SmallVec; + let ignore_ban = request.label.is_some(); + let mut candidates: SmallVec<[&Target; 32]> = self .targets .iter() @@ -439,6 +441,10 @@ impl LoadBalancer { }); } + if let Some(ref label) = request.label { + candidates.retain(|target| target.pool.addr().labels.contains(label)); + } + let primary_reads = primary_reads(self.rw_split, request.read_only, &candidates); if !primary_reads { @@ -449,12 +455,11 @@ impl LoadBalancer { return Err(Error::AllReplicasDown); } - // make sure we don't use the banned targets calculations, - // since it can skew the distribution when we have banned instances + // Exclude banned targets from balancing unless a label explicitly selects them. let mut healthy: SmallVec<[&Target; 32]> = candidates .iter() .copied() - .filter(|target| !target.ban.banned()) + .filter(|target| !target.ban.banned() || ignore_ban) .collect(); if !healthy.is_empty() { @@ -504,7 +509,7 @@ impl LoadBalancer { for target in &healthy { // we still check banned state since the candidate // could be banned in the middle of `get` awaiting - if target.ban.banned() { + if target.ban.banned() && !ignore_ban { continue; } match target.pool.get(request).await { diff --git a/pgdog/src/backend/pool/lb/request.rs b/pgdog/src/backend/pool/lb/request.rs index ff0cf1f79..c815703d3 100644 --- a/pgdog/src/backend/pool/lb/request.rs +++ b/pgdog/src/backend/pool/lb/request.rs @@ -9,7 +9,7 @@ pub(crate) struct LbRequest { request: PoolRequest, /// Whether the client is configured read-only. pub(super) read_only: bool, - pub(super) labels: Vec, + pub(super) label: Option, } impl LbRequest { @@ -17,13 +17,12 @@ impl LbRequest { pub(crate) fn new( id: FrontendPid, route: &Route, - labels: Vec, read_only: bool, created_at: Instant, ) -> Self { Self { request: PoolRequest::new(id, route, created_at), - labels, + label: route.label.clone(), read_only, } } @@ -33,7 +32,7 @@ impl LbRequest { Self { request: PoolRequest::unrouted(id), read_only: false, - labels: Vec::new(), + label: None, } } } @@ -45,3 +44,18 @@ impl Deref for LbRequest { &self.request } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_label_is_copied_to_request() { + for label in [None, Some("analytics")] { + let mut route = Route::default(); + route.label = label.map(str::to_owned); + let request = LbRequest::new(FrontendPid::default(), &route, false, Instant::now()); + assert_eq!(request.label.as_deref(), label); + } + } +} diff --git a/pgdog/src/backend/pool/lb/test/label.rs b/pgdog/src/backend/pool/lb/test/label.rs new file mode 100644 index 000000000..0af81af24 --- /dev/null +++ b/pgdog/src/backend/pool/lb/test/label.rs @@ -0,0 +1,57 @@ +use super::*; + +fn labeled_replicas() -> LoadBalancer { + let mut reporting = create_test_pool_config("127.0.0.1", 5432); + reporting.address.labels = vec!["reporting".into()]; + let mut analytics = create_test_pool_config("localhost", 5432); + analytics.address.labels = vec!["reporting".into(), "analytics".into()]; + + let replicas = LoadBalancer::new( + &None, + &[reporting, analytics], + LoadBalancingStrategy::RoundRobin, + ReadWriteSplit::ExcludePrimary, + Default::default(), + ); + replicas.launch(); + replicas +} + +#[tokio::test] +async fn test_label_filtering() { + let replicas = labeled_replicas(); + let mut route = Route::read(ShardWithPriority::default()); + route.label = Some("analytics".into()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + for _ in 0..4 { + let conn = replicas + .get(&request) + .await + .expect("matching labeled replica"); + assert_eq!(conn.pool.id(), replicas.targets[1].pool.id()); + } + + replicas.shutdown(); +} + +#[tokio::test] +async fn test_label_ignores_ban() { + let replicas = labeled_replicas(); + let target = &replicas.targets[1]; + target.ban.ban(Error::ServerError, Duration::from_secs(60)); + assert!(target.ban.banned()); + + let mut route = Route::read(ShardWithPriority::default()); + route.label = Some("analytics".into()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + let conn = replicas + .get(&request) + .await + .expect("labeled request ignores the replica ban"); + assert_eq!(conn.pool.id(), target.pool.id()); + drop(conn); + + replicas.shutdown(); +} diff --git a/pgdog/src/backend/pool/lb/test/mod.rs b/pgdog/src/backend/pool/lb/test/mod.rs index d037cbeb1..6d3e135bf 100644 --- a/pgdog/src/backend/pool/lb/test/mod.rs +++ b/pgdog/src/backend/pool/lb/test/mod.rs @@ -14,6 +14,7 @@ use pgdog_stats::{LsnStats as StatsLsnStats, ReplicaLag}; use super::*; use monitor::Monitor; +mod label; mod primary_reads; mod role_detection; @@ -474,7 +475,7 @@ async fn test_lsn_filtering() { let mut route = Route::read(ShardWithPriority::default()); route.lsn = Some(caught_up); - let request = LbRequest::new(Default::default(), &route, vec![], false, Instant::now()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); // A replica exactly at the requested LSN is eligible; the lagging one is not. for _ in 0..4 { @@ -484,7 +485,7 @@ async fn test_lsn_filtering() { // Fall back to primary when every replica is behind the requested LSN. route.lsn = Some(ahead); - let request = LbRequest::new(Default::default(), &route, vec![], false, Instant::now()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); let conn = replicas.get(&request).await.expect("primary fallback"); assert_eq!(conn.pool.id(), primary_id); } @@ -509,13 +510,7 @@ async fn test_include_primary_if_replica_lag() { let mut route = Route::read(ShardWithPriority::default()); for lsn in [None, Some(200), Some(201)] { route.lsn = lsn; - let request = LbRequest::new( - Default::default(), - &route, - vec![], - read_only, - Instant::now(), - ); + let request = LbRequest::new(Default::default(), &route, read_only, Instant::now()); // Round robin must never select the primary while an eligible replica exists. for _ in 0..4 { @@ -554,7 +549,6 @@ async fn test_user_read_only_include_primary() { let request = LbRequest::new( Default::default(), &Route::read(ShardWithPriority::default()), - vec![], true, Instant::now(), ); @@ -601,7 +595,6 @@ async fn test_user_read_only_include_primary_no_replicas() { let request = LbRequest::new( Default::default(), &Route::read(ShardWithPriority::default()), - vec![], true, Instant::now(), ); diff --git a/pgdog/src/frontend/client/query_engine/connect.rs b/pgdog/src/frontend/client/query_engine/connect.rs index 9f56bccc5..44f43da4f 100644 --- a/pgdog/src/frontend/client/query_engine/connect.rs +++ b/pgdog/src/frontend/client/query_engine/connect.rs @@ -34,13 +34,7 @@ impl QueryEngine { // Pass through the cluster's `read_only` flag (propagated from `User.read_only`) // to determine if we should exclude the primary from being allowed to read. let read_only = self.backend.cluster()?.read_only(); - let request = LbRequest::new( - context.id, - connect_route, - vec![], - read_only, - context.query_start, - ); + let request = LbRequest::new(context.id, connect_route, read_only, context.query_start); self.stats.waiting(request.created_at); self.comms.update_stats(self.stats); diff --git a/pgdog/src/frontend/client/query_engine/query.rs b/pgdog/src/frontend/client/query_engine/query.rs index 57240c56e..377f393f6 100644 --- a/pgdog/src/frontend/client/query_engine/query.rs +++ b/pgdog/src/frontend/client/query_engine/query.rs @@ -52,7 +52,11 @@ impl QueryEngine { // Rewriter can tell us how many shards we need. let rewrite_connect_route = query_planner .as_ref() - .and_then(|rewrite| rewrite.connect_route()); + .and_then(|rewrite| rewrite.connect_route()) + .map(|mut route| { + route.label.clone_from(&client_request.route().label); + route + }); let connect_route = if let Some(ref rewrite_connect_route) = rewrite_connect_route { rewrite_connect_route diff --git a/pgdog/src/frontend/router/parser/comment/directive.rs b/pgdog/src/frontend/router/parser/comment/directive.rs index 54b4cf9de..9033a0616 100644 --- a/pgdog/src/frontend/router/parser/comment/directive.rs +++ b/pgdog/src/frontend/router/parser/comment/directive.rs @@ -14,8 +14,9 @@ pub(super) static SHARD: Lazy = pub(super) static SHARDING_KEY: Lazy = Lazy::new(|| { Regex::new(r#"pgdog_sharding_key: *(?:"([^"]*)"|'([^']*)'|([0-9a-zA-Z-]+))"#).unwrap() }); -pub(super) static ROLE: Lazy = - Lazy::new(|| Regex::new(r#"pgdog_role: *(primary|replica)"#).unwrap()); +pub(super) static ROLE: Lazy = Lazy::new(|| { + Regex::new(r"pgdog_role: *(\S*?)(?:\s|\*/|$)").expect("valid role directive regex") +}); pub(super) static LSN: Lazy = Lazy::new(|| { Regex::new(r"pgdog_lsn: *([0-9a-fA-F]{1,8}/[0-9a-fA-F]{1,8})(?:\s|\*/|$)") .expect("valid LSN directive regex") @@ -31,6 +32,7 @@ pub(super) fn get_matched_value<'a>(caps: &'a regex::Captures<'a>) -> Option<&'a pub(crate) struct Directive { pub(crate) shard_or_lookup: Option, pub(crate) role: Option, + pub(crate) label: Option, pub(crate) sharding_key: Option, pub(crate) lsn: Option, } @@ -40,6 +42,7 @@ pub(super) fn shard_role_from_comment( schema: &ShardingSchema, ) -> Result { let mut role = None; + let mut label = None; let lsn = LSN .captures(comment) .and_then(|cap| cap.get(1)?.as_str().parse::().ok()); @@ -50,7 +53,8 @@ pub(super) fn shard_role_from_comment( match r.as_str() { "primary" => role = Some(Role::Primary), "replica" => role = Some(Role::Replica), - _ => return Err(Error::RegexError), + "" => {} + value => label = Some(value.to_owned()), } } if let Some(cap) = SHARDING_KEY.captures(comment) @@ -60,6 +64,7 @@ pub(super) fn shard_role_from_comment( return Ok(Directive { shard_or_lookup: Some(ShardOrLookup::Shard(schema.shard().into())), role, + label, lsn, sharding_key: Some(sharding_key.to_string()), }); @@ -67,6 +72,7 @@ pub(super) fn shard_role_from_comment( return Ok(Directive { shard_or_lookup: Some(shard_for_bare_key(sharding_key, schema, None)?), role, + label, lsn, sharding_key: Some(sharding_key.to_string()), }); @@ -84,6 +90,7 @@ pub(super) fn shard_role_from_comment( .unwrap_or(Shard::All), )), role, + label, lsn, sharding_key: None, }); @@ -92,6 +99,7 @@ pub(super) fn shard_role_from_comment( Ok(Directive { shard_or_lookup: None, role, + label, lsn, sharding_key: None, }) diff --git a/pgdog/src/frontend/router/parser/comment/mod.rs b/pgdog/src/frontend/router/parser/comment/mod.rs index 030b32757..842fa4b22 100644 --- a/pgdog/src/frontend/router/parser/comment/mod.rs +++ b/pgdog/src/frontend/router/parser/comment/mod.rs @@ -95,6 +95,7 @@ pub(crate) fn parse_edge_comment<'a>( None => Directive { shard_or_lookup: None, role: None, + label: None, sharding_key: None, lsn: None, }, @@ -107,6 +108,9 @@ pub(crate) fn parse_edge_comment<'a>( if directive.role.is_none() { directive.role = t_directive.role; } + if directive.label.is_none() { + directive.label = t_directive.label; + } if directive.sharding_key.is_none() { directive.sharding_key = t_directive.sharding_key; } @@ -129,7 +133,7 @@ pub(crate) fn parse_edge_comment<'a>( shard: directive.shard_or_lookup, role: directive.role, lsn: directive.lsn, - label: None, + label: directive.label, }, spans, }) diff --git a/pgdog/src/frontend/router/parser/comment/tests.rs b/pgdog/src/frontend/router/parser/comment/tests.rs index 33f0f947e..b6a03f26d 100644 --- a/pgdog/src/frontend/router/parser/comment/tests.rs +++ b/pgdog/src/frontend/router/parser/comment/tests.rs @@ -168,11 +168,53 @@ fn test_replica_role_detection() { } #[test] -fn test_invalid_role_detection() { - let schema = test_schema(); - let query = "SELECT * FROM users /* pgdog_role: invalid */"; - let result = parse_edge_comment(query, &schema).unwrap(); - assert_eq!(result.comment.role, None); +fn test_label_detection() { + let schema = lookup_schema(); + for label in [ + "analytics", + "primary-us", + "replica_2", + "eu/west", + "a*b", + "東京", + ] { + for query in [ + format!("/* pgdog_role: {label} */ SELECT 1"), + format!("SELECT 1 /* pgdog_role:{label}*/"), + format!("/* pgdog_role: {label}\tpgdog_shard: 1 */ SELECT 1"), + format!("/* pgdog_role: {label} pgdog_sharding_key: 42 */ SELECT 1"), + ] { + let result = parse_edge_comment(&query, &schema).expect("valid label directive"); + assert_eq!(result.comment.role, None, "{query}"); + assert_eq!(result.comment.label.as_deref(), Some(label), "{query}"); + } + } +} + +#[test] +fn test_role_value_boundaries() { + for (value, role, label) in [ + ("primary", Some(Role::Primary), None), + ("replica", Some(Role::Replica), None), + ("", None, None), + ("analytics extra", None, Some("analytics")), + ("analytics\nextra", None, Some("analytics")), + ] { + let query = format!("/* pgdog_role:{value}*/ SELECT 1"); + let result = parse_edge_comment(&query, &test_schema()).expect("valid directive"); + assert_eq!(result.comment.role, role); + assert_eq!(result.comment.label.as_deref(), label); + } +} + +#[test] +fn test_leading_label_takes_precedence() { + let result = parse_edge_comment( + "/* pgdog_role: analytics */ SELECT 1 /* pgdog_role: reporting */", + &test_schema(), + ) + .expect("valid label directives"); + assert_eq!(result.comment.label.as_deref(), Some("analytics")); } #[test] diff --git a/pgdog/src/frontend/router/parser/error.rs b/pgdog/src/frontend/router/parser/error.rs index 6b35dd4d7..c4a45ce00 100644 --- a/pgdog/src/frontend/router/parser/error.rs +++ b/pgdog/src/frontend/router/parser/error.rs @@ -73,9 +73,6 @@ pub(crate) enum Error { #[error("two-phase transaction control statements are not allowed when two-phase is enabled")] NoTwoPc, - #[error("regex error")] - RegexError, - #[error("cross-shard truncate not supported when schema-sharding is used")] CrossShardTruncateSchemaSharding, diff --git a/pgdog/src/frontend/router/parser/query/mod.rs b/pgdog/src/frontend/router/parser/query/mod.rs index 632aa1467..e507234c1 100644 --- a/pgdog/src/frontend/router/parser/query/mod.rs +++ b/pgdog/src/frontend/router/parser/query/mod.rs @@ -109,6 +109,11 @@ impl QueryParser { | Command::StartTransaction { route, .. } | Command::CommitTransaction { route, .. } | Command::RollbackTransaction { route, .. } => { + route.label = context + .router_context + .comment + .and_then(|comment| comment.label.clone()); + if route.is_cross_shard() && context.shards == 1 { context .shards_calculator diff --git a/pgdog/src/frontend/router/parser/query/test/test_comments.rs b/pgdog/src/frontend/router/parser/query/test/test_comments.rs index 88c8b6461..f46eae74a 100644 --- a/pgdog/src/frontend/router/parser/query/test/test_comments.rs +++ b/pgdog/src/frontend/router/parser/query/test/test_comments.rs @@ -26,6 +26,25 @@ fn test_comment_pgdog_lsn() { } } +#[test] +fn test_comment_pgdog_label() { + let mut test = QueryParserTest::new(); + + for (sql, expected) in [ + ("/* pgdog_role: analytics */ SELECT 1", Some("analytics")), + ("SELECT 1 /* pgdog_role: reporting */", Some("reporting")), + ("/* pgdog_role: primary */ SELECT 1", None), + ("SELECT 1", None), + ( + "/* pgdog_role: analytics */ SHOW search_path", + Some("analytics"), + ), + ] { + let command = test.execute(vec![Query::new(sql).into()]); + assert_eq!(command.route().label.as_deref(), expected, "{sql}"); + } +} + #[test] fn test_comment_pgdog_role_primary() { let mut test = QueryParserTest::new(); diff --git a/pgdog/src/frontend/router/parser/route.rs b/pgdog/src/frontend/router/parser/route.rs index e18944d17..62aa0d7b1 100644 --- a/pgdog/src/frontend/router/parser/route.rs +++ b/pgdog/src/frontend/router/parser/route.rs @@ -148,6 +148,8 @@ pub(crate) struct Route { table_change: TableChange, /// Minimum replica LSN in its 64-bit representation. pub(crate) lsn: Option, + /// Backend label requested by a routing comment. + pub(crate) label: Option, } impl Display for Route { From 5e8a5101f30553b3b36040103b3fd2cf0f82a16f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 10:26:11 -0800 Subject: [PATCH 3/8] jsonschema --- .schema/pgdog.schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.schema/pgdog.schema.json b/.schema/pgdog.schema.json index e06abe6f0..89a9a3af2 100644 --- a/.schema/pgdog.schema.json +++ b/.schema/pgdog.schema.json @@ -461,6 +461,14 @@ "format": "uint64", "minimum": 0 }, + "labels": { + "description": "Labels.", + "type": "array", + "default": [], + "items": { + "type": "string" + } + }, "lb_weight": { "description": "Used for weighted load balancing.", "type": "integer", From ac448c700dbcb3b32fb2c96cc680213612022190 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 10:32:25 -0800 Subject: [PATCH 4/8] edge case --- pgdog/src/backend/pool/lb/mod.rs | 19 +++++++++++-------- pgdog/src/backend/pool/lb/test/label.rs | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pgdog/src/backend/pool/lb/mod.rs b/pgdog/src/backend/pool/lb/mod.rs index 00c3ea199..74caba77a 100644 --- a/pgdog/src/backend/pool/lb/mod.rs +++ b/pgdog/src/backend/pool/lb/mod.rs @@ -53,6 +53,12 @@ impl Candidate for &Target { } } +fn has_unbanned_replica(candidates: &[impl Candidate]) -> bool { + candidates + .iter() + .any(|target| target.role() == Role::Replica && !target.banned()) +} + /// Decide if the primary stays in `candidates` for a read. /// /// `candidates` must already exclude resharding-only targets and replicas @@ -62,11 +68,7 @@ impl Candidate for &Target { fn primary_reads(rw_split: ReadWriteSplit, read_only: bool, candidates: &[impl Candidate]) -> bool { use ReadWriteSplit::*; - let has_unbanned_replica = || { - candidates - .iter() - .any(|target| target.role() == Role::Replica && !target.banned()) - }; + let has_unbanned_replica = || has_unbanned_replica(candidates); match rw_split { // If `read_only` is true, only allow if there's no replicas. @@ -426,8 +428,6 @@ impl LoadBalancer { use LoadBalancingStrategy::*; use smallvec::SmallVec; - let ignore_ban = request.label.is_some(); - let mut candidates: SmallVec<[&Target; 32]> = self .targets .iter() @@ -455,7 +455,10 @@ impl LoadBalancer { return Err(Error::AllReplicasDown); } - // Exclude banned targets from balancing unless a label explicitly selects them. + // Let labeled requests use banned targets when no unbanned replica remains. + let ignore_ban = request.label.is_some() && !has_unbanned_replica(&candidates); + + // Exclude banned targets from balancing unless they are needed as a fallback. let mut healthy: SmallVec<[&Target; 32]> = candidates .iter() .copied() diff --git a/pgdog/src/backend/pool/lb/test/label.rs b/pgdog/src/backend/pool/lb/test/label.rs index 0af81af24..5349567f8 100644 --- a/pgdog/src/backend/pool/lb/test/label.rs +++ b/pgdog/src/backend/pool/lb/test/label.rs @@ -35,6 +35,26 @@ async fn test_label_filtering() { replicas.shutdown(); } +#[tokio::test] +async fn test_label_respects_ban_when_unbanned_replica_exists() { + let replicas = labeled_replicas(); + replicas.targets[0].ban.ban(Error::ManualBan, Duration::MAX); + + let mut route = Route::read(ShardWithPriority::default()); + route.label = Some("reporting".into()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + for _ in 0..4 { + let conn = replicas + .get(&request) + .await + .expect("unbanned labeled replica"); + assert_eq!(conn.pool.id(), replicas.targets[1].pool.id()); + } + + replicas.shutdown(); +} + #[tokio::test] async fn test_label_ignores_ban() { let replicas = labeled_replicas(); From 24fb48ea23da2f32533a3b167385dbaf110caae9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 10:42:25 -0800 Subject: [PATCH 5/8] handle set --- pgdog/src/backend/pool/lb/test/label.rs | 65 ++++++++++++++++++- pgdog/src/frontend/router/parameter_hints.rs | 9 +++ pgdog/src/frontend/router/parser/query/mod.rs | 16 +++-- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/pgdog/src/backend/pool/lb/test/label.rs b/pgdog/src/backend/pool/lb/test/label.rs index 5349567f8..d652e40f4 100644 --- a/pgdog/src/backend/pool/lb/test/label.rs +++ b/pgdog/src/backend/pool/lb/test/label.rs @@ -5,15 +5,20 @@ fn labeled_replicas() -> LoadBalancer { reporting.address.labels = vec!["reporting".into()]; let mut analytics = create_test_pool_config("localhost", 5432); analytics.address.labels = vec!["reporting".into(), "analytics".into()]; + let mut backup = create_test_pool_config("127.0.0.1", 5432); + backup.address.labels = vec!["backup".into()]; let replicas = LoadBalancer::new( &None, - &[reporting, analytics], + &[reporting, analytics, backup], LoadBalancingStrategy::RoundRobin, ReadWriteSplit::ExcludePrimary, Default::default(), ); - replicas.launch(); + // Keep bans under test control instead of running the monitor that clears them. + for target in &replicas.targets { + target.pool.launch(); + } replicas } @@ -75,3 +80,59 @@ async fn test_label_ignores_ban() { replicas.shutdown(); } + +#[tokio::test] +async fn test_label_skips_banned_replica_when_matching_replica_is_available() { + let replicas = labeled_replicas(); + replicas.targets[0] + .ban + .ban(Error::ServerError, Duration::from_secs(60)); + + let mut route = Route::read(ShardWithPriority::default()); + route.label = Some("reporting".into()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + for _ in 0..4 { + let conn = replicas + .get(&request) + .await + .expect("unbanned matching replica"); + assert_eq!(conn.pool.id(), replicas.targets[1].pool.id()); + assert!(replicas.targets[0].ban.banned()); + } + + replicas.shutdown(); +} + +#[tokio::test] +async fn test_label_ignores_bans_when_all_matching_replicas_are_banned() { + let replicas = labeled_replicas(); + for target in &replicas.targets[..2] { + target.ban.ban(Error::ServerError, Duration::from_secs(60)); + } + assert!(!replicas.targets[2].ban.banned()); + + let mut route = Route::read(ShardWithPriority::default()); + route.label = Some("reporting".into()); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + let mut selected = HashSet::new(); + for _ in 0..4 { + let conn = replicas + .get(&request) + .await + .expect("matching replica despite bans"); + selected.insert(conn.pool.id()); + } + assert_eq!( + selected, + HashSet::from([replicas.targets[0].pool.id(), replicas.targets[1].pool.id()]) + ); + assert!( + replicas.targets[..2] + .iter() + .all(|target| target.ban.banned()) + ); + + replicas.shutdown(); +} diff --git a/pgdog/src/frontend/router/parameter_hints.rs b/pgdog/src/frontend/router/parameter_hints.rs index 6231afb43..a9a273571 100644 --- a/pgdog/src/frontend/router/parameter_hints.rs +++ b/pgdog/src/frontend/router/parameter_hints.rs @@ -96,6 +96,15 @@ impl ParameterHints<'_> { Ok(()) } + /// Get a custom backend label from the role parameter. + pub(crate) fn compute_label(&self) -> Option<&str> { + let value = self.pgdog_role?.as_str()?; + if matches!(value, "" | "primary" | "replica") || value.chars().any(char::is_whitespace) { + return None; + } + Some(value) + } + /// Compute role from parameter value. pub(crate) fn compute_role(&self) -> Option { let role = match self.pgdog_role { diff --git a/pgdog/src/frontend/router/parser/query/mod.rs b/pgdog/src/frontend/router/parser/query/mod.rs index e507234c1..d9729d571 100644 --- a/pgdog/src/frontend/router/parser/query/mod.rs +++ b/pgdog/src/frontend/router/parser/query/mod.rs @@ -109,10 +109,18 @@ impl QueryParser { | Command::StartTransaction { route, .. } | Command::CommitTransaction { route, .. } | Command::RollbackTransaction { route, .. } => { - route.label = context - .router_context - .comment - .and_then(|comment| comment.label.clone()); + route.label = match context.router_context.comment { + // Comment role has higher priority than SET, + // so any role/comment set there will ignore label set via `SET pgdog.role`. + Some(comment) if comment.role.is_some() || comment.label.is_some() => { + comment.label.clone() + } + _ => context + .router_context + .parameter_hints + .compute_label() + .map(str::to_owned), + }; if route.is_cross_shard() && context.shards == 1 { context From 36dc7b2c2613c47d4b0304fec1ceedef4d475f8f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 10:43:47 -0800 Subject: [PATCH 6/8] add test --- pgdog/src/frontend/router/parser/query/test/test_set.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pgdog/src/frontend/router/parser/query/test/test_set.rs b/pgdog/src/frontend/router/parser/query/test/test_set.rs index 556fa2739..5bfd2725a 100644 --- a/pgdog/src/frontend/router/parser/query/test/test_set.rs +++ b/pgdog/src/frontend/router/parser/query/test/test_set.rs @@ -13,6 +13,13 @@ use crate::{ use super::Error; use super::setup::*; +#[test] +fn test_set_pgdog_role_label() { + let mut test = QueryParserTest::new().with_param("pgdog.role", "analytics"); + let command = test.execute(vec![Query::new("SELECT 1").into()]); + assert_eq!(command.route().label.as_deref(), Some("analytics")); +} + #[test] fn test_set_local_default_preserves_local() { let mut test = QueryParserTest::new(); From fc883e4266b6c71cf6e7542c75d608e58ddebd5b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 11:26:48 -0800 Subject: [PATCH 7/8] labels required --- pgdog-config/src/database.rs | 4 ++++ pgdog/src/backend/pool/address.rs | 5 +++++ pgdog/src/backend/pool/lb/mod.rs | 2 ++ pgdog/src/backend/pool/lb/test/label.rs | 30 +++++++++++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/pgdog-config/src/database.rs b/pgdog-config/src/database.rs index b6918bb01..6229c5f25 100644 --- a/pgdog-config/src/database.rs +++ b/pgdog-config/src/database.rs @@ -234,6 +234,10 @@ pub struct Database { /// Labels. #[serde(default)] pub labels: Vec, + + /// Only queries containing labels will be routed to this replica. + #[serde(default)] + pub label_required: bool, } impl Database { diff --git a/pgdog/src/backend/pool/address.rs b/pgdog/src/backend/pool/address.rs index 877427deb..3d8966385 100644 --- a/pgdog/src/backend/pool/address.rs +++ b/pgdog/src/backend/pool/address.rs @@ -51,6 +51,9 @@ pub(crate) struct Address { pub(crate) tls: ServerTls, /// Labels. pub(crate) labels: Vec, + /// Labels are required for this host to be included in the load balancer + /// target list. + pub(crate) label_required: bool, } impl Address { @@ -95,6 +98,7 @@ impl Address { configured_role: database.role, tls: database.tls.clone(), labels: database.labels.clone(), + label_required: database.label_required, } } @@ -228,6 +232,7 @@ impl Address { configured_role: Role::Primary, tls: ServerTls::default(), labels: vec![], + label_required: false, } } } diff --git a/pgdog/src/backend/pool/lb/mod.rs b/pgdog/src/backend/pool/lb/mod.rs index 74caba77a..6c212087f 100644 --- a/pgdog/src/backend/pool/lb/mod.rs +++ b/pgdog/src/backend/pool/lb/mod.rs @@ -443,6 +443,8 @@ impl LoadBalancer { if let Some(ref label) = request.label { candidates.retain(|target| target.pool.addr().labels.contains(label)); + } else { + candidates.retain(|target| !target.pool.addr().label_required); } let primary_reads = primary_reads(self.rw_split, request.read_only, &candidates); diff --git a/pgdog/src/backend/pool/lb/test/label.rs b/pgdog/src/backend/pool/lb/test/label.rs index d652e40f4..8eecbd564 100644 --- a/pgdog/src/backend/pool/lb/test/label.rs +++ b/pgdog/src/backend/pool/lb/test/label.rs @@ -22,6 +22,36 @@ fn labeled_replicas() -> LoadBalancer { replicas } +#[tokio::test] +async fn test_label_required() { + let reporting = create_test_pool_config("127.0.0.1", 5432); + let mut analytics = create_test_pool_config("localhost", 5432); + analytics.address.labels = vec!["analytics".into()]; + analytics.address.label_required = true; + + let replicas = LoadBalancer::new( + &None, + &[reporting, analytics], + LoadBalancingStrategy::RoundRobin, + ReadWriteSplit::ExcludePrimary, + Default::default(), + ); + replicas.launch(); + + for (label, target) in [(None, 0), (Some("analytics"), 1)] { + let mut route = Route::read(ShardWithPriority::default()); + route.label = label.map(str::to_owned); + let request = LbRequest::new(Default::default(), &route, false, Instant::now()); + + for _ in 0..4 { + let conn = replicas.get(&request).await.expect("eligible replica"); + assert_eq!(conn.pool.id(), replicas.targets[target].pool.id()); + } + } + + replicas.shutdown(); +} + #[tokio::test] async fn test_label_filtering() { let replicas = labeled_replicas(); From 56897d2556d99049571b65f3f73cea5dd4ec3333 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 11:31:50 -0800 Subject: [PATCH 8/8] jsonschema --- .schema/pgdog.schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.schema/pgdog.schema.json b/.schema/pgdog.schema.json index 89a9a3af2..4a00b0c7f 100644 --- a/.schema/pgdog.schema.json +++ b/.schema/pgdog.schema.json @@ -461,6 +461,11 @@ "format": "uint64", "minimum": 0 }, + "label_required": { + "description": "Only queries containing labels will be routed to this replica.", + "type": "boolean", + "default": false + }, "labels": { "description": "Labels.", "type": "array",