diff --git a/integration/sql/cases/958_count_distinct_sharding_key_case.sql b/integration/sql/cases/958_count_distinct_sharding_key_case.sql new file mode 100644 index 000000000..938eebd17 --- /dev/null +++ b/integration/sql/cases/958_count_distinct_sharding_key_case.sql @@ -0,0 +1,20 @@ +-- description: count(DISTINCT sharding key) adds up each shard's count +-- tags: standard sharded +-- transactional: true + +SELECT count(DISTINCT id) FROM sql_regression_samples; + +SELECT count(DISTINCT id)::int, count(*) FROM sql_regression_samples; + +SELECT count(DISTINCT sql_regression_samples.id) FROM sql_regression_samples; + +SELECT count(DISTINCT s.id) FROM sql_regression_samples s WHERE amount > 5; + +SELECT grp, count(DISTINCT id) FROM sql_regression_samples GROUP BY grp ORDER BY grp; + +SELECT grp, count(DISTINCT id), sum(amount) FROM sql_regression_samples +GROUP BY grp HAVING count(*) > 1 ORDER BY grp; + +SELECT count(DISTINCT id) FILTER (WHERE amount >= 10) FROM sql_regression_samples; + +SELECT count(DISTINCT id) FROM sql_regression_samples WHERE id < 0; diff --git a/integration/sql/cases/958_count_distinct_sharding_key_setup.sql b/integration/sql/cases/958_count_distinct_sharding_key_setup.sql new file mode 100644 index 000000000..7bcfbe9ef --- /dev/null +++ b/integration/sql/cases/958_count_distinct_sharding_key_setup.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS sql_regression_samples; +CREATE TABLE sql_regression_samples (id BIGINT PRIMARY KEY, grp TEXT, amount INTEGER); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (1, 'a', 10); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (2, 'a', 10); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (101, 'a', 20); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (3, 'b', 12); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (102, 'b', NULL); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (4, NULL, 5); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (103, NULL, 6); +INSERT INTO sql_regression_samples (id, grp, amount) VALUES (104, 'c', 30); diff --git a/integration/sql/cases/958_count_distinct_sharding_key_teardown.sql b/integration/sql/cases/958_count_distinct_sharding_key_teardown.sql new file mode 100644 index 000000000..f14a9af47 --- /dev/null +++ b/integration/sql/cases/958_count_distinct_sharding_key_teardown.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sql_regression_samples; diff --git a/pgdog/src/backend/pool/connection/aggregate.rs b/pgdog/src/backend/pool/connection/aggregate.rs index 510babe77..8c8fce987 100644 --- a/pgdog/src/backend/pool/connection/aggregate.rs +++ b/pgdog/src/backend/pool/connection/aggregate.rs @@ -71,6 +71,7 @@ impl<'a> Accumulator<'a> { pub(crate) fn from_aggregate( aggregate: &'a Aggregate, helpers: &HashMap, + disjoint_distinct: &[usize], ) -> Result, Error> { aggregate .targets() @@ -80,7 +81,11 @@ impl<'a> Accumulator<'a> { let accumulator = Accumulator { target, - state: State::new(target, helper)?, + state: State::new( + target, + helper, + disjoint_distinct.contains(&target.column()), + )?, }; Ok(accumulator) @@ -112,8 +117,18 @@ enum State { impl State { /// Construct a new aggregate state based on the function provided /// Errors if the function is not one we support. - fn new(target: &AggregateTarget, helper: HelperColumns) -> Result { - match (target.function(), target.is_distinct()) { + /// + /// `disjoint_distinct` is set for a `DISTINCT` aggregate over the sharding + /// key: shards never share a key, so it merges like the plain aggregate. + fn new( + target: &AggregateTarget, + helper: HelperColumns, + disjoint_distinct: bool, + ) -> Result { + match ( + target.function(), + target.is_distinct() && !disjoint_distinct, + ) { (AggregateFunction::Avg, false) => Ok(Self::Avg(avg::Avg::new( target.column(), helper.count.ok_or_else(|| Error::UnsupportedAggregation { @@ -263,6 +278,7 @@ pub(super) struct Aggregates<'a> { decoder: &'a Decoder, aggregate: &'a Aggregate, helper_columns: HashMap, + disjoint_distinct: &'a [usize], } impl<'a> Aggregates<'a> { @@ -270,7 +286,7 @@ impl<'a> Aggregates<'a> { rows: &'a VecDeque, decoder: &'a Decoder, aggregate: &'a Aggregate, - plan: Option<&ProjectionRewritePlan>, + plan: Option<&'a ProjectionRewritePlan>, ) -> Option { let mut helper_columns: HashMap = HashMap::new(); @@ -333,6 +349,9 @@ impl<'a> Aggregates<'a> { mappings: HashMap::new(), aggregate, helper_columns, + disjoint_distinct: plan + .map(|plan| plan.disjoint_distinct.as_slice()) + .unwrap_or_default(), }) } else { None @@ -359,8 +378,11 @@ impl<'a> Aggregates<'a> { let entry = match self.mappings.entry(grouping) { Entry::Occupied(o) => o.into_mut(), Entry::Vacant(v) => { - let accumulators = - Accumulator::from_aggregate(self.aggregate, &self.helper_columns)?; + let accumulators = Accumulator::from_aggregate( + self.aggregate, + &self.helper_columns, + self.disjoint_distinct, + )?; // Gather all col vals corresponding to list of passthrough indices. let mut passthrough = Vec::new(); @@ -581,6 +603,54 @@ mod test { assert_eq!(total_count, 5); } + #[test] + fn aggregate_count_distinct_on_sharding_key_sums_shard_counts() { + let aggregate = parse("SELECT grp, COUNT(DISTINCT id)::int FROM users GROUP BY grp"); + let decoder = Decoder::from(RowDescription::new(&[ + integer_field("grp"), + integer_field("count"), + ])); + + // (grp, distinct ids on the shard) + let rows = [("1", "2"), ("2", "1"), ("1", "3")] + .into_iter() + .map(|(grp, count)| { + let mut row = DataRow::new(); + row.add(grp).add(count); + row + }) + .collect::>(); + + // Shards may share values of any other column, so the counts can't be added. + let unplanned = Aggregates::new(&rows, &decoder, &aggregate, None) + .unwrap() + .aggregate(); + assert!(matches!( + unplanned, + Err(Error::UnsupportedAggregation { .. }) + )); + + let plan = ProjectionRewritePlan { + disjoint_distinct: vec![1], + ..Default::default() + }; + let merged = Aggregates::new(&rows, &decoder, &aggregate, Some(&plan)) + .unwrap() + .aggregate() + .unwrap(); + let mut counts = merged + .iter() + .map(|row| { + ( + row.get::(0, Format::Text).unwrap(), + row.get::(1, Format::Text).unwrap(), + ) + }) + .collect::>(); + counts.sort_unstable(); + assert_eq!(counts, [(1, 5), (2, 1)]); + } + #[test] fn aggregate_count_preserves_binary_result_types() { use pgdog_postgres_types::{Float, Numeric}; diff --git a/pgdog/src/frontend/client/query_engine/multi_step/subquery/request.rs b/pgdog/src/frontend/client/query_engine/multi_step/subquery/request.rs index 986fee4ef..7561a43c7 100644 --- a/pgdog/src/frontend/client/query_engine/multi_step/subquery/request.rs +++ b/pgdog/src/frontend/client/query_engine/multi_step/subquery/request.rs @@ -44,6 +44,7 @@ impl<'a> SubqueryMulti<'a> { projection::finalize_after_route( &mut client_request, &cluster.schema(), + cluster.sharded_tables(), rewrite_result.offset_plan(), )?; diff --git a/pgdog/src/frontend/client/query_engine/route_query.rs b/pgdog/src/frontend/client/query_engine/route_query.rs index ff0c71227..91b309074 100644 --- a/pgdog/src/frontend/client/query_engine/route_query.rs +++ b/pgdog/src/frontend/client/query_engine/route_query.rs @@ -162,6 +162,7 @@ impl QueryEngine { match projection::finalize_after_route( client_request, &cluster.schema(), + cluster.sharded_tables(), rewrite_result.and_then(RewriteResult::offset_plan), ) { Ok(()) => {} diff --git a/pgdog/src/frontend/client/query_engine/test/rewrite_offset.rs b/pgdog/src/frontend/client/query_engine/test/rewrite_offset.rs index 447c11bee..ec5210fad 100644 --- a/pgdog/src/frontend/client/query_engine/test/rewrite_offset.rs +++ b/pgdog/src/frontend/client/query_engine/test/rewrite_offset.rs @@ -1,3 +1,4 @@ +use crate::backend::ShardedTables; use crate::backend::schema::Schema; use crate::frontend::router::parser::Limit; use crate::frontend::router::parser::rewrite::statement::{ @@ -157,6 +158,7 @@ async fn test_offset_with_unique_id_simple() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), rewrite_result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -232,6 +234,7 @@ async fn test_offset_with_unique_id_extended() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), rewrite_result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -284,6 +287,7 @@ async fn split_anonymous_pagination_keeps_original_plan() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -318,6 +322,7 @@ async fn split_anonymous_pagination_keeps_original_plan() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); diff --git a/pgdog/src/frontend/client/query_engine/test/rewrite_projection.rs b/pgdog/src/frontend/client/query_engine/test/rewrite_projection.rs index 46859b856..4c172bfdf 100644 --- a/pgdog/src/frontend/client/query_engine/test/rewrite_projection.rs +++ b/pgdog/src/frontend/client/query_engine/test/rewrite_projection.rs @@ -1,3 +1,4 @@ +use crate::backend::ShardedTables; use crate::backend::schema::Schema; use crate::frontend::router::Ast; use crate::frontend::router::parser::rewrite::statement::plan::RewriteResult; @@ -46,6 +47,7 @@ async fn direct_aggregate_keeps_base_sql() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -76,6 +78,7 @@ async fn cross_shard_aggregate_adds_and_tracks_helpers() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -115,6 +118,7 @@ async fn cross_shard_having_rewrites_and_replaces_route_aggregate() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -158,6 +162,7 @@ async fn direct_having_keeps_native_postgres_semantics() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -201,6 +206,7 @@ async fn named_prepared_aggregate_uses_cross_shard_variant() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -267,6 +273,7 @@ async fn named_prepared_direct_aggregate_keeps_base_variant() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -313,6 +320,7 @@ async fn cross_shard_order_by_projects_missing_sort_column() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -364,6 +372,7 @@ async fn cross_shard_order_by_helper_keeps_nulls_placement() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -391,7 +400,13 @@ fn cached_projection_does_not_depend_on_first_route_order() { None, )); - projection::finalize_after_route(&mut first, &Schema::default(), None).unwrap(); + projection::finalize_after_route( + &mut first, + &Schema::default(), + &ShardedTables::default(), + None, + ) + .unwrap(); let first_query = match &first.messages[0] { ProtocolMessage::Query(query) => query, _ => panic!("expected Query"), @@ -419,7 +434,13 @@ fn cached_projection_does_not_depend_on_first_route_order() { None, )); - projection::finalize_after_route(&mut second, &Schema::default(), None).unwrap(); + projection::finalize_after_route( + &mut second, + &Schema::default(), + &ShardedTables::default(), + None, + ) + .unwrap(); assert_eq!( second.route().order_by(), &[ @@ -442,7 +463,13 @@ fn aliased_projected_sort_column_remaps_route() { None, )); - projection::finalize_after_route(&mut request, &Schema::default(), None).unwrap(); + projection::finalize_after_route( + &mut request, + &Schema::default(), + &ShardedTables::default(), + None, + ) + .unwrap(); let query = match &request.messages[0] { ProtocolMessage::Query(query) => query, @@ -477,7 +504,13 @@ fn duplicate_sort_column_names_use_injected_helper() { None, )); - projection::finalize_after_route(&mut request, &Schema::default(), None).unwrap(); + projection::finalize_after_route( + &mut request, + &Schema::default(), + &ShardedTables::default(), + None, + ) + .unwrap(); let query = match &request.messages[0] { ProtocolMessage::Query(query) => query, @@ -509,7 +542,13 @@ fn helper_replaces_the_matching_duplicate_order_by_position() { None, )); - projection::finalize_after_route(&mut request, &Schema::default(), None).unwrap(); + projection::finalize_after_route( + &mut request, + &Schema::default(), + &ShardedTables::default(), + None, + ) + .unwrap(); assert_eq!( request.route().order_by(), @@ -544,6 +583,7 @@ async fn aggregate_order_by_and_offset_compose_after_route() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -610,6 +650,7 @@ async fn split_anonymous_prepare_rewrites_each_execution_once() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -650,6 +691,7 @@ async fn split_anonymous_prepare_rewrites_each_execution_once() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -701,6 +743,7 @@ async fn named_statement_can_switch_from_direct_to_cross_shard_variant() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); @@ -729,6 +772,7 @@ async fn named_statement_can_switch_from_direct_to_cross_shard_variant() { projection::finalize_after_route( client_request, &Schema::default(), + &ShardedTables::default(), result.as_ref().and_then(RewriteResult::offset_plan), ) .unwrap(); diff --git a/pgdog/src/frontend/client/query_engine/test/sharded.rs b/pgdog/src/frontend/client/query_engine/test/sharded.rs index d4a980bc3..a494c3d1a 100644 --- a/pgdog/src/frontend/client/query_engine/test/sharded.rs +++ b/pgdog/src/frontend/client/query_engine/test/sharded.rs @@ -1,6 +1,6 @@ use crate::{ expect_message, - net::{CommandComplete, Parameters, Query, ReadyForQuery}, + net::{CommandComplete, DataRow, Parameters, Query, ReadyForQuery, RowDescription}, }; use super::prelude::*; @@ -255,3 +255,52 @@ async fn test_sharded_delete_returning_from_all_shards() { let cc = CommandComplete::try_from(cc_msg.clone()).unwrap(); assert_eq!(cc.command(), "DELETE 2"); } + +#[tokio::test] +async fn test_sharded_count_distinct_on_sharding_key_sums_shards() { + let mut client = TestClient::new_sharded(Parameters::default()).await; + + let id_shard0 = client.random_id_for_shard(0); + let id_shard1 = client.random_id_for_shard(1); + let ids = format!("{}, {}", id_shard0, id_shard1); + + // Cleanup first + client + .send_simple(Query::new(format!( + "DELETE FROM sharded WHERE id IN ({ids})" + ))) + .await; + client.read_until('Z').await.unwrap(); + + client + .send_simple(Query::new(format!( + "INSERT INTO sharded (id, value) VALUES ({}, 'cd1'), ({}, 'cd2')", + id_shard0, id_shard1 + ))) + .await; + client.read_until('Z').await.unwrap(); + + // Each shard counts its own key; no key is on both, so the counts add up. + client + .send_simple(Query::new(format!( + "SELECT count(DISTINCT id), count(DISTINCT id)::int FROM sharded WHERE id IN ({ids})" + ))) + .await; + expect_message!(client.read().await, RowDescription); + let row = expect_message!(client.read().await, DataRow); + assert_eq!(row.get_int(0, true), Some(2)); + assert_eq!(row.get_int(1, true), Some(2)); + assert_eq!( + expect_message!(client.read().await, CommandComplete).command(), + "SELECT 1" + ); + expect_message!(client.read().await, ReadyForQuery); + + // Cleanup + client + .send_simple(Query::new(format!( + "DELETE FROM sharded WHERE id IN ({ids})" + ))) + .await; + client.read_until('Z').await.unwrap(); +} diff --git a/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/mod.rs b/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/mod.rs index 119a8f468..f53e9ef02 100644 --- a/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/mod.rs +++ b/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/mod.rs @@ -1,11 +1,10 @@ mod engine; -// Used by the DISTINCT aggregate rewrite. -#[cfg_attr(not(test), expect(dead_code))] mod sharding_key; pub(crate) use super::projection::AggregateHelper; pub(crate) use engine::AggregatesRewrite; +pub(crate) use sharding_key::disjoint_distinct; /// Type of aggregate function added to the result set. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/sharding_key.rs b/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/sharding_key.rs index 83f358d52..24bfc61de 100644 --- a/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/sharding_key.rs +++ b/pgdog/src/frontend/router/parser/rewrite/statement/aggregate/sharding_key.rs @@ -8,10 +8,48 @@ use itertools::Itertools; use pg_raw_parse::{Node, nodes}; use crate::backend::replication::ShardedTables; -use crate::frontend::router::parser::{Column, Table}; +use crate::frontend::router::parser::{AggregateFunction, Column, Function, Table}; + +/// SELECT list positions of `count(DISTINCT key)`, where `key` is the sharding key. +/// +/// No two shards count the same key, so their counts add up to the distinct +/// count across all of them, with or without `GROUP BY`. +pub(crate) fn disjoint_distinct(select: &nodes::SelectStmt, tables: &ShardedTables) -> Vec { + // Resolved on the first DISTINCT aggregate; most queries have none. + let mut key = None; + + select + .target_list() + .iter() + .enumerate() + .filter(|(_, target)| { + let Some(call) = Function::extract_func_call(target.val()) else { + return false; + }; + let is_count = call + .funcname() + .iter() + .filter_map(Node::as_str) + .next_back() + .and_then(AggregateFunction::from_builtin_name) + == Some(AggregateFunction::Count); + if !call.agg_distinct || !is_count { + return false; + } + let Ok(argument) = call.args().iter().exactly_one() else { + return false; + }; + + key.get_or_insert_with(|| ShardingKey::new(select, tables)) + .as_ref() + .is_some_and(|key| key.matches(argument)) + }) + .map(|(position, _)| position) + .collect() +} /// Answers whether an expression is the sharding key of the queried table. -pub(crate) struct ShardingKey<'a> { +struct ShardingKey<'a> { table: Table<'a>, tables: &'a ShardedTables, } @@ -20,7 +58,7 @@ impl<'a> ShardingKey<'a> { /// `None` unless the SELECT reads exactly one plain table. Joins, /// subqueries and functions in `FROM` can repeat a key on several shards, /// and a CTE can shadow the table's name. - pub(crate) fn new(select: &'a nodes::SelectStmt, tables: &'a ShardedTables) -> Option { + fn new(select: &'a nodes::SelectStmt, tables: &'a ShardedTables) -> Option { if select.with_clause().is_some() { return None; } @@ -37,7 +75,7 @@ impl<'a> ShardingKey<'a> { /// Whether `node` is a bare reference to the sharding key column. /// Expressions over the key, e.g. `user_id % 2`, don't count: /// two shards can produce the same result. - pub(crate) fn matches(&self, node: Node<'_>) -> bool { + fn matches(&self, node: Node<'_>) -> bool { let Node::ColumnRef(column) = node else { return false; }; @@ -55,7 +93,9 @@ impl<'a> ShardingKey<'a> { /// Whether `GROUP BY` includes the sharding key, by name or by SELECT /// list position, so that no group spans shards. - pub(crate) fn in_group_by(&self, select: &nodes::SelectStmt) -> bool { + // Not called yet: the NULL key group spans shards and needs its own handling. + #[cfg_attr(not(test), expect(dead_code))] + fn in_group_by(&self, select: &nodes::SelectStmt) -> bool { select.group_clause().iter().any(|item| match item { // Postgres counts SELECT list positions from 1. Node::A_Const(ordinal) => ordinal @@ -122,6 +162,42 @@ mod tests { Some((key.matches(first.val()), key.in_group_by(&select))) } + #[test] + fn finds_count_distinct_on_key() { + let tables = orders_by_user(); + + for (sql, expected) in [ + ("SELECT count(DISTINCT user_id) FROM orders", vec![0]), + ( + "SELECT count(DISTINCT o.user_id)::int FROM orders o", + vec![0], + ), + ( + "SELECT status, count(DISTINCT user_id), count(*) FROM orders GROUP BY status", + vec![1], + ), + ( + "SELECT count(DISTINCT user_id) FILTER (WHERE status = 'paid') FROM orders", + vec![0], + ), + ("SELECT count(user_id) FROM orders", vec![]), + ("SELECT count(DISTINCT product_id) FROM orders", vec![]), + ("SELECT count(DISTINCT user_id % 2) FROM orders", vec![]), + ( + "SELECT count(DISTINCT (user_id, status)) FROM orders", + vec![], + ), + // Summing the keys of each shard is a later change. + ("SELECT sum(DISTINCT user_id) FROM orders", vec![]), + ( + "SELECT count(DISTINCT user_id) FROM orders JOIN users ON users.id = orders.user_id", + vec![], + ), + ] { + assert_eq!(disjoint_distinct(&select(sql), &tables), expected, "{sql}"); + } + } + #[test] fn matches_bare_key_column() { let tables = orders_by_user(); diff --git a/pgdog/src/frontend/router/parser/rewrite/statement/projection.rs b/pgdog/src/frontend/router/parser/rewrite/statement/projection.rs index 96487d580..6983a6af2 100644 --- a/pgdog/src/frontend/router/parser/rewrite/statement/projection.rs +++ b/pgdog/src/frontend/router/parser/rewrite/statement/projection.rs @@ -5,11 +5,12 @@ //! AST only after routing and cached independently from per-execution Bind values. use super::Error; -use super::aggregate::{AggregatesRewrite, HelperKind}; +use super::aggregate::{AggregatesRewrite, HelperKind, disjoint_distinct}; use super::group_by::{self, GroupByRewrite}; use super::having; use super::offset::{self, OffsetPlan}; use super::order_by; +use crate::backend::ShardedTables; use crate::backend::schema::Schema; use crate::frontend::router::parser::{ Aggregate, AggregateFunction, Function, HavingRewritePlan, OrderBy, aggregate::GroupByKey, @@ -64,6 +65,9 @@ pub(crate) struct ProjectionRewritePlan { pub(crate) order_by_helpers: Vec, pub(crate) having: Option>, pub(crate) group_by: Option>, + /// SELECT list positions of `DISTINCT` aggregates over the sharding key. + /// Shards never share a key, so these merge like the plain aggregate. + pub(crate) disjoint_distinct: Vec, } /// The filter and aggregate metadata produced by rewriting a HAVING query. @@ -97,6 +101,7 @@ impl ProjectionRewritePlan { && self.order_by_helpers.is_empty() && self.having.is_none() && self.group_by.is_none() + && self.disjoint_distinct.is_empty() } /// The aggregate of the rewritten query, if it differs from the client's. @@ -141,6 +146,7 @@ pub(crate) struct PostRouteRewrite { pub(crate) fn finalize_after_route( request: &mut ClientRequest, schema: &Schema, + tables: &ShardedTables, offset_plan: Option<&OffsetPlan>, ) -> Result<(), Error> { if !request.route().is_cross_shard() { @@ -153,7 +159,7 @@ pub(crate) fn finalize_after_route( let rewrite_offset = offset_plan.is_some_and(|plan| !plan.prepare_execute); let Some(rewrite) = ast .post_route_rewrite - .get_or_try_init(|| build(&ast.ast, schema, rewrite_offset))? + .get_or_try_init(|| build(&ast.ast, schema, tables, rewrite_offset))? else { return Ok(()); }; @@ -279,6 +285,7 @@ pub(crate) fn pages_merged_groups(select: &nodes::SelectStmt, aggregate: &Aggreg fn build( ast: &StmtList, schema: &Schema, + tables: &ShardedTables, rewrite_offset: bool, ) -> Result, Error> { let Some(Node::SelectStmt(select)) = ast.stmts().next() else { @@ -330,7 +337,10 @@ fn build( return Err(Error::OrderByExpressionUnsupported); } - let mut plan = ProjectionRewritePlan::default(); + let mut plan = ProjectionRewritePlan { + disjoint_distinct: disjoint_distinct(select, tables), + ..Default::default() + }; let rewritten = make::try_owned(|mem| { let mut select = mem.make_unique(select); if has_having { @@ -457,13 +467,51 @@ mod tests { use super::*; use crate::backend::schema::relation::Relation; + use crate::frontend::router::sharding::ShardedTable; + + #[test] + fn count_distinct_on_sharding_key_is_planned() { + let tables = ShardedTables::from( + [ShardedTable { + name: Some("items".into()), + column: "id".into(), + ..Default::default() + }] + .as_slice(), + ); + + // (query, sharded tables, SELECT list positions that merge as a plain count) + for (sql, tables, expected) in [ + ("SELECT count(DISTINCT id) FROM items", &tables, vec![0]), + ( + "SELECT value, count(DISTINCT id) FROM items GROUP BY value HAVING count(*) > 1", + &tables, + vec![1], + ), + ("SELECT count(DISTINCT value) FROM items", &tables, vec![]), + ( + "SELECT count(DISTINCT id) FROM items", + &ShardedTables::default(), + vec![], + ), + ] { + let ast = pg_raw_parse::parse(sql).unwrap(); + let rewrite = build(&ast, &Schema::default(), tables, false).unwrap(); + let marked = rewrite + .as_ref() + .and_then(|rewrite| rewrite.plan.as_deref()) + .map(|plan| plan.disjoint_distinct.as_slice()) + .unwrap_or_default(); + assert_eq!(marked, expected, "{sql}"); + } + } #[test] fn rejects_count_sort_key() { let ast = pg_raw_parse::parse("SELECT value FROM items GROUP BY value ORDER BY count(*)") .unwrap(); - let rewrite_result = build(&ast, &Schema::default(), false); + let rewrite_result = build(&ast, &Schema::default(), &ShardedTables::default(), false); assert!(matches!( rewrite_result, @@ -504,7 +552,7 @@ mod tests { ), ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let rewrite = build(&ast, &Schema::default(), false) + let rewrite = build(&ast, &Schema::default(), &ShardedTables::default(), false) .unwrap() .expect("GROUP BY requires a rewrite"); assert_eq!(&*rewrite.sql, shard_sql, "{sql}"); @@ -520,7 +568,9 @@ mod tests { fn hidden_group_by_keys_are_dropped() { let ast = pg_raw_parse::parse("SELECT category FROM items GROUP BY category, region").unwrap(); - let rewrite = build(&ast, &Schema::default(), false).unwrap().unwrap(); + let rewrite = build(&ast, &Schema::default(), &ShardedTables::default(), false) + .unwrap() + .unwrap(); let row_description = RowDescription::new(&[ crate::net::Field::text("category"), crate::net::Field::text("__pgdog_group_0"), @@ -539,7 +589,9 @@ mod tests { ] { let ast = pg_raw_parse::parse(sql).unwrap(); assert!( - build(&ast, &Schema::default(), false).unwrap().is_none(), + build(&ast, &Schema::default(), &ShardedTables::default(), false) + .unwrap() + .is_none(), "{sql}" ); } @@ -554,16 +606,22 @@ mod tests { "SELECT category, count(*) FROM items GROUP BY GROUPING SETS ((category), ())", ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let error = build(&ast, &Schema::default(), false).unwrap_err(); + let error = + build(&ast, &Schema::default(), &ShardedTables::default(), false).unwrap_err(); assert!(matches!(error, Error::UnsupportedGroupBy(_)), "{sql}"); } } fn shard_sql(sql: &str, rewrite_offset: bool) -> Option { let ast = pg_raw_parse::parse(sql).unwrap(); - build(&ast, &Schema::default(), rewrite_offset) - .unwrap() - .map(|rewrite| rewrite.sql.to_string()) + build( + &ast, + &Schema::default(), + &ShardedTables::default(), + rewrite_offset, + ) + .unwrap() + .map(|rewrite| rewrite.sql.to_string()) } #[test] @@ -657,7 +715,8 @@ mod tests { "SELECT category, COUNT(*) FROM items GROUP BY category LIMIT $1::numeric", ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let error = build(&ast, &Schema::default(), false).unwrap_err(); + let error = + build(&ast, &Schema::default(), &ShardedTables::default(), false).unwrap_err(); assert!(matches!(error, Error::UnsupportedGroupBy(_)), "{sql}"); } } @@ -713,7 +772,7 @@ mod tests { FROM purchases GROUP BY __pgdog_having_0 HAVING COUNT(*) > 2", ) .unwrap(); - let rewrite = build(&ast, &Schema::default(), false) + let rewrite = build(&ast, &Schema::default(), &ShardedTables::default(), false) .unwrap() .expect("HAVING requires a rewrite"); @@ -758,7 +817,7 @@ mod tests { ), ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let rewrite = build(&ast, &Schema::default(), false) + let rewrite = build(&ast, &Schema::default(), &ShardedTables::default(), false) .unwrap() .expect("HAVING requires a rewrite"); let plan = rewrite.plan.expect("projection plan"); @@ -819,7 +878,7 @@ mod tests { let Some(Node::SelectStmt(select)) = ast.stmts().next() else { panic!("expected SELECT") }; - let rewrite = build(&ast, &schema, false).unwrap(); + let rewrite = build(&ast, &schema, &ShardedTables::default(), false).unwrap(); assert_eq!( rewrite.as_ref().map(|rewrite| rewrite.sql.as_ref()), shard_sql, @@ -860,7 +919,7 @@ mod tests { ) .unwrap(); assert!(matches!( - build(&ast, &Schema::default(), false), + build(&ast, &Schema::default(), &ShardedTables::default(), false), Err(Error::UnsupportedGroupBy(_)) )); } @@ -896,7 +955,7 @@ mod tests { ), ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let error = build(&ast, schema, false).unwrap_err(); + let error = build(&ast, schema, &ShardedTables::default(), false).unwrap_err(); assert!( matches!( error, @@ -926,7 +985,7 @@ mod tests { ), ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let rewrite = build(&ast, &schema, false) + let rewrite = build(&ast, &schema, &ShardedTables::default(), false) .unwrap() .expect("HAVING requires a rewrite"); assert_eq!(rewrite.sql.as_ref(), shard_sql, "{sql}"); @@ -958,7 +1017,8 @@ mod tests { HAVING COUNT(*) > 2 LIMIT (SELECT 1)", ] { let ast = pg_raw_parse::parse(sql).unwrap(); - let error = build(&ast, &Schema::default(), false).unwrap_err(); + let error = + build(&ast, &Schema::default(), &ShardedTables::default(), false).unwrap_err(); assert!(matches!(error, Error::UnsupportedHaving(_)), "{sql}"); } }