Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions integration/sql/cases/958_count_distinct_sharding_key_case.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
-- description: count(DISTINCT sharding key) adds up each shard's count
-- tags: standard sharded
-- transactional: true

SELECT count(DISTINCT id) FROM sql_regression_samples;

SELECT count(DISTINCT id)::int, count(*) FROM sql_regression_samples;

SELECT count(DISTINCT sql_regression_samples.id) FROM sql_regression_samples;

SELECT count(DISTINCT s.id) FROM sql_regression_samples s WHERE amount > 5;

SELECT grp, count(DISTINCT id) FROM sql_regression_samples GROUP BY grp ORDER BY grp;

SELECT grp, count(DISTINCT id), sum(amount) FROM sql_regression_samples
GROUP BY grp HAVING count(*) > 1 ORDER BY grp;

SELECT count(DISTINCT id) FILTER (WHERE amount >= 10) FROM sql_regression_samples;

SELECT count(DISTINCT id) FROM sql_regression_samples WHERE id < 0;
10 changes: 10 additions & 0 deletions integration/sql/cases/958_count_distinct_sharding_key_setup.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
DROP TABLE IF EXISTS sql_regression_samples;
CREATE TABLE sql_regression_samples (id BIGINT PRIMARY KEY, grp TEXT, amount INTEGER);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (1, 'a', 10);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (2, 'a', 10);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (101, 'a', 20);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (3, 'b', 12);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (102, 'b', NULL);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (4, NULL, 5);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (103, NULL, 6);
INSERT INTO sql_regression_samples (id, grp, amount) VALUES (104, 'c', 30);
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE IF EXISTS sql_regression_samples;
82 changes: 76 additions & 6 deletions pgdog/src/backend/pool/connection/aggregate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ impl<'a> Accumulator<'a> {
pub(crate) fn from_aggregate(
aggregate: &'a Aggregate,
helpers: &HashMap<usize, HelperColumns>,
disjoint_distinct: &[usize],
) -> Result<Vec<Self>, Error> {
aggregate
.targets()
Expand All @@ -80,7 +81,11 @@ impl<'a> Accumulator<'a> {

let accumulator = Accumulator {
target,
state: State::new(target, helper)?,
state: State::new(
target,
helper,
disjoint_distinct.contains(&target.column()),
)?,
};

Ok(accumulator)
Expand Down Expand Up @@ -112,8 +117,18 @@ enum State {
impl State {
/// Construct a new aggregate state based on the function provided
/// Errors if the function is not one we support.
fn new(target: &AggregateTarget, helper: HelperColumns) -> Result<Self, Error> {
match (target.function(), target.is_distinct()) {
///
/// `disjoint_distinct` is set for a `DISTINCT` aggregate over the sharding
/// key: shards never share a key, so it merges like the plain aggregate.
fn new(
target: &AggregateTarget,
helper: HelperColumns,
disjoint_distinct: bool,
) -> Result<Self, Error> {
match (
target.function(),
target.is_distinct() && !disjoint_distinct,
) {
(AggregateFunction::Avg, false) => Ok(Self::Avg(avg::Avg::new(
target.column(),
helper.count.ok_or_else(|| Error::UnsupportedAggregation {
Expand Down Expand Up @@ -263,14 +278,15 @@ pub(super) struct Aggregates<'a> {
decoder: &'a Decoder,
aggregate: &'a Aggregate,
helper_columns: HashMap<usize, HelperColumns>,
disjoint_distinct: &'a [usize],
}

impl<'a> Aggregates<'a> {
pub(super) fn new(
rows: &'a VecDeque<DataRow>,
decoder: &'a Decoder,
aggregate: &'a Aggregate,
plan: Option<&ProjectionRewritePlan>,
plan: Option<&'a ProjectionRewritePlan>,
) -> Option<Self> {
let mut helper_columns: HashMap<usize, HelperColumns> = HashMap::new();

Expand Down Expand Up @@ -333,6 +349,9 @@ impl<'a> Aggregates<'a> {
mappings: HashMap::new(),
aggregate,
helper_columns,
disjoint_distinct: plan
.map(|plan| plan.disjoint_distinct.as_slice())
.unwrap_or_default(),
})
} else {
None
Expand All @@ -359,8 +378,11 @@ impl<'a> Aggregates<'a> {
let entry = match self.mappings.entry(grouping) {
Entry::Occupied(o) => o.into_mut(),
Entry::Vacant(v) => {
let accumulators =
Accumulator::from_aggregate(self.aggregate, &self.helper_columns)?;
let accumulators = Accumulator::from_aggregate(
self.aggregate,
&self.helper_columns,
self.disjoint_distinct,
)?;

// Gather all col vals corresponding to list of passthrough indices.
let mut passthrough = Vec::new();
Expand Down Expand Up @@ -581,6 +603,54 @@ mod test {
assert_eq!(total_count, 5);
}

#[test]
fn aggregate_count_distinct_on_sharding_key_sums_shard_counts() {
let aggregate = parse("SELECT grp, COUNT(DISTINCT id)::int FROM users GROUP BY grp");
let decoder = Decoder::from(RowDescription::new(&[
integer_field("grp"),
integer_field("count"),
]));

// (grp, distinct ids on the shard)
let rows = [("1", "2"), ("2", "1"), ("1", "3")]
.into_iter()
.map(|(grp, count)| {
let mut row = DataRow::new();
row.add(grp).add(count);
row
})
.collect::<VecDeque<_>>();

// Shards may share values of any other column, so the counts can't be added.
let unplanned = Aggregates::new(&rows, &decoder, &aggregate, None)
.unwrap()
.aggregate();
assert!(matches!(
unplanned,
Err(Error::UnsupportedAggregation { .. })
));

let plan = ProjectionRewritePlan {
disjoint_distinct: vec![1],
..Default::default()
};
let merged = Aggregates::new(&rows, &decoder, &aggregate, Some(&plan))
.unwrap()
.aggregate()
.unwrap();
let mut counts = merged
.iter()
.map(|row| {
(
row.get::<i32>(0, Format::Text).unwrap(),
row.get::<i32>(1, Format::Text).unwrap(),
)
})
.collect::<Vec<_>>();
counts.sort_unstable();
assert_eq!(counts, [(1, 5), (2, 1)]);
}

#[test]
fn aggregate_count_preserves_binary_result_types() {
use pgdog_postgres_types::{Float, Numeric};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ impl<'a> SubqueryMulti<'a> {
projection::finalize_after_route(
&mut client_request,
&cluster.schema(),
cluster.sharded_tables(),
rewrite_result.offset_plan(),
)?;

Expand Down
1 change: 1 addition & 0 deletions pgdog/src/frontend/client/query_engine/route_query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ impl QueryEngine {
match projection::finalize_after_route(
client_request,
&cluster.schema(),
cluster.sharded_tables(),
rewrite_result.and_then(RewriteResult::offset_plan),
) {
Ok(()) => {}
Expand Down
5 changes: 5 additions & 0 deletions pgdog/src/frontend/client/query_engine/test/rewrite_offset.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::backend::ShardedTables;
use crate::backend::schema::Schema;
use crate::frontend::router::parser::Limit;
use crate::frontend::router::parser::rewrite::statement::{
Expand Down Expand Up @@ -157,6 +158,7 @@ async fn test_offset_with_unique_id_simple() {
projection::finalize_after_route(
client_request,
&Schema::default(),
&ShardedTables::default(),
rewrite_result.as_ref().and_then(RewriteResult::offset_plan),
)
.unwrap();
Expand Down Expand Up @@ -232,6 +234,7 @@ async fn test_offset_with_unique_id_extended() {
projection::finalize_after_route(
client_request,
&Schema::default(),
&ShardedTables::default(),
rewrite_result.as_ref().and_then(RewriteResult::offset_plan),
)
.unwrap();
Expand Down Expand Up @@ -284,6 +287,7 @@ async fn split_anonymous_pagination_keeps_original_plan() {
projection::finalize_after_route(
client_request,
&Schema::default(),
&ShardedTables::default(),
result.as_ref().and_then(RewriteResult::offset_plan),
)
.unwrap();
Expand Down Expand Up @@ -318,6 +322,7 @@ async fn split_anonymous_pagination_keeps_original_plan() {
projection::finalize_after_route(
client_request,
&Schema::default(),
&ShardedTables::default(),
result.as_ref().and_then(RewriteResult::offset_plan),
)
.unwrap();
Expand Down
Loading
Loading