From e4088619dee5cc877a771eae85a3230c1b7f42e7 Mon Sep 17 00:00:00 2001 From: englefly Date: Tue, 18 Aug 2026 20:01:43 +0800 Subject: [PATCH 1/3] branch-4.1 [fix](fe) Propagate constants out of constant CTE to enable partition pruning Problem Summary: A CTE that defines a single constant row (e.g. `WITH params AS (SELECT CAST(...) AS begin_time, ...)`) is inlined into its consumers as a one-row relation, but the constant projects were only registered as uniform slots without values. Constant propagation and predicate inference outside the relation therefore could not substitute the constant values into predicates over the CTE columns, so predicates like `dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY) AND DATE_SUB(params.begin_time, INTERVAL 1 DAY)` were not folded, the `dt` predicates were left as nested-loop-join conjuncts with runtime filters, and partition pruning failed: the scan read all 526/537 partitions instead of the single needed partition. Fix: `LogicalOneRowRelation.computeUniform` now registers the values of constant projects as uniform constants, so constant propagation can substitute them into predicates over the CTE columns, fold functions such as DATE_SUB over them, and push the resulting predicates into the scan for partition pruning. None - Test: FE unit test ConstantCteTest (asserts both scans prune to the expected single partition); regression test test_constant_cte_partition_prune. - Behavior changed: No - Does this need documentation: No --- .../plans/logical/LogicalOneRowRelation.java | 13 +- .../rules/rewrite/ConstantCteTest.java | 121 ++++++++++++++++++ .../test_constant_cte_partition_prune.groovy | 69 ++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java create mode 100644 regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 5f8446fb0153ab..e2584b04fb0393 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -169,7 +169,18 @@ public void computeUnique(DataTrait.Builder builder) { @Override public void computeUniform(DataTrait.Builder builder) { - getOutput().forEach(builder::addUniformSlot); + for (NamedExpression project : getProjects()) { + if (project instanceof Alias && project.child(0).isConstant()) { + // A one row relation's constant projects are known literals/constant expressions, + // register the value so that constant propagation and predicate inference outside + // the relation can fold predicates over these slots (e.g. a constant CTE whose + // consumers reference `date_sub(params.begin_time, ...)`), which enables + // partition pruning on the referenced tables. + builder.addUniformSlotAndLiteral(project.toSlot(), project.child(0)); + } else { + builder.addUniformSlot(project.toSlot()); + } + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java new file mode 100644 index 00000000000000..cc6242a0fc6d57 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; + +/** + * Test that a CTE defining a constant single row (e.g. `params`) propagates the constants + * out, so that downstream predicates over the CTE columns can be constant-folded and the + * partition pruning on the referenced tables works. + * + *

After the constant CTE is inlined, the first consumer predicate + * `dt BETWEEN params.begin_time AND params.end_time` is folded and the scan prunes to the + * day partition; the second consumer predicate + * `dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY) + * AND DATE_SUB(params.begin_time, INTERVAL 1 DAY)` + * was not folded, so the scan read all partitions. + */ +class ConstantCteTest extends TestWithFeService implements MemoPatternMatchSupported { + + // the constant CTE `params` participates in two joins; the second join uses + // DATE_SUB() over the CTE columns, which needs constant folding to prune partitions. + private static final String SQL = "WITH params AS (\n" + + " SELECT\n" + + " CAST('2026-07-28 00:00:00' AS DATETIME) AS begin_time,\n" + + " CAST('2026-07-28 23:59:59' AS DATETIME) AS end_time,\n" + + " DATEDIFF(CAST('2026-07-28 23:59:59' AS DATETIME), " + + "CAST('2026-07-28 00:00:00' AS DATETIME)) + 1 AS period_days\n" + + "),\n" + + "current_data AS (\n" + + " SELECT SUM(v) AS total_value\n" + + " FROM cte_prune_t\n" + + " JOIN params ON 1=1\n" + + " WHERE dt BETWEEN params.begin_time AND params.end_time\n" + + "),\n" + + "last_period_data AS (\n" + + " SELECT SUM(v) AS total_value\n" + + " FROM cte_prune_t\n" + + " JOIN params ON 1=1\n" + + " WHERE dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY)\n" + + " AND DATE_SUB(params.begin_time, INTERVAL 1 DAY)\n" + + ")\n" + + "SELECT * FROM current_data, last_period_data"; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + createTable("CREATE TABLE `test`.`cte_prune_t` (\n" + + " `dt` DATE NULL COMMENT \"\",\n" + + " `sn` VARCHAR(50) NULL COMMENT \"\",\n" + + " `v` DOUBLE NULL COMMENT \"\"\n" + + ") DUPLICATE KEY(`dt`, `sn`)\n" + + "PARTITION BY RANGE(`dt`)\n" + + "(PARTITION p20260101 VALUES [(\"2026-01-01\"), (\"2026-01-02\")),\n" + + " PARTITION p20260726 VALUES [(\"2026-07-26\"), (\"2026-07-27\")),\n" + + " PARTITION p20260727 VALUES [(\"2026-07-27\"), (\"2026-07-28\")),\n" + + " PARTITION p20260728 VALUES [(\"2026-07-28\"), (\"2026-07-29\")),\n" + + " PARTITION p20260729 VALUES [(\"2026-07-29\"), (\"2026-07-30\")),\n" + + " PARTITION p20260901 VALUES [(\"2026-09-01\"), (\"2026-09-02\")))\n" + + "DISTRIBUTED BY HASH(`sn`) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + FeConstants.runningUnitTest = true; + } + + @Test + void testConstantCteFoldJoinPredicateAndPrunePartition() { + // params has 2 consumers; force inline to match the reported scenario + connectContext.getSessionVariable().inlineCTEReferencedThreshold = 2; + + PlanChecker planChecker = PlanChecker.from(connectContext) + .analyze(SQL) + .rewrite(); + Plan plan = planChecker.getCascadesContext().getRewritePlan(); + String planString = plan.treeString(); + + List scans = plan.collectToList(LogicalOlapScan.class::isInstance); + Assertions.assertEquals(2, scans.size(), + "both current_data and last_period_data should scan cte_prune_t, plan: " + planString); + Set selectedPartitions = Sets.newHashSet(); + for (LogicalOlapScan scan : scans) { + // current_data only needs p20260728, last_period_data only needs p20260727; + // both must prune to exactly one partition + Assertions.assertEquals(1, scan.getSelectedPartitionIds().size(), + "scan on cte_prune_t should prune to exactly one partition, plan: " + planString); + selectedPartitions.add(((OlapTable) scan.getTable()) + .getPartition(scan.getSelectedPartitionIds().get(0)).getName()); + } + Assertions.assertEquals(Sets.newHashSet("p20260727", "p20260728"), selectedPartitions, + "current_data should prune to p20260728 and last_period_data to p20260727, plan: " + + planString); + } +} diff --git a/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy new file mode 100644 index 00000000000000..94db5e5e07b8ba --- /dev/null +++ b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_constant_cte_partition_prune") { + sql "DROP TABLE IF EXISTS constant_cte_prune_t" + sql """ + CREATE TABLE constant_cte_prune_t ( + dt DATE, + sn VARCHAR(50), + v DOUBLE + ) ENGINE=OLAP + DUPLICATE KEY(dt, sn) + PARTITION BY RANGE(dt) + (PARTITION p20260101 VALUES [("2026-01-01"), ("2026-01-02")), + PARTITION p20260726 VALUES [("2026-07-26"), ("2026-07-27")), + PARTITION p20260727 VALUES [("2026-07-27"), ("2026-07-28")), + PARTITION p20260728 VALUES [("2026-07-28"), ("2026-07-29")), + PARTITION p20260729 VALUES [("2026-07-29"), ("2026-07-30")), + PARTITION p20260901 VALUES [("2026-09-01"), ("2026-09-02"))) + DISTRIBUTED BY HASH(sn) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + """ + + // force the constant CTE `params` to be inlined (it is referenced twice), matching the + // reported scenario; after inlining the constants must propagate out so that predicates + // over the CTE columns (including DATE_SUB() over them) are folded and the scans prune + // to a single day partition + sql "SET inline_cte_referenced_threshold=2" + + explain { + sql """ + WITH params AS ( + SELECT CAST('2026-07-28 00:00:00' AS DATETIME) AS begin_time, + CAST('2026-07-28 23:59:59' AS DATETIME) AS end_time, + DATEDIFF(CAST('2026-07-28 23:59:59' AS DATETIME), + CAST('2026-07-28 00:00:00' AS DATETIME)) + 1 AS period_days + ), + current_data AS ( + SELECT SUM(v) AS total_value + FROM constant_cte_prune_t JOIN params ON 1=1 + WHERE dt BETWEEN params.begin_time AND params.end_time + ), + last_period_data AS ( + SELECT SUM(v) AS total_value + FROM constant_cte_prune_t JOIN params ON 1=1 + WHERE dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY) + AND DATE_SUB(params.begin_time, INTERVAL 1 DAY) + ) + SELECT * FROM current_data, last_period_data + """ + // current_data only needs 2026-07-28, last_period_data only needs 2026-07-27 + contains("partitions=1/6 (p20260728)") + contains("partitions=1/6 (p20260727)") + } +} From 614ba8ff75b3a66dd32bae0e166950bb952ef401 Mon Sep 17 00:00:00 2001 From: englefly Date: Fri, 21 Aug 2026 13:34:31 +0800 Subject: [PATCH 2/3] branch-4.1 [fix](feut) Update PruneNestedColumnTest#testPushDownThroughJoin for one-row-relation constant propagation ### What problem does this PR solve? Problem Summary: LogicalOneRowRelation.computeUniform now registers the values of a one-row relation's constant projects as uniform literals (constant CTE propagation). For the query `select coalesce(element_at(s, 'city'), 'abc') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id`, the join condition `a.id = b.id` now folds to `a.id = 100`, the predicate is pushed into a filter above the scan, and the join becomes a cross join. The left project therefore no longer needs to output the join key `id` and keeps only the pushed-down nested column access `element_at(s, 'city')` (1 project instead of 2). The old test asserted the previous plan shape and failed; update the assertion to the new expected plan, which is semantically equivalent and enables pushing `id = 100` into the scan. ### Release note None ### Check List (For Author) - Test: PruneNestedColumnTest (50/50) and ConstantCteTest pass - Behavior changed: No - Does this need documentation: No --- .../nereids/rules/rewrite/PruneNestedColumnTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 725c0c4cd082b7..4d7a8a4488e3b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -778,9 +778,12 @@ public void testPushDownThroughJoin() { logicalOlapScan() ) ).when(p -> { - Assertions.assertEquals(2, p.getProjects().size()); - Assertions.assertTrue(p.getProjects().stream() - .anyMatch(o -> o instanceof Alias && o.child(0) instanceof ElementAt)); + // the one-row relation's constant `id` is propagated into the + // left side (`id = 100` pushed into the filter below), so the + // project above the filter only keeps the pushed-down access + Assertions.assertEquals(1, p.getProjects().size()); + Assertions.assertTrue(p.getProjects().get(0) instanceof Alias + && p.getProjects().get(0).child(0) instanceof ElementAt); return true; }), logicalOneRowRelation() From 8aced3460a56f965f5fdaee22ca1715d7318bf08 Mon Sep 17 00:00:00 2001 From: englefly Date: Fri, 21 Aug 2026 13:56:29 +0800 Subject: [PATCH 3/3] branch-4.1 [test](regression) Fix regression tests broken by one-row-relation constant propagation ### What problem does this PR solve? Problem Summary: - pull_up_predicate_literal: after LogicalOneRowRelation.computeUniform registers the values of a one-row relation's constant projects as uniform literals, the join condition `tmp.col1 = ds.col1 AND tmp.col2 = ds.col2` (tmp being a one-row relation with the literals 'abc'/'def') is folded into `ds.col1 = 'abc' AND ds.col2 = 'def'` and pushed into a filter below the scan; the join itself keeps no condition and becomes a nested loop join. Regenerate the .out file for the two view-based queries; the remaining queries in the suite use project-over-scan constant subqueries and are unaffected. - test_constant_cte_partition_prune: the new test explains the expected single-partition pruning of a constant CTE, but on an empty table PRUNE_EMPTY_PARTITION rewrites the scans into empty relations, so the `partitions=1/6 (p20260728)` info never appears. Seed each partition with one row so the scans stay and the asserted pruning shows up. ### Release note None ### Check List (For Author) - Test: regression tests pull_up_predicate_literal and test_constant_cte_partition_prune pass - Behavior changed: No - Does this need documentation: No --- .../infer_predicate/pull_up_predicate_literal.out | 4 ++-- .../cte/test_constant_cte_partition_prune.groovy | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out index adc6bb8087c0bf..e1baa52072671b 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out @@ -2,7 +2,7 @@ -- !test_pull_up_literal1 -- PhysicalResultSink --PhysicalProject -----hashJoin[INNER_JOIN] hashCondition=((col1 = ds.col1) and (col2 = ds.col2)) otherCondition=() +----NestedLoopJoin[INNER_JOIN] ------PhysicalOneRowRelation ------filter((ds.col1 = 'abc') and (ds.col2 = 'def')) --------PhysicalOlapScan[test_pull_up_predicate_literal] @@ -10,7 +10,7 @@ PhysicalResultSink -- !test_pull_up_literal2 -- PhysicalResultSink --PhysicalProject -----hashJoin[INNER_JOIN] hashCondition=((col1 = ds.col1) and (col2 = ds.col2)) otherCondition=() +----NestedLoopJoin[INNER_JOIN] ------PhysicalOneRowRelation ------filter((ds.col1 = 'abc') and (ds.col2 = 'def')) --------PhysicalOlapScan[test_pull_up_predicate_literal] diff --git a/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy index 94db5e5e07b8ba..be08e75674796b 100644 --- a/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy +++ b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy @@ -41,6 +41,17 @@ suite("test_constant_cte_partition_prune") { // to a single day partition sql "SET inline_cte_referenced_threshold=2" + // seed every partition so that PRUNE_EMPTY_PARTITION does not eliminate the scans + // (an all-empty table would rewrite the scan to an empty relation and the + // partition-pruning info below would not show up in the explain output) + sql """INSERT INTO constant_cte_prune_t VALUES + ('2026-01-01', 'sn1', 1.0), + ('2026-07-26', 'sn2', 2.0), + ('2026-07-27', 'sn3', 3.0), + ('2026-07-28', 'sn4', 4.0), + ('2026-07-29', 'sn5', 5.0), + ('2026-09-01', 'sn6', 6.0)""" + explain { sql """ WITH params AS (