From 567e775de2999d2ffb39f4d0a71c111312b824a6 Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 2 Sep 2026 08:32:56 +0800 Subject: [PATCH] [fix](fe) Reject conflicting path partition columns ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: A file table-valued function could append a path partition column whose name already existed in the inferred file schema. The duplicate schema made direct column references ambiguous and allowed SELECT * to reach an internal backend error. Reject case-insensitive conflicts while constructing the schema so the invalid configuration fails during analysis. ### Release note File table-valued functions now reject path partition columns that conflict with existing file columns. ### Check List (For Author) - Test: Regression test and manual test - Regression test: test_path_partition_column_collision - Manual test: DESC FUNCTION and SELECT * with a conflicting local CSV path partition - Behavior changed: Yes. Conflicting path partition columns now return a clear analysis error. - Does this need documentation: No --- .../ExternalFileTableValuedFunction.java | 5 +++ .../partition_col=from_path/data.csv | 3 ++ ...est_path_partition_column_collision.groovy | 37 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 regression-test/data/external_table_p0/tvf/test_path_partition_column_collision/partition_col=from_path/data.csv create mode 100644 regression-test/suites/external_table_p0/tvf/test_path_partition_column_collision.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 01abd8b482123e..879dd3a072e060 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -481,6 +481,11 @@ private void fillColumns(InternalService.PFetchTableSchemaResult result) { // HACK(tsy): path columns are all treated as STRING type now, after BE supports reading all columns // types by all format readers from file meta, maybe reading path columns types from BE then. for (String colName : pathPartitionKeys) { + String colLowerName = colName.toLowerCase(); + if (!columnLowerNames.add(colLowerName)) { + throw new NotSupportedException( + "Path partition column conflicts with an existing column: " + colName); + } columns.add(new Column(colName, ScalarType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH), false)); } } diff --git a/regression-test/data/external_table_p0/tvf/test_path_partition_column_collision/partition_col=from_path/data.csv b/regression-test/data/external_table_p0/tvf/test_path_partition_column_collision/partition_col=from_path/data.csv new file mode 100644 index 00000000000000..5ea1ce60c0d60d --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_path_partition_column_collision/partition_col=from_path/data.csv @@ -0,0 +1,3 @@ +partition_col,value +from_file,1 +from_file,2 diff --git a/regression-test/suites/external_table_p0/tvf/test_path_partition_column_collision.groovy b/regression-test/suites/external_table_p0/tvf/test_path_partition_column_collision.groovy new file mode 100644 index 00000000000000..f0a103dee26595 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_path_partition_column_collision.groovy @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_path_partition_column_collision", "p0,external") { + String ak = getS3AK() + String sk = getS3SK() + String s3Endpoint = getS3Endpoint() + String bucket = context.config.otherConfigs.get("s3BucketName") + + test { + sql """ + desc function s3( + "URI" = "https://${bucket}.${s3Endpoint}/regression/tvf/test_path_partition_column_collision/partition_col=from_path/data.csv", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "FORMAT" = "csv_with_names", + "column_separator" = ",", + "use_path_style" = "false", -- aliyun does not support path_style + "path_partition_keys" = "partition_col"); + """ + exception "Path partition column conflicts with an existing column: partition_col" + } +}