From 61d6f6f45927bdf11667e5f7578565b9d04dab05 Mon Sep 17 00:00:00 2001 From: Refrain Date: Wed, 2 Sep 2026 09:02:38 +0800 Subject: [PATCH] [fix](fe) Preserve inferred JSON column name casing ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: JSON schema inference lowercased field names before creating the table-valued function schema, while JSON readers match object keys case-sensitively. Uppercase keys therefore appeared as queryable lowercase columns but were silently materialized as NULL. Preserve the inferred key casing for JSON only, while retaining case-insensitive duplicate detection and the existing lowercase behavior for other file formats. ### Release note JSON table-valued functions now read inferred mixed-case and uppercase keys without silently producing NULL values. ### Check List (For Author) - Test: Regression test and manual tests - Regression test: test_json_uppercase_keys (raw and jsonpaths control) - Manual test: File Scanner V2, legacy scanner, and http_stream returned the two expected rows - Manual compatibility test: CSV inferred column names remained lowercase - Behavior changed: Yes. Inferred JSON field names preserve source casing. - Does this need documentation: No --- .../ExternalFileTableValuedFunction.java | 11 +++-- .../tvf/test_json_uppercase_keys.out | 9 ++++ .../test_json_uppercase_keys/case_keys.jsonl | 2 + .../tvf/test_json_uppercase_keys.groovy | 49 +++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out create mode 100644 regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl create mode 100644 regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 01abd8b482123e..f86abbac1c9c84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -465,15 +465,18 @@ private void fillColumns(InternalService.PFetchTableSchemaResult result) { } // add fetched file columns Set columnLowerNames = new HashSet<>(); + boolean preserveColumnCase = fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_JSON; for (int idx = 0; idx < result.getColumnNums(); ++idx) { PTypeDesc type = result.getColumnTypes(idx); - String colName = result.getColumnNames(idx).toLowerCase(); + String originalColName = result.getColumnNames(idx); + String colLowerName = originalColName.toLowerCase(); // Since doris does not distinguish between upper and lower case columns when querying, in order to avoid // query ambiguity, two columns with the same name but different capitalization are not allowed. - if (columnLowerNames.contains(colName)) { - throw new NotSupportedException("Repeated lowercase column names: " + colName); + if (columnLowerNames.contains(colLowerName)) { + throw new NotSupportedException("Repeated lowercase column names: " + colLowerName); } else { - columnLowerNames.add(colName); + columnLowerNames.add(colLowerName); + String colName = preserveColumnCase ? originalColName : colLowerName; columns.add(new Column(colName, getColumnType(type.getTypesList(), 0).key(), true)); } } diff --git a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out new file mode 100644 index 00000000000000..1dba9bb2590374 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !json_uppercase_keys -- +1 Beijing 12.5 +2 Shanghai 7.75 + +-- !json_uppercase_keys_with_jsonpaths -- +1 Beijing 12.5 +2 Shanghai 7.75 + diff --git a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl new file mode 100644 index 00000000000000..8b1e11f4b3392f --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl @@ -0,0 +1,2 @@ +{"ID":1,"City":"Beijing","Metric":12.5} +{"ID":2,"City":"Shanghai","Metric":7.75} diff --git a/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy new file mode 100644 index 00000000000000..9953855e883e17 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_json_uppercase_keys", "p0,external") { + String ak = getS3AK() + String sk = getS3SK() + String s3Endpoint = getS3Endpoint() + String bucket = context.config.otherConfigs.get("s3BucketName") + String uri = "https://${bucket}.${s3Endpoint}/regression/tvf/test_json_uppercase_keys/case_keys.jsonl" + + order_qt_json_uppercase_keys """ + select id, city, metric from + s3( + "URI" = "${uri}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "FORMAT" = "json", + "read_json_by_line" = "true", + "fuzzy_parse" = "true", + "use_path_style" = "false") order by id; + """ + + order_qt_json_uppercase_keys_with_jsonpaths """ + select id, city, metric from + s3( + "URI" = "${uri}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "FORMAT" = "json", + "read_json_by_line" = "true", + "fuzzy_parse" = "true", + "jsonpaths" = '["\$.ID", "\$.City", "\$.Metric"]', + "use_path_style" = "false") order by id; + """ +}