diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 01abd8b482123e..a0b07a916593b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -553,14 +553,24 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti .setFileScanRange(ByteString.copyFrom(new TSerializer().serialize(fileScanRange))).build(); } - private boolean isFileContentEmpty(TBrokerFileStatus fileStatus) { + boolean isFileContentEmpty(TBrokerFileStatus fileStatus) { if (fileStatus.isIsDir() || fileStatus.size == 0) { return true; } if (Util.isCsvFormat(fileFormatProperties.getFileFormatType()) || fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_JSON) { int magicNumberBytes = 0; - switch (fileFormatProperties.getCompressionType()) { + TFileCompressType compressType = fileFormatProperties.getCompressionType(); + if (compressType == TFileCompressType.UNKNOWN) { + TFileCompressType inferredCompressType = Util.getOrInferCompressType( + compressType, fileStatus.getPath()); + // Preserve the established reader error for auto-inferred empty LZO and other formats. + // This path-based empty-file shortcut is currently supported only for gzip. + if (inferredCompressType == TFileCompressType.GZ) { + compressType = inferredCompressType; + } + } + switch (compressType) { case GZ: magicNumberBytes = 20; break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index 34df496439589c..75f68a4870e998 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -23,6 +23,8 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.FileFormatUtils; +import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.thrift.TBrokerFileStatus; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -63,6 +65,20 @@ public void testHiveParquetTimeZoneRejectsAmbiguousShortAlias() { Assert.assertTrue(exception.getMessage().contains("short timezone aliases are not supported")); } + @Test + public void testAutoInferredEmptyContentOnlySkipsGzip() throws AnalysisException { + ExternalFileTableValuedFunction tvf = Mockito.mock( + ExternalFileTableValuedFunction.class, Mockito.CALLS_REAL_METHODS); + tvf.fileFormatProperties = FileFormatProperties.createFileFormatProperties( + FileFormatConstants.FORMAT_CSV); + tvf.fileFormatProperties.analyzeFileFormatProperties(Maps.newHashMap(), true); + + Assert.assertTrue(tvf.isFileContentEmpty( + new TBrokerFileStatus("empty.csv.gz", false, 20, true))); + Assert.assertFalse(tvf.isFileContentEmpty( + new TBrokerFileStatus("empty.csv.lzo", false, 42, true))); + } + @Test public void testCsvSchemaParse() { Config.enable_date_conversion = true; diff --git a/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema.out b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema.out new file mode 100644 index 00000000000000..9afdca4650e56f --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !csv_empty_first_gzip -- +1 alpha 1.10 +2 beta 2.20 + +-- !json_empty_first_gzip -- +1 beijing 1.1 +2 shanghai 2.2 + diff --git a/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/a_empty.csv.gz b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/a_empty.csv.gz new file mode 100644 index 00000000000000..229151a5a27ab0 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/a_empty.csv.gz differ diff --git a/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/b_valid.csv.gz b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/b_valid.csv.gz new file mode 100644 index 00000000000000..5ac60b417f91b0 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/csv/b_valid.csv.gz differ diff --git a/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/a_empty.jsonl.gz b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/a_empty.jsonl.gz new file mode 100644 index 00000000000000..229151a5a27ab0 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/a_empty.jsonl.gz differ diff --git a/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/b_valid.jsonl.gz b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/b_valid.jsonl.gz new file mode 100644 index 00000000000000..d7dac3b26850ec Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/test_empty_first_compressed_schema/json/b_valid.jsonl.gz differ diff --git a/regression-test/suites/external_table_p0/tvf/test_empty_first_compressed_schema.groovy b/regression-test/suites/external_table_p0/tvf/test_empty_first_compressed_schema.groovy new file mode 100644 index 00000000000000..2e164c2cbe9682 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_empty_first_compressed_schema.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_empty_first_compressed_schema", "p0,external") { + String ak = getS3AK() + String sk = getS3SK() + String s3Endpoint = getS3Endpoint() + String bucket = context.config.otherConfigs.get("s3BucketName") + String baseUri = "https://${bucket}.${s3Endpoint}/regression/tvf/test_empty_first_compressed_schema" + + order_qt_csv_empty_first_gzip """ + select id, name, metric from + s3( + "URI" = "${baseUri}/csv/*.csv.gz", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "FORMAT" = "csv_with_names", + "column_separator" = ",", + "use_path_style" = "false") order by id; + """ + + order_qt_json_empty_first_gzip """ + select id, city, metric from + s3( + "URI" = "${baseUri}/json/*.jsonl.gz", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}", + "FORMAT" = "json", + "read_json_by_line" = "true", + "fuzzy_parse" = "true", + "use_path_style" = "false") order by id; + """ +}