From d3b1a185a56a52f50eebb0b1748f884308852fbe Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Jul 2026 20:56:56 +0800 Subject: [PATCH 01/10] [fix](iceberg) Honor partial name mappings for legacy files ### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: Legacy Iceberg Parquet and ORC files without field IDs are resolved through name mapping. With a partial mapping, FE omitted the optional mapping metadata for unmapped fields, so both V1 and V2 BE readers fell back to the current physical name and read unrelated data. Preserve table-level mapping presence by transporting explicit empty per-field lists and make those lists authoritative in both readers, so unmapped fields materialize their default or NULL while mapped aliases and scans without name mapping retain their existing behavior. ### Release note Fix reads of migrated Iceberg files with partial name mapping so fields omitted from the mapping materialize their default value or NULL instead of matching a physical column by the current name. ### Check List (For Author) - Test: Unit Test - ExternalUtilTest - Focused V1 and V2 name-mapping BE unit tests under ASAN - Behavior changed: Yes. Partial Iceberg name mappings are now authoritative for legacy files without field IDs. - Does this need documentation: No --- .../table/table_schema_change_helper.cpp | 3 + be/src/format_v2/column_data.h | 3 + be/src/format_v2/column_mapper.cpp | 17 +++-- .../format_v2/table/schema_history_util.cpp | 1 + be/src/format_v2/table_reader.cpp | 2 + .../table/table_schema_change_helper_test.cpp | 65 +++++++++++++++++++ be/test/format_v2/table_reader_test.cpp | 23 +++++++ .../apache/doris/datasource/ExternalUtil.java | 8 ++- .../doris/datasource/ExternalUtilTest.java | 20 ++++++ 9 files changed, 133 insertions(+), 9 deletions(-) diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 3d4cb6b5a6c99c..4728a8e2747549 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -73,6 +73,9 @@ bool find_file_field_idx_by_name_mapping( return true; } } + // An explicit empty or unmatched Iceberg mapping means the legacy field is absent; + // falling back to its current name would bind an unrelated physical column. + return false; } return table_field.__isset.name && try_match(table_field.name); diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index ac510caaf07ac4..aae67774bef5fb 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -248,6 +248,9 @@ struct ColumnDefinition { // Historical or external names for the same logical field. Table formats such as Iceberg can // use this to resolve partition path keys after column rename. std::vector name_mapping {}; + // Distinguishes no table-level mapping from an explicit empty field mapping. The latter must + // not fall back to the current name when matching fields in legacy files. + bool has_name_mapping = false; DataTypePtr type; // Semantic nested children for this schema node. // diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 313bbf376f860e..55cba7100978c8 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -82,12 +82,16 @@ bool column_has_name(const ColumnDefinition& column, const std::string& name) { } bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { - if (column_has_name(rhs, lhs.name)) { - return true; - } - if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { - return true; + if (!lhs.has_name_mapping) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } } + // Explicit Iceberg name mapping is authoritative: an empty alias list represents a field that + // did not exist in the imported file, so only transported aliases may match. return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { return column_has_name(rhs, alias); }); @@ -332,7 +336,8 @@ std::string ColumnDefinition::debug_string() const { out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) << ", name_mapping=" << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) - << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << ", has_name_mapping=" << has_name_mapping << ", local_id=" << local_id + << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 10109839e6987d..b0e39d013a6b47 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,6 +77,7 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; + column->has_name_mapping = field.__isset.name_mapping; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5f37538c1f2639..733da0c2b2cb6d 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -189,6 +189,7 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .has_name_mapping = field.__isset.name_mapping, .type = std::move(type), .children = {}, .default_expr = nullptr, @@ -488,6 +489,7 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; + column->has_name_mapping = context->schema_column->has_name_mapping; return Status::OK(); } diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 3fcb9d39878255..65da16a37b2f03 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -35,6 +36,30 @@ namespace doris { class MockTableSchemaChangeHelper : public TableSchemaChangeHelper {}; +namespace { + +schema::external::TStructField partial_name_mapping_root_field() { + TColumnType int_type; + int_type.type = TPrimitiveType::INT; + + schema::external::TStructField root_field; + for (const auto& [name, id, aliases] : + std::vector>> {{"a", 1, {"a"}}, + {"b", 2, {}}}) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(int_type); + field->__set_name_mapping(aliases); + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + root_field.fields.emplace_back(std::move(field_ptr)); + } + return root_field; +} + +} // namespace + TEST(PartitionColumnFillerTest, FillNullableStringPartitionValue) { SlotDescriptor slot; slot._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); @@ -425,6 +450,30 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + FieldDescriptor parquet_field; + for (const auto& name : {"a", "b"}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; @@ -536,6 +585,22 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergOrcPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + std::shared_ptr ans_node; + ASSERT_TRUE( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, NestedMapArrayStruct) { // struct, struct>> SlotDescriptor slot1; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 05ef805ae71267..572000012aa32c 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1942,6 +1942,29 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { + auto unmapped_field = external_schema_field("b", 2); + unmapped_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_TRUE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 6a986da45303d6..0bda4432711b48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -178,9 +178,11 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { - // for iceberg set name mapping. - root.setNameMapping(new ArrayList<>(nameMapping.get(dorisColumn.getUniqueId()))); + if (nameMapping != null && !nameMapping.isEmpty()) { + // An empty per-field mapping is authoritative and prevents BE from matching a legacy + // file column by its current name when the table has an Iceberg name mapping. + root.setNameMapping(new ArrayList<>( + nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); } TNestedField nestedField = new TNestedField(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..123137f1c07c9d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -28,6 +28,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.schema.external.TArrayField; import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TNestedField; import org.apache.doris.thrift.schema.external.TSchema; import org.apache.doris.thrift.schema.external.TStructField; @@ -265,4 +266,23 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + + @Test + public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column mappedColumn = new Column("a", Type.INT, true); + mappedColumn.setUniqueId(1); + Column unmappedColumn = new Column("b", Type.INT, true); + unmappedColumn.setUniqueId(2); + + Map> nameMapping = new HashMap<>(); + nameMapping.put(mappedColumn.getUniqueId(), Collections.singletonList("a")); + ExternalUtil.initSchemaInfoForAllColumn( + params, 600L, Arrays.asList(mappedColumn, unmappedColumn), nameMapping); + + List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); + Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + } } From c88080583101fe53c759f205ff66fb1c54ee411d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 05:57:44 +0800 Subject: [PATCH 02/10] fix(iceberg): preserve name mapping presence --- be/src/exec/scan/access_path_parser.cpp | 3 ++ be/test/exec/scan/access_path_parser_test.cpp | 37 ++++++++++++++++-- .../apache/doris/datasource/ExternalUtil.java | 38 ++++++++++++------- .../iceberg/source/IcebergScanNode.java | 29 +++++++------- .../doris/datasource/ExternalUtilTest.java | 21 ++++++++++ .../iceberg/source/IcebergScanNodeTest.java | 26 +++++++++++++ 6 files changed, 124 insertions(+), 30 deletions(-) diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..186e8bc456aeec 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,6 +87,9 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + // The presence bit is part of the mapping contract: an explicit empty mapping must remain + // authoritative after access-path pruning instead of enabling current-name fallback. + column->has_name_mapping = schema_column->has_name_mapping; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..c84e7557ed9d6d 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -59,11 +59,13 @@ TColumnAccessPath meta_access_path() { format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, std::vector children = {}, - std::vector aliases = {}) { + std::vector aliases = {}, + bool has_name_mapping = false) { return { .identifier = Field::create_field(id), .name = std::move(name), .name_mapping = std::move(aliases), + .has_name_mapping = has_name_mapping, .type = std::move(type), .children = std::move(children), }; @@ -167,7 +169,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { .type = struct_type, .children = { - field(101, "a", int_type), + field(101, "a", int_type, {}, {}, true), field(205, "b", int_type, {}, {"old_b"}), }, }; @@ -199,6 +201,15 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { expect_child(column.children[0], 205, "b"); EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "a"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); + } { auto column = root_column(100, "s", struct_type); auto status = AccessPathParser::build_nested_children( @@ -206,6 +217,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); expect_child(column.children[1], 205, "b"); } @@ -235,7 +247,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { field(201, "element", element_type, { field(202, "item", string_type, {}, {"old_item"}), - field(203, "quantity", int_type), + field(203, "quantity", int_type, {}, {}, true), }), }, }; @@ -254,6 +266,18 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { EXPECT_EQ(column.children[0].children[0].name_mapping, std::vector({"old_item"})); } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "quantity"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[0].has_name_mapping); + } { auto column = root_column(200, "items", array_type); auto status = AccessPathParser::build_nested_children( @@ -264,6 +288,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { ASSERT_EQ(column.children[0].children.size(), 2); expect_child(column.children[0].children[0], 202, "item"); expect_child(column.children[0].children[1], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[1].has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -293,7 +318,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { field(302, "value", value_type, { field(303, "full_name", string_type, {}, {"name"}), - field(304, "age", int_type), + field(304, "age", int_type, {}, {}, true), field(305, "gender", string_type), }), }, @@ -329,6 +354,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { expect_child(column.children[1], 302, "value"); ASSERT_EQ(column.children[1].children.size(), 1); expect_child(column.children[1].children[0], 304, "age"); + EXPECT_TRUE(column.children[1].children[0].has_name_mapping); } { auto column = root_column(300, "m", map_type); @@ -357,6 +383,9 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); ASSERT_EQ(column.children[1].children.size(), 3); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + EXPECT_TRUE(age->has_name_mapping); } for (const auto& invalid_path : std::vector> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 0bda4432711b48..491908744f59e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -124,8 +124,7 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { - initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), base64InitialDefaults); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); + tSchema.setRootField(getExternalSchemaForAllColumn( + columns, nameMapping, hasNameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -161,11 +170,13 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { - return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + return getExternalSchema(columnType, dorisColumn, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); @@ -178,7 +189,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && !nameMapping.isEmpty()) { + if (hasNameMapping) { // An empty per-field mapping is authoritative and prevents BE from matching a legacy // file column by its current name when the table has an Iceberg name mapping. root.setNameMapping(new ArrayList<>( @@ -201,7 +212,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( - subField.getType(), subColumn, nameMapping, base64InitialDefaults)); + subField.getType(), subColumn, nameMapping, hasNameMapping, + base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -214,7 +226,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -225,13 +237,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 972009fb1e7ccf..867d7c6fa26054 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -270,25 +270,27 @@ protected void doInitialize() throws UserException { /** * Extract name mapping from Iceberg table properties. - * Returns a map from field ID to list of mapped names. + * Returns a present map, possibly empty, only when the property parsed successfully. */ - private Map> extractNameMapping() { + private Optional>> extractNameMapping() { Map> result = new HashMap<>(); + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } try { - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson != null && !nameMappingJson.isEmpty()) { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping != null) { - // Extract mappings from NameMapping - // NameMapping contains field mappings, we need to convert them to our format - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - } + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); } + // Optional presence is semantic here: a valid [] still disables current-name fallback. + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); } catch (Exception e) { // If name mapping parsing fails, continue without it LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); } - return result; } private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { @@ -532,13 +534,14 @@ private List getOrderedPathPartitionKeys() { public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); // Extract name mapping from Iceberg table properties - Map> nameMapping = extractNameMapping(); + Optional>> nameMapping = extractNameMapping(); // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), - nameMapping, getBase64EncodedInitialDefaultsForScan()); + nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), + getBase64EncodedInitialDefaultsForScan()); } @VisibleForTesting diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 123137f1c07c9d..77f674e4925327 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -285,4 +285,25 @@ public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); } + + @Test + public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping() { + Column column = new Column("a", Type.INT, true); + column.setUniqueId(1); + + TFileScanRangeParams absentParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn( + absentParams, 700L, Collections.singletonList(column), Collections.emptyMap()); + TField absentField = absentParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertFalse(absentField.isSetNameMapping()); + + TFileScanRangeParams emptyParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, + Collections.singletonList(column), Collections.emptyMap(), true, Collections.emptyMap()); + TField emptyField = emptyParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertTrue(emptyField.isSetNameMapping()); + Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index c823cb2cc89465..39072774cd73e2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -42,6 +42,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -59,11 +60,20 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; + @SuppressWarnings("unchecked") + private static Optional>> extractNameMapping( + IcebergScanNode node) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("extractNameMapping"); + method.setAccessible(true); + return (Optional>>) method.invoke(node); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private TableScan tableScan; @@ -108,6 +118,22 @@ void addSlot(int slotId, Column column) { } } + @Test + public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Table table = Mockito.mock(Table.class); + setIcebergTable(node, table); + + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Assert.assertFalse(extractNameMapping(node).isPresent()); + + Mockito.when(table.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + Optional>> emptyMapping = extractNameMapping(node); + Assert.assertTrue(emptyMapping.isPresent()); + Assert.assertTrue(emptyMapping.get().isEmpty()); + } + @Test public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { Schema systemTableSchema = new Schema( From 54ab787b88acf8fff12a336fa92a2a19166f614c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 09:49:23 +0800 Subject: [PATCH 03/10] fix(iceberg): preserve mapping semantics across upgrades --- .../table/table_schema_change_helper.cpp | 56 ++++--- be/src/format_v2/table/iceberg_reader.h | 21 ++- .../format_v2/table/schema_history_util.cpp | 3 +- be/src/format_v2/table_reader.cpp | 3 +- .../table/table_schema_change_helper_test.cpp | 151 ++++++++++++++++++ .../format_v2/table/iceberg_reader_test.cpp | 13 +- be/test/format_v2/table_reader_test.cpp | 25 +++ .../apache/doris/datasource/ExternalUtil.java | 5 +- .../doris/datasource/ExternalUtilTest.java | 6 + gensrc/thrift/ExternalTableSchema.thrift | 5 +- 10 files changed, 241 insertions(+), 47 deletions(-) diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 4728a8e2747549..a9859c08a6429f 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -73,9 +73,12 @@ bool find_file_field_idx_by_name_mapping( return true; } } - // An explicit empty or unmatched Iceberg mapping means the legacy field is absent; - // falling back to its current name would bind an unrelated physical column. - return false; + if (table_field.__isset.name_mapping_is_authoritative && + table_field.name_mapping_is_authoritative) { + // Only a compatible FE can make the mapping authoritative; older FE plans must retain + // their legacy current-name fallback throughout a rolling BE upgrade. + return false; + } } return table_field.__isset.name && try_match(table_field.name); @@ -559,17 +562,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& parquet_fields_schema = parquet_field_desc.get_fields_schema(); std::map file_column_id_idx_map; - bool all_have_field_id = true; + // Iceberg considers the schema ID-bearing when any field has an ID; requiring all IDs would + // discard authoritative matches merely because an unrelated sibling is ID-less. + bool has_field_id = false; for (size_t idx = 0; idx < parquet_fields_schema.size(); idx++) { - if (parquet_fields_schema[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_fields_schema[idx].field_id != -1) { + has_field_id = true; + file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!has_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_fields_schema); } @@ -577,7 +581,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (has_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; @@ -663,17 +667,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - bool all_have_field_id = true; + // Apply the same any-ID rule recursively so nested mixed-ID structs retain ID matches. + bool has_field_id = false; for (size_t idx = 0; idx < parquet_field.children.size(); idx++) { - if (parquet_field.children[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_field.children[idx].field_id != -1) { + has_field_id = true; + file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!has_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_field.children); } @@ -681,7 +685,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (has_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; @@ -828,19 +832,19 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - bool all_have_field_id = true; + // ORC follows Iceberg's any-ID projection rule just like Parquet. + bool has_field_id = false; for (size_t idx = 0; idx < orc_root->getSubtypeCount(); idx++) { - if (!orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { - all_have_field_id = false; - break; + if (orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { + has_field_id = true; + auto field_id = + std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); + file_column_id_idx_map.emplace(field_id, idx); } - auto field_id = - std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); - file_column_id_idx_map.emplace(field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!has_field_id) { file_column_name_idx_map = build_lowercase_orc_field_name_idx_map(orc_root); } @@ -848,7 +852,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const auto& table_column_name = table_field.field_ptr->name; size_t file_field_idx = 0; bool matched = false; - if (all_have_field_id) { + if (has_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_field_idx = id_it->second; diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index d328f574f2b22b..5d40acc119a47c 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -57,7 +57,7 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) + return !_data_reader.file_schema.empty() && _has_any_field_id(_data_reader.file_schema) ? format::TableColumnMappingMode::BY_FIELD_ID : format::TableColumnMappingMode::BY_NAME; } @@ -77,24 +77,21 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - bool _has_field_id(const std::vector& schema) const { + bool _has_any_field_id(const std::vector& schema) const { for (const auto& field : schema) { - // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the - // first-phase scan. That virtual column is not an Iceberg data field and therefore has - // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, - // otherwise old data files whose physical names predate a rename (for example, - // table column `new_new_id` stored as file column `id`) are materialized as defaults. + // Iceberg's hasIds contract is existential. Ignore synthesized columns and keep ID + // projection as soon as any real field (including a nested field) carries an ID. if (field.column_type != format::ColumnType::DATA_COLUMN) { continue; } - if (!field.has_identifier_field_id()) { - return false; + if (field.has_identifier_field_id()) { + return true; } - if (!_has_field_id(field.children)) { - return false; + if (_has_any_field_id(field.children)) { + return true; } } - return true; + return false; } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index b0e39d013a6b47..440211607d633d 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,7 +77,8 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; - column->has_name_mapping = field.__isset.name_mapping; + column->has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 733da0c2b2cb6d..e63485d6f8f513 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -189,7 +189,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, - .has_name_mapping = field.__isset.name_mapping, + .has_name_mapping = field.__isset.name_mapping_is_authoritative && + field.name_mapping_is_authoritative, .type = std::move(type), .children = {}, .default_expr = nullptr, diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 65da16a37b2f03..5d20542187b30c 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -51,6 +51,7 @@ schema::external::TStructField partial_name_mapping_root_field() { field->__set_id(id); field->__set_type(int_type); field->__set_name_mapping(aliases); + field->__set_name_mapping_is_authoritative(true); schema::external::TFieldPtr field_ptr; field_ptr.__set_field_ptr(field); root_field.fields.emplace_back(std::move(field_ptr)); @@ -58,6 +59,29 @@ schema::external::TStructField partial_name_mapping_root_field() { return root_field; } +schema::external::TStructField nested_partial_name_mapping_root_field() { + TColumnType struct_type; + struct_type.type = TPrimitiveType::STRUCT; + + auto nested_fields = partial_name_mapping_root_field(); + nested_fields.fields[0].field_ptr->__set_name_mapping({}); + + auto field = std::make_shared(); + field->__set_name("s"); + field->__set_id(10); + field->__set_type(struct_type); + field->__set_name_mapping({}); + field->__set_name_mapping_is_authoritative(true); + field->nestedField.__set_struct_field(std::move(nested_fields)); + field->__isset.nestedField = true; + + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + schema::external::TStructField root_field; + root_field.fields.emplace_back(std::move(field_ptr)); + return root_field; +} + } // namespace TEST(PartitionColumnFillerTest, FillNullableStringPartitionValue) { @@ -450,6 +474,30 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetLegacyEmptyNameMappingFallsBack) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields.resize(1); + root_field.fields[0].field_ptr->__set_name_mapping({}); + root_field.fields[0].field_ptr->__isset.name_mapping_is_authoritative = false; + + FieldDescriptor parquet_field; + FieldSchema file_field; + file_field.name = "a"; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergParquetPartialNameMappingIsStrict) { auto root_field = partial_name_mapping_root_field(); @@ -474,6 +522,68 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetPartialNameMappingIsStrict) { " b (not exists)\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + FieldDescriptor parquet_field; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = field_id; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + + FieldSchema struct_field; + struct_field.name = "s"; + struct_field.field_id = 10; + std::vector child_types; + Strings child_names; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema child; + child.name = name; + child.field_id = field_id; + child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + child_types.emplace_back(child.data_type); + child_names.emplace_back(child.name); + struct_field.children.emplace_back(std::move(child)); + } + struct_field.data_type = std::make_shared(child_types, child_names); + + FieldDescriptor parquet_field; + parquet_field._fields.emplace_back(std::move(struct_field)); + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; @@ -601,6 +711,47 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcPartialNameMappingIsStrict) { " b (not exists)\n"); } +TEST(MockTableSchemaChangeHelper, IcebergOrcMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + orc_type->getSubtype(0)->setAttribute(IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + + std::unique_ptr orc_type( + orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->setAttribute(attribute, "10"); + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, NestedMapArrayStruct) { // struct, struct>> SlotDescriptor slot1; diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 0831c98750ecbf..1383047d836961 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -1843,9 +1843,9 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { TableColumnMappingMode::BY_FIELD_ID); } -// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data -// column without an Iceberg field id still disables field-id mapping. -TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { +// Iceberg treats a schema as ID-bearing when any physical data field has an ID. Keeping ID mode +// preserves authoritative matches even when a sibling was written without an ID. +TEST(IcebergV2ReaderTest, IcebergMappingModeUsesAnyDataColumnFieldId) { IcebergTableReaderMappingModeTestHelper reader; std::vector file_schema { make_file_column(1, "id", std::make_shared()), @@ -1854,8 +1854,13 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { }; file_schema[1].identifier = Field {}; + EXPECT_EQ(reader.mapping_mode_for_schema(file_schema), TableColumnMappingMode::BY_FIELD_ID); + + file_schema[0].identifier = Field {}; + file_schema[0].children.emplace_back( + make_file_column(3, "nested", std::make_shared())); EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), - TableColumnMappingMode::BY_NAME); + TableColumnMappingMode::BY_FIELD_ID); } TEST(IcebergV2ReaderTest, IcebergTableReaderDoesNotPushDownAggregateWithPositionDelete) { diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 572000012aa32c..8a02c11bffc5e6 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1945,6 +1945,7 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { auto unmapped_field = external_schema_field("b", 2); unmapped_field.field_ptr->__set_name_mapping({}); + unmapped_field.field_ptr->__set_name_mapping_is_authoritative(true); TFileScanRangeParams scan_params; scan_params.__set_current_schema_id(1); scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); @@ -1965,6 +1966,30 @@ TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); } +TEST(TableReaderTest, LegacyFeEmptyNameMappingStillMatchesCurrentFileName) { + auto legacy_field = external_schema_field("b", 2); + legacy_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {legacy_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_FALSE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 491908744f59e8..37dbf4cdd390ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -190,10 +190,11 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, } if (hasNameMapping) { - // An empty per-field mapping is authoritative and prevents BE from matching a legacy - // file column by its current name when the table has an Iceberg name mapping. + // The explicit capability keeps old-FE plans on legacy fallback while making an empty + // per-field mapping authoritative for plans produced by a compatible FE. root.setNameMapping(new ArrayList<>( nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); + root.setNameMappingIsAuthoritative(true); } TNestedField nestedField = new TNestedField(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 77f674e4925327..3bee2d9747f02d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -255,6 +255,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertTrue(field1.isNameMappingIsAuthoritative()); Assert.assertEquals("7", field1.getInitialDefaultValue()); Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); @@ -263,6 +264,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertTrue(field2.isNameMappingIsAuthoritative()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } @@ -282,8 +284,10 @@ public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(0).getFieldPtr().isNameMappingIsAuthoritative()); Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + Assert.assertTrue(fields.get(1).getFieldPtr().isNameMappingIsAuthoritative()); } @Test @@ -297,6 +301,7 @@ public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping TField absentField = absentParams.getHistorySchemaInfo().get(0) .getRootField().getFields().get(0).getFieldPtr(); Assert.assertFalse(absentField.isSetNameMapping()); + Assert.assertFalse(absentField.isSetNameMappingIsAuthoritative()); TFileScanRangeParams emptyParams = new TFileScanRangeParams(); ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, @@ -305,5 +310,6 @@ public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping .getRootField().getFields().get(0).getFieldPtr(); Assert.assertTrue(emptyField.isSetNameMapping()); Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + Assert.assertTrue(emptyField.isNameMappingIsAuthoritative()); } } diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 46ae54781ae825..86915e46d28bfb 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -58,7 +58,10 @@ struct TField { // True when initial_default_value is Base64 and must be decoded before constructing the Doris // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. - 8: optional bool initial_default_value_is_base64 + 8: optional bool initial_default_value_is_base64, + // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the + // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. + 9: optional bool name_mapping_is_authoritative } From adfe227951effcd57c90ee511f45c6a1972b3361 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 15:09:58 +0800 Subject: [PATCH 04/10] [fix](iceberg) align mixed-id nested projection semantics --- be/src/exec/scan/access_path_parser.cpp | 4 + be/src/format/orc/vorc_reader.cpp | 49 ++++++++++-- .../format/parquet/vparquet_column_reader.cpp | 46 +++++++++-- .../table/table_schema_change_helper.cpp | 42 ++++++++-- .../format/table/table_schema_change_helper.h | 30 +++++++- be/src/format_v2/column_mapper.cpp | 76 ++++++++++++++++++- be/src/format_v2/column_mapper.h | 2 + be/src/format_v2/table/iceberg_reader.h | 4 + be/src/format_v2/table_reader.h | 8 +- be/test/exec/scan/access_path_parser_test.cpp | 17 +++++ .../table/iceberg/iceberg_reader_test.cpp | 34 ++++----- .../table/table_schema_change_helper_test.cpp | 37 +++++++++ be/test/format_v2/column_mapper_test.cpp | 37 +++++++++ .../format_v2/table/iceberg_reader_test.cpp | 8 +- be/test/format_v2/table_reader_test.cpp | 8 +- 15 files changed, 355 insertions(+), 47 deletions(-) diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index 186e8bc456aeec..f0346abaaf300e 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -90,6 +90,10 @@ void inherit_schema_metadata(format::ColumnDefinition* column, // The presence bit is part of the mapping contract: an explicit empty mapping must remain // authoritative after access-path pruning instead of enabling current-name fallback. column->has_name_mapping = schema_column->has_name_mapping; + // Initial defaults describe the logical value of fields absent from older files. Nested + // access-path pruning must retain them just like it retains rename metadata. + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..303d9a6e5475d9 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -34,6 +34,7 @@ #include "exprs/vexpr.h" #include "exprs/vslot_ref.h" #include "exprs/vtopn_pred.h" +#include "util/url_coding.h" // IWYU pragma: no_include #include // IWYU pragma: keep @@ -105,6 +106,31 @@ #include "util/unaligned.h" namespace doris { +static Status build_orc_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + class RuntimeState; namespace io { struct IOContext; @@ -2145,15 +2171,28 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, for (int missing_field : missing_fields) { ColumnPtr& doris_field = doris_struct.get_column_ptr(missing_field); - if (!doris_field->is_nullable()) { + const auto& table_column_name = doris_struct_type->get_name_by_position(missing_field); + const auto& doris_type = doris_struct_type->get_element(missing_field); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_orc_initial_default_column( + root_node->children_initial_default_value(table_column_name), doris_type, + num_values, &initial_default)); + if (initial_default.get() != nullptr) { + // ORC projection may synthesize a missing nested field, but its Iceberg initial + // default remains the logical value for every row in the older file. + auto mutable_field = IColumn::mutate(std::move(doris_field)); + mutable_field->insert_range_from(*initial_default, 0, num_values); + doris_field = std::move(mutable_field); + } else if (!doris_field->is_nullable()) { return Status::InternalError( "Child field of '{}' is not nullable, but is missing in orc file", col_name); + } else { + auto mutable_field = IColumn::mutate(std::move(doris_field)); + reinterpret_cast(mutable_field.get()) + ->insert_many_defaults(num_values); + doris_field = std::move(mutable_field); } - auto mutable_field = IColumn::mutate(std::move(doris_field)); - reinterpret_cast(mutable_field.get()) - ->insert_many_defaults(num_values); - doris_field = std::move(mutable_field); } for (auto read_field : read_fields) { diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 2726044c193b4c..90096a9101467a 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -40,8 +40,34 @@ #include "format/parquet/vparquet_column_chunk_reader.h" #include "io/fs/tracing_file_reader.h" #include "runtime/runtime_profile.h" +#include "util/url_coding.h" namespace doris { +static Status build_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, const std::vector& rep_levels, const std::vector& def_levels) { @@ -990,11 +1016,21 @@ Status StructColumnReader::read_column_data( for (auto idx : missing_column_idxs) { auto& doris_field = doris_struct.get_column_ptr(idx); auto& doris_type = doris_struct_type->get_element(idx); - DCHECK(doris_type->is_nullable()); - doris_field = IColumn::mutate(std::move(doris_field)); - auto mutable_column = doris_field->assert_mutable(); - auto* nullable_column = static_cast(mutable_column.get()); - nullable_column->insert_many_defaults(missing_column_sz); + auto mutable_field = IColumn::mutate(std::move(doris_field)); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_initial_default_column( + root_node->children_initial_default_value(doris_struct_type->get_element_name(idx)), + doris_type, missing_column_sz, &initial_default)); + if (initial_default.get() != nullptr) { + // Iceberg initial defaults are logical row values, including for nested fields absent + // from the physical file; append them instead of the type's generic NULL/default. + mutable_field->insert_range_from(*initial_default, 0, missing_column_sz); + } else { + DCHECK(doris_type->is_nullable()); + static_cast(mutable_field.get()) + ->insert_many_defaults(missing_column_sz); + } + doris_field = std::move(mutable_field); } if (null_map_ptr != nullptr) { diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index a9859c08a6429f..50c2b8eabd37db 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -55,6 +55,30 @@ std::map build_lowercase_orc_field_name_idx_map(const orc:: return file_column_name_idx_map; } +bool orc_subtree_has_field_id(const orc::Type* type, const std::string& attribute) { + if (type->hasAttributeKey(attribute)) { + return true; + } + for (uint64_t idx = 0; idx < type->getSubtypeCount(); ++idx) { + if (orc_subtree_has_field_id(type->getSubtype(idx), attribute)) { + return true; + } + } + return false; +} + +std::optional initial_default_value( + const schema::external::TField& field) { + if (!field.__isset.initial_default_value) { + return std::nullopt; + } + return TableSchemaChangeHelper::InitialDefaultValue { + .value = field.initial_default_value, + .is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, + }; +} + bool find_file_field_idx_by_name_mapping( const schema::external::TField& table_field, const std::map& file_column_name_idx_map, size_t* file_column_idx) { @@ -540,7 +564,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( exist_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } else { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); } } node = struct_node; @@ -593,7 +618,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } @@ -697,7 +723,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } @@ -832,11 +859,11 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - // ORC follows Iceberg's any-ID projection rule just like Parquet. - bool has_field_id = false; + // Iceberg ORC builds projection by ID for the entire subtree. An ID-less wrapper containing + // an ID-bearing child is therefore absent rather than rebound by its physical name. + bool has_field_id = orc_subtree_has_field_id(orc_root, field_id_attribute_key); for (size_t idx = 0; idx < orc_root->getSubtypeCount(); idx++) { if (orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { - has_field_id = true; auto field_id = std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); file_column_id_idx_map.emplace(field_id, idx); @@ -864,7 +891,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 2d54b3216cac89..6b1adb138fc962 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -19,6 +19,7 @@ #include #include +#include #include #include "common/status.h" @@ -41,6 +42,10 @@ namespace doris { class TableSchemaChangeHelper { public: + struct InitialDefaultValue { + std::string value; + bool is_base64 = false; + }; ~TableSchemaChangeHelper() = default; class Node { @@ -67,6 +72,11 @@ class TableSchemaChangeHelper { "children_column_exists should not be called on base TableInfoNode"); } + virtual std::optional children_initial_default_value( + std::string) const { + return std::nullopt; + } + virtual std::shared_ptr get_element_node() const { throw std::logic_error("get_element_node should not be called on base TableInfoNode"); } @@ -78,7 +88,9 @@ class TableSchemaChangeHelper { throw std::logic_error("get_value_node should not be called on base TableInfoNode"); } - virtual void add_not_exist_children(std::string table_column_name) { + virtual void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) { throw std::logic_error( "add_not_exist_children should not be called on base TableInfoNode"); }; @@ -131,6 +143,7 @@ class TableSchemaChangeHelper { const std::shared_ptr node; const std::string column_name; const bool exists; + const std::optional initial_default; }; // table column name -> { node, file_column_name, exists_in_file} @@ -167,14 +180,23 @@ class TableSchemaChangeHelper { return children.at(table_column_name).exists; } - void add_not_exist_children(std::string table_column_name) override { - children.emplace(table_column_name, StructChild {nullptr, "", false}); + std::optional children_initial_default_value( + std::string table_column_name) const override { + DCHECK(children.contains(table_column_name)); + return children.at(table_column_name).initial_default; + } + + void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) override { + children.emplace(table_column_name, + StructChild {nullptr, "", false, std::move(initial_default)}); } void add_children(std::string table_column_name, std::string file_column_name, std::shared_ptr children_node) override { children.emplace(table_column_name, - StructChild {children_node, file_column_name, true}); + StructChild {children_node, file_column_name, true, std::nullopt}); } const std::map& get_children() const { return children; } diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 55cba7100978c8..eba294deb0b789 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -52,11 +52,59 @@ #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "util/url_coding.h" namespace doris::format { namespace { +Status parse_initial_default(const ColumnDefinition& column, std::optional* value) { + DORIS_CHECK(value != nullptr); + value->reset(); + if (!column.initial_default_value.has_value()) { + return Status::OK(); + } + const auto nested_type = remove_nullable(column.type); + Field parsed; + if (column.initial_default_value_is_base64 || + nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(*column.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + column.name); + } + parsed = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); + } + *value = std::move(parsed); + return Status::OK(); +} + +bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { + for (const auto& table_child : table.children) { + if (!table_child.has_identifier_field_id()) { + continue; + } + const auto file_child = + std::ranges::find_if(file.children, [&](const ColumnDefinition& candidate) { + return candidate.has_identifier_field_id() && + candidate.get_identifier_field_id() == + table_child.get_identifier_field_id(); + }); + if (file_child != file.children.end() || + std::ranges::any_of(file.children, [&](const ColumnDefinition& candidate) { + return has_shared_descendant_field_id(table_child, candidate); + })) { + return true; + } + } + return false; +} + std::string mapping_mode_to_string(TableColumnMappingMode mode) { switch (mode) { case TableColumnMappingMode::BY_FIELD_ID: @@ -327,7 +375,9 @@ static bool is_binary_comparison_predicate(const VExprSPtr& expr) { std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; - out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection + << "}"; return out.str(); } @@ -1770,6 +1820,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); + } else if (table_column.initial_default_value.has_value()) { + std::optional initial_default; + RETURN_IF_ERROR(parse_initial_default(table_column, &initial_default)); + DORIS_CHECK(initial_default.has_value()); + _set_constant_mapping(mapping, VExprContext::create_shared(VLiteral::create_shared( + table_column.type, *initial_default))); } else { if (table_column.is_partition_key) { return Status::InvalidArgument( @@ -2089,7 +2145,19 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( }); return field_it == file_schema.end() ? nullptr : &*field_it; } - return matcher_for_mode(_options.mode).find(table_column, file_schema); + const auto* matched = matcher_for_mode(_options.mode).find(table_column, file_schema); + if (matched != nullptr || _options.mode != TableColumnMappingMode::BY_FIELD_ID || + !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { + return matched; + } + const auto* wrapper = find_column_by_name(table_column, file_schema); + if (wrapper == nullptr || wrapper->has_identifier_field_id() || wrapper->children.empty() || + !has_shared_descendant_field_id(table_column, *wrapper)) { + return nullptr; + } + // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is + // selected. ORC instead synthesizes the missing parent, so this fallback is opt-in by format. + return wrapper; } Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, @@ -2152,6 +2220,10 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + // A missing nested field still has its Iceberg initial-default value in every row + // written before the field was added; carry it into recursive materialization. + RETURN_IF_ERROR( + parse_initial_default(table_child, &child_mapping.initial_default_value)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 4f417a680aad60..c3793777f4cbc6 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -150,12 +150,14 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; + std::optional initial_default_value; std::string debug_string() const; }; struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + bool allow_idless_complex_wrapper_projection = false; std::string debug_string() const; }; diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 5d40acc119a47c..1f776df8dba3e5 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -63,6 +63,10 @@ class IcebergTableReader : public format::TableReader { } protected: + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->allow_idless_complex_wrapper_projection = _format == FileFormat::PARQUET; + } + Status materialize_virtual_columns(Block* table_block) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 73bade81bc4c65..78e7143570c3d0 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -360,6 +360,7 @@ class TableReader { Status create_next_reader(bool* eos); virtual Status create_file_reader(std::unique_ptr* reader); virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { DORIS_CHECK(file_schema != nullptr); return Status::OK(); @@ -376,6 +377,7 @@ class TableReader { RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); + configure_mapper_options(&_mapper_options); _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); DORIS_CHECK(_data_reader.column_mapper != nullptr); @@ -1319,7 +1321,11 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) + (child_mapping->initial_default_value.has_value() + ? child_mapping->table_type->create_column_const( + rows, *child_mapping->initial_default_value) + : child_mapping->table_type + ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); continue; } diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index c84e7557ed9d6d..eb4198df640592 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -397,4 +397,21 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { } } +TEST(AccessPathParserTest, PreservesNestedInitialDefaultMetadata) { + auto binary_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {binary_type}, Strings {"data"}); + auto defaulted_child = field(101, "data", binary_type); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto schema = field(100, "s", struct_type, {defaulted_child}); + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "data"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].initial_default_value, std::optional("AAEC/w==")); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6c504e0b49a46b..6475b79eea541a 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -1299,17 +1299,16 @@ TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.parquet").string(); const auto delete_file = (test_dir / "equality-delete.parquet").string(); - // The id-less columns put the complete Parquet file in BY_NAME mode. The unrelated - // stale_added column deliberately retains field id 1, which must not override the historical - // alias selected for the hidden added_column equality key. - write_iceberg_three_int_parquet_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + // Iceberg's hasIds rule is existential: once any physical field has an ID, equality keys bind + // to that authoritative ID instead of an ID-less historical alias. + write_iceberg_three_int_parquet_file(data_file, "id", 0, {1, 2, 3}, "legacy_added", std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); write_iceberg_int_equality_delete_parquet_file(delete_file, "added_column", 1, 7); @@ -1387,26 +1386,26 @@ TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_fi bool eof = false; const auto status = reader.get_next_block(&block, &read_rows, &eof); ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(read_rows, 2); - ASSERT_EQ(block.rows(), 2); + ASSERT_EQ(read_rows, 3); + ASSERT_EQ(block.rows(), 3); EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); - EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "2"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 2), "3"); std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_orc_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_orc_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.orc").string(); const auto delete_file = (test_dir / "equality-delete.orc").string(); - // One id-less column switches the whole file to BY_NAME. The hidden current key is physically - // stored as its id-less historical alias, while an unrelated migrated column still carries - // the key's stale field id. Equality-delete binding must select legacy_added, not stale_added. - write_iceberg_three_int_orc_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", - std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + // Mixed ORC schemas also stay in ID projection when any field has an ID; the key with ID 1 is + // authoritative even though an ID-less historical alias is present. + write_iceberg_three_int_orc_file(data_file, "id", 0, {1, 2, 3}, "legacy_added", std::nullopt, + {5, 7, 9}, "stale_added", 1, {70, 70, 70}); write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); schema::external::TStructField root_field; @@ -1478,10 +1477,11 @@ TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_ bool eof = false; const auto status = reader.get_next_block(&block, &read_rows, &eof); ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(read_rows, 2); - ASSERT_EQ(block.rows(), 2); + ASSERT_EQ(read_rows, 3); + ASSERT_EQ(block.rows(), 3); EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); - EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "2"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 2), "3"); std::filesystem::remove_all(test_dir); } diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 5d20542187b30c..1895fee300b89c 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -550,6 +550,12 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetMixedFieldIdsPreferExistingIds) TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistingIds) { auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); FieldSchema struct_field; struct_field.name = "s"; @@ -582,6 +588,11 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistin " a (file: a)\n" " ScalarNode\n" " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); } TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { @@ -732,6 +743,12 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcMixedFieldIdsPreferExistingIds) { TEST(MockTableSchemaChangeHelper, IcebergOrcNestedMixedFieldIdsPreferExistingIds) { auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); std::unique_ptr orc_type( orc::Type::buildTypeFromString("struct>")); @@ -750,6 +767,26 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcNestedMixedFieldIdsPreferExistingIds " a (file: a)\n" " ScalarNode\n" " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcDoesNotBindIdlessWrapperByName) { + auto root_field = nested_partial_name_mapping_root_field(); + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (not exists)\n"); } TEST(MockTableSchemaChangeHelper, NestedMapArrayStruct) { diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 3bca3c4152c888..89e6c5b6b5b125 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -253,6 +253,43 @@ TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { } } +TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { + auto table_child = field_id_col("a", 1, i32()); + auto table_struct = struct_col("s", 10, {table_child}); + auto file_child = field_id_col("legacy_a", 1, i32(), 0); + auto file_struct = struct_name_col("s", {file_child}, 0); + + TableColumnMapper parquet_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(parquet_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(parquet_mapper.mappings().size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(parquet_mapper.mappings()[0].child_mappings.size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].child_mappings[0].file_local_id, 0); + + TableColumnMapper orc_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(orc_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { + auto defaulted_child = field_id_col("data", 2, str()); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); + auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + const auto& missing = mapper.mappings()[0].child_mappings[1]; + ASSERT_TRUE(missing.initial_default_value.has_value()); + EXPECT_EQ(missing.initial_default_value->get_type(), TYPE_STRING); + EXPECT_EQ(missing.initial_default_value->get(), std::string("\0\1\2\xff", 4)); +} + void expect_mapping(const ColumnMapping& mapping, size_t global_index, const std::string& table_name, int32_t file_local_id, const std::string& file_name, const DataTypePtr& file_type, diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 1383047d836961..b340eabd978add 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -2448,7 +2448,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; std::filesystem::remove_all(test_dir); @@ -2456,8 +2456,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); - // The real key has no id and is mapped by its historical name. A different physical column - // carries the stale id 0, forcing the entire split into BY_NAME mode. + // Iceberg's existential hasIds contract makes the physical field carrying ID 0 authoritative; + // an ID-less alias cannot switch a mixed schema back to name projection. write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", {100, 200, 300}, 0); write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); @@ -2482,7 +2482,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({100, 200, 300})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 8a02c11bffc5e6..316e708dc177fb 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -4077,7 +4077,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4090,6 +4090,8 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { const auto string_type = std::make_shared(); auto id_child = make_table_column(0, "id", int_type); auto missing_child = make_table_column(99, "missing_child", string_type); + missing_child.initial_default_value = "AAEC/w=="; + missing_child.initial_default_value_is_base64 = true; auto struct_type = std::make_shared(DataTypes {int_type, string_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); @@ -4124,7 +4126,9 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& defaults = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(1))); + EXPECT_EQ(defaults.get_data_at(0).to_string(), std::string("\0\1\2\xff", 4)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); From 3050fa2a2caedb13d7308fa74be4f06d4a42941e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 17:47:42 +0800 Subject: [PATCH 05/10] [fix](iceberg) preserve initial defaults across schema evolution --- be/src/format/orc/vorc_reader.cpp | 4 + .../format/parquet/vparquet_column_reader.cpp | 4 + be/src/format/table/iceberg_reader_mixin.h | 131 ++++++++++++++---- be/src/format_v2/column_mapper.cpp | 45 ++++-- be/src/format_v2/column_mapper.h | 4 +- be/src/format_v2/table/iceberg_reader.cpp | 15 +- be/src/format_v2/table_reader.cpp | 5 + be/src/format_v2/table_reader.h | 5 +- .../table/iceberg/iceberg_reader_test.cpp | 63 +++++++-- be/test/format_v2/column_mapper_test.cpp | 32 ++++- .../format_v2/table/iceberg_reader_test.cpp | 85 +++++++++++- be/test/format_v2/table_reader_test.cpp | 69 ++++++++- .../datasource/iceberg/IcebergUtils.java | 17 ++- .../iceberg/source/IcebergScanNode.java | 12 +- .../doris/datasource/ExternalUtilTest.java | 20 +++ .../datasource/iceberg/IcebergUtilsTest.java | 14 ++ .../iceberg/source/IcebergScanNodeTest.java | 28 +++- 17 files changed, 477 insertions(+), 76 deletions(-) diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 303d9a6e5475d9..728595bdad13a7 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -124,6 +124,10 @@ static Status build_orc_initial_default_column( value = nested_type->get_primitive_type() == TYPE_VARBINARY ? Field::create_field(StringView(decoded)) : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); } else { RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); } diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 90096a9101467a..53e44babbd8297 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -61,6 +61,10 @@ static Status build_initial_default_column( value = nested_type->get_primitive_type() == TYPE_VARBINARY ? Field::create_field(StringView(decoded)) : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); } else { RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); } diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index aee211aee019cd..918ace70972ae9 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -157,6 +157,30 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { protected: // ---- Hook implementations ---- + Status on_fill_missing_columns(Block* block, size_t rows, + const std::vector& cols) override { + std::vector generic_columns; + for (const auto& name : cols) { + const auto* field = _find_current_schema_field(name); + if (field == nullptr || !field->__isset.initial_default_value) { + generic_columns.push_back(name); + continue; + } + if (!this->col_name_to_block_idx_ref()->contains(name)) { + return Status::InternalError("Missing column: {} not found in block {}", name, + block->dump_structure()); + } + const auto position = this->col_name_to_block_idx_ref()->at(name); + ColumnPtr value; + RETURN_IF_ERROR(_build_owned_initial_default_column( + *field, block->get_by_position(position).type, rows, &value)); + // Iceberg metadata is authoritative for pre-add rows; a generic FE expression may be + // the Base64 carrier rather than the logical BINARY/FIXED/UUID value. + block->get_by_position(position).column = value->convert_to_full_column_if_const(); + } + return BaseReader::on_fill_missing_columns(block, rows, generic_columns); + } + // Called before reading a block: expand block for equality delete columns + detect row_id Status on_before_read_block(Block* block) override { RETURN_IF_ERROR(_expand_block_if_need(block)); @@ -270,6 +294,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { Status _expand_block_if_need(Block* block); Status _shrink_block_if_need(Block* block); + const schema::external::TStructField* _current_schema_root() const; + const schema::external::TField* _find_current_schema_field(const std::string& name) const; + Status _build_owned_initial_default_column(const schema::external::TField& field, + const DataTypePtr& type, size_t rows, + ColumnPtr* column) const; Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type); Status _materialize_missing_equality_delete_column(Block* block, const std::string& name, @@ -664,6 +693,76 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { return Status::OK(); } +template +const schema::external::TStructField* IcebergReaderMixin::_current_schema_root() const { + const auto& scan_params = this->get_scan_params(); + if (!scan_params.__isset.history_schema_info || scan_params.history_schema_info.empty()) { + return nullptr; + } + const schema::external::TSchema* current_schema = &scan_params.history_schema_info.front(); + if (scan_params.__isset.current_schema_id) { + const auto schema_it = std::ranges::find_if( + scan_params.history_schema_info, [&](const schema::external::TSchema& schema) { + return schema.__isset.schema_id && + schema.schema_id == scan_params.current_schema_id; + }); + if (schema_it == scan_params.history_schema_info.end()) { + return nullptr; + } + current_schema = &*schema_it; + } + return current_schema->__isset.root_field ? ¤t_schema->root_field : nullptr; +} + +template +const schema::external::TField* IcebergReaderMixin::_find_current_schema_field( + const std::string& name) const { + const auto* root = _current_schema_root(); + if (root == nullptr) { + return nullptr; + } + const auto field = std::ranges::find_if(root->fields, [&](const auto& field_ptr) { + return field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.name && field_ptr.field_ptr->name == name; + }); + return field == root->fields.end() ? nullptr : field->field_ptr.get(); +} + +template +Status IcebergReaderMixin::_build_owned_initial_default_column( + const schema::external::TField& field, const DataTypePtr& type, size_t rows, + ColumnPtr* column) const { + DORIS_CHECK(type != nullptr); + DORIS_CHECK(column != nullptr); + DORIS_CHECK(field.__isset.initial_default_value); + const auto nested_type = remove_nullable(type); + const bool is_base64 = + (field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64) || + (field.__isset.type && thrift_to_type(field.type.type) == TYPE_VARBINARY); + Field value; + if (is_base64) { + std::string decoded; + if (!base64_decode(field.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + field.name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded); + } + // Variable-width Fields borrow decoded. Materialize before it leaves scope so every V1 + // missing-column and equality-delete boundary keeps UUID/FIXED payloads alive. + *column = type->create_column_const(rows, value); + return Status::OK(); + } + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(field.initial_default_value, value)); + *column = type->create_column_const(rows, value); + return Status::OK(); +} + template Status IcebergReaderMixin::_register_missing_equality_delete_column( int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) { @@ -703,33 +802,15 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( "Missing Iceberg schema metadata for equality-delete field id {}", field_id); } - Field value; + ColumnPtr value_column; if (table_field->__isset.initial_default_value) { - const auto nested_type = remove_nullable(delete_key_type); - const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && - table_field->initial_default_value_is_base64) || - (table_field->__isset.type && - thrift_to_type(table_field->type.type) == TYPE_VARBINARY); - if (default_is_base64) { - std::string decoded_default; - if (!base64_decode(table_field->initial_default_value, &decoded_default)) { - return Status::InvalidArgument( - "Invalid Base64 Iceberg initial default for field {}", table_field->name); - } - if (nested_type->get_primitive_type() == TYPE_VARBINARY) { - value = Field::create_field(StringView(decoded_default)); - } else { - DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); - value = Field::create_field(decoded_default); - } - } else { - RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( - table_field->initial_default_value, value)); - } + RETURN_IF_ERROR(_build_owned_initial_default_column(*table_field, delete_key_type, 1, + &value_column)); + } else { + value_column = delete_key_type->create_column_const(1, Field()); } - const bool inserted = _missing_equality_delete_values - .emplace(name, delete_key_type->create_column_const(1, value)) - .second; + const bool inserted = + _missing_equality_delete_values.emplace(name, std::move(value_column)).second; DORIS_CHECK(inserted); this->register_synthesized_column_handler( name, [this, name](Block* block, size_t rows) -> Status { diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index eba294deb0b789..542608f77b00d8 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -58,9 +58,9 @@ namespace doris::format { namespace { -Status parse_initial_default(const ColumnDefinition& column, std::optional* value) { +Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { DORIS_CHECK(value != nullptr); - value->reset(); + *value = nullptr; if (!column.initial_default_value.has_value()) { return Status::OK(); } @@ -76,11 +76,27 @@ Status parse_initial_default(const ColumnDefinition& column, std::optionalget_primitive_type() == TYPE_VARBINARY ? Field::create_field(StringView(decoded)) : Field::create_field(decoded); + // Variable-width Fields borrow their input. Materialize while decoded is alive so the + // resulting column owns the payload before it crosses a mapping/literal boundary. + *value = column.type->create_column_const(1, parsed); + return Status::OK(); } else { RETURN_IF_ERROR( nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); } - *value = std::move(parsed); + *value = column.type->create_column_const(1, parsed); + return Status::OK(); +} + +Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { + DORIS_CHECK(literal != nullptr); + ColumnPtr owned_value; + RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); + DORIS_CHECK(static_cast(owned_value)); + Field value; + owned_value->get(0, value); + // VLiteral copies the borrowed Field into its own column while owned_value is still alive. + *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); return Status::OK(); } @@ -1817,15 +1833,15 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.initial_default_value.has_value()) { + VExprContextSPtr initial_default; + RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); + // Iceberg metadata is the authoritative logical value for files written before the field + // existed; the generic FE expression may still contain its Base64 transport text. + _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); - } else if (table_column.initial_default_value.has_value()) { - std::optional initial_default; - RETURN_IF_ERROR(parse_initial_default(table_column, &initial_default)); - DORIS_CHECK(initial_default.has_value()); - _set_constant_mapping(mapping, VExprContext::create_shared(VLiteral::create_shared( - table_column.type, *initial_default))); } else { if (table_column.is_partition_key) { return Status::InvalidArgument( @@ -2195,6 +2211,13 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c const auto* file_child = find_file_child_for_mapping(table_child, file_field, _options.mode, table_child_idx, synthesized_table_children); + if (file_child == nullptr && !synthesized_table_children && + _options.mode == TableColumnMappingMode::BY_FIELD_ID && + _options.allow_idless_complex_wrapper_projection) { + // Parquet can retain an ID-less wrapper at any depth when a selected descendant + // has an ID; apply the same opt-in fallback used for root lookup recursively. + file_child = _find_file_field(table_child, file_field.children); + } if (synthesized_table_children && file_child != nullptr) { const auto file_child_id = file_child->file_local_id(); if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != @@ -2222,8 +2245,8 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; // A missing nested field still has its Iceberg initial-default value in every row // written before the field was added; carry it into recursive materialization. - RETURN_IF_ERROR( - parse_initial_default(table_child, &child_mapping.initial_default_value)); + RETURN_IF_ERROR(build_initial_default_column( + table_child, &child_mapping.initial_default_column)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index c3793777f4cbc6..0b42ac39a5cbea 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -150,7 +150,9 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; - std::optional initial_default_value; + // One-row constant owns variable-width payloads; Field is only a borrowed + // StringView and cannot safely outlive the Base64 decode buffer used to construct it. + ColumnPtr initial_default_column; std::string debug_string() const; }; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 962a414a4c23f1..40fe41783fecd5 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -91,7 +91,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; + VExprSPtr literal; if (table_field.initial_default_value_is_base64 || table_field.type->get_primitive_type() == TYPE_VARBINARY) { // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its @@ -104,19 +104,26 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit table_field.name); } if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); + const auto initial_default = + Field::create_field(StringView(decoded_default)); + // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and + // long FIXED defaults otherwise retain a pointer into freed decode storage. + literal = VLiteral::create_shared(table_field.type, initial_default); } else { DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); + literal = VLiteral::create_shared(table_field.type, + Field::create_field(decoded_default)); } } else { // An added field's initial default is its logical value in every older data file that lacks // the physical column. FE normalizes the string for the current Doris table type. + Field initial_default; RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( *table_field.initial_default_value, initial_default)); + literal = VLiteral::create_shared(table_field.type, initial_default); } - auto literal = VLiteral::create_shared(table_field.type, initial_default); + DORIS_CHECK(literal != nullptr); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index e63485d6f8f513..182ff79ffaccd8 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -491,6 +491,11 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; column->has_name_mapping = context->schema_column->has_name_mapping; + // Projected roots already carry a generic FE default expression, but Iceberg binary defaults + // need the raw Base64 marker so missing-file materialization can decode rather than copy text. + column->initial_default_value = context->schema_column->initial_default_value; + column->initial_default_value_is_base64 = + context->schema_column->initial_default_value_is_base64; return Status::OK(); } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 78e7143570c3d0..6718a79eee6634 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1321,9 +1321,8 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - (child_mapping->initial_default_value.has_value() - ? child_mapping->table_type->create_column_const( - rows, *child_mapping->initial_default_value) + (child_mapping->initial_default_column + ? child_mapping->initial_default_column->clone_resized(rows) : child_mapping->table_type ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6475b79eea541a..9590c9d3f336c5 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -1076,12 +1076,12 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul }; schema::external::TStructField root_field; - root_field.__set_fields( - {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, - "2024-01-01 00:00:00.123456", -1, 6, false), - make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), - make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, - true)}); + root_field.__set_fields({make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, + "Ej5FZ+ibEtOkVkJmFBdAAA==", 16, -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, + "AAEC/w==", -1, -1, true)}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); @@ -1102,7 +1102,7 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul &runtime_state, cache.get()); const auto timestamp_type = make_nullable(std::make_shared(6)); - const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto varbinary_type = make_nullable(std::make_shared(16)); const auto string_type = make_nullable(std::make_shared()); Block block; block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); @@ -1126,7 +1126,7 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), "2024-01-01 00:00:00.123456"); EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), - std::string("\x00\x01\x02\xff", 4)); + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), std::string("\x00\x01\x02\xff", 4)); EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); @@ -1134,6 +1134,53 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); } +TEST_F(IcebergReaderTest, v1_top_level_missing_binary_prefers_iceberg_initial_default) { + auto field = std::make_shared(); + field->__set_name("added_uuid"); + field->__set_id(1); + field->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + field->__set_initial_default_value_is_base64(true); + TColumnType thrift_type; + thrift_type.__set_type(TPrimitiveType::VARBINARY); + thrift_type.__set_len(16); + field->__set_type(thrift_type); + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + schema::external::TStructField root_field; + root_field.__set_fields({std::move(field_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto varbinary_type = make_nullable(std::make_shared(16)); + Block block; + block.insert({varbinary_type->create_column(), varbinary_type, "added_uuid"}); + std::unordered_map positions = {{"added_uuid", 0}}; + reader.TEST_set_column_name_to_block_index(&positions); + + ASSERT_TRUE(reader.on_fill_missing_columns(&block, 2, {"added_uuid"}).ok()); + + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(0).column, 0), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_default) { schema::external::TStructField root_field; root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 89e6c5b6b5b125..761f371d3382e0 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -274,9 +274,28 @@ TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); } +TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { + auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); + auto table_outer = struct_col("outer", 10, {table_inner}); + auto file_inner = struct_name_col("inner", {field_id_col("leaf", 30, i32(), 0)}, 0); + auto file_outer = struct_col("outer", 10, {file_inner}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_outer}, {}, {file_outer}).ok()); + + const auto& outer_mapping = mapper.mappings()[0]; + ASSERT_TRUE(outer_mapping.file_local_id.has_value()); + ASSERT_EQ(outer_mapping.child_mappings.size(), 1); + const auto& inner_mapping = outer_mapping.child_mappings[0]; + ASSERT_TRUE(inner_mapping.file_local_id.has_value()); + ASSERT_EQ(inner_mapping.child_mappings.size(), 1); + EXPECT_TRUE(inner_mapping.child_mappings[0].file_local_id.has_value()); +} + TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { - auto defaulted_child = field_id_col("data", 2, str()); - defaulted_child.initial_default_value = "AAEC/w=="; + auto defaulted_child = field_id_col("data", 2, varbinary()); + defaulted_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; defaulted_child.initial_default_value_is_base64 = true; auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); @@ -285,9 +304,12 @@ TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); const auto& missing = mapper.mappings()[0].child_mappings[1]; - ASSERT_TRUE(missing.initial_default_value.has_value()); - EXPECT_EQ(missing.initial_default_value->get_type(), TYPE_STRING); - EXPECT_EQ(missing.initial_default_value->get(), std::string("\0\1\2\xff", 4)); + ASSERT_TRUE(missing.initial_default_column); + Field value; + missing.initial_default_column->get(0, value); + EXPECT_EQ(value.get_type(), TYPE_VARBINARY); + EXPECT_EQ(std::string(value.get()), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); } void expect_mapping(const ColumnMapping& mapping, size_t global_index, diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index b340eabd978add..b28c179e391b16 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -495,6 +495,30 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value) { + const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); + auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); + auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); + ASSERT_TRUE(inner_result.ok()) << inner_result.status(); + auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); + auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + const auto outer_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"10"}); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false) + ->WithMetadata(outer_metadata); + auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -2245,7 +2269,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + const std::string binary_default( + "\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2254,10 +2279,10 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "Ej5FZ+ibEtOkVkJmFBdAAA==", + external_primitive_type(TPrimitiveType::VARBINARY, 16), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -2448,6 +2473,56 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_recursive_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 316e708dc177fb..d4c52d5c1fbcb2 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -41,6 +41,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -48,6 +49,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" @@ -1942,6 +1944,60 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, IcebergInitialDefaultMetadataOverridesGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_top_level_binary_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto int_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); + auto id_column = make_table_column(-1, "id", int_type); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + ASSERT_TRUE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(binary_column.initial_default_value_is_base64); + + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { auto unmapped_field = external_schema_field("b", 2); unmapped_field.field_ptr->__set_name_mapping({}); @@ -4087,12 +4143,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) write_struct_parquet_file(file_path, 7); const auto int_type = std::make_shared(); - const auto string_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); - missing_child.initial_default_value = "AAEC/w=="; + auto missing_child = make_table_column(99, "missing_child", varbinary_type); + missing_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; missing_child.initial_default_value_is_base64 = true; - auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + auto struct_type = std::make_shared(DataTypes {int_type, varbinary_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); struct_column.children = {id_child, missing_child}; @@ -4126,9 +4182,10 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - const auto& defaults = assert_cast( + const auto& defaults = assert_cast( expect_not_null_nullable_nested_column(struct_result.get_column(1))); - EXPECT_EQ(defaults.get_data_at(0).to_string(), std::string("\0\1\2\xff", 4)); + EXPECT_EQ(defaults.get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index b6f1d22046d76e..62150c564004f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1115,8 +1115,18 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + String serializedDefault = serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz); + // Column constructs complex children without Iceberg field metadata. Copy through the + // public default-info API so recursive fields retain their logical pre-add value. + Column defaultCarrier = new Column(column.getName(), column.getType(), false, null, + column.isAllowNull(), serializedDefault, ""); + column.setDefaultValueInfo(defaultCarrier); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1135,7 +1145,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1192,7 +1203,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 867d7c6fa26054..a75226a9c065cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -552,12 +552,16 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } + IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); + if (selectedSnapshot == null) { + // A schema-only update does not create a data snapshot. Ordinary scans expose current + // table columns, so their default metadata must come from that same current schema. + return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); + } TableScan tableScan = createTableScan(); Snapshot snapshot = tableScan.snapshot(); - // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. - // Resolve the selected snapshot's schema id explicitly so this metadata describes the same - // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make - // BE decode a historical non-binary default as Base64, or fail to decode a binary default. + // Explicit time travel and ref scans expose the selected snapshot schema rather than the + // current table schema, so resolve its schema id instead of using TableScan.schema(). Schema scanSchema = snapshot == null ? tableScan.schema() : tableScan.table().schemas().get(snapshot.schemaId()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3bee2d9747f02d..9422f08812de2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -269,6 +269,26 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + @Test + public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { + StructType structType = new StructType( + new StructField("added_int", Type.INT, "nested default", true)); + Column structColumn = new Column("s", structType, true); + structColumn.setUniqueId(10); + Column child = structColumn.getChildren().get(0); + child.setUniqueId(11); + child.setDefaultValueInfo(new Column("added_int", Type.INT, false, null, true, "7", "")); + TFileScanRangeParams params = new TFileScanRangeParams(); + + ExternalUtil.initSchemaInfoForAllColumn( + params, 1L, Collections.singletonList(structColumn), Collections.emptyMap()); + + TField childField = params.getHistorySchemaInfo().get(0).getRootField().getFields().get(0) + .getFieldPtr().getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals("7", childField.getInitialDefaultValue()); + Assert.assertFalse(childField.isSetInitialDefaultValueIsBase64()); + } + @Test public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { TFileScanRangeParams params = new TFileScanRangeParams(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 349e1e0bb82ab1..f29ccd6182bd9f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -364,6 +364,20 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); } + @Test + public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional("added_int") + .withId(11) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build()))); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 39072774cd73e2..7eb74451eeeee7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -182,7 +182,7 @@ public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws } @Test - public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") .withId(7) .ofType(Types.BinaryType.get()) @@ -197,6 +197,7 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Mockito.when(snapshot.schemaId()).thenReturn(11); Table table = Mockito.mock(Table.class); Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + Mockito.when(table.schema()).thenReturn(currentSchema); TableScan snapshotScan = Mockito.mock(TableScan.class); Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); @@ -204,8 +205,33 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); + setIcebergTable(node, table); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertTrue(defaults.isEmpty()); + } + + @Test + public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(snapshotScan); + Mockito.doReturn(Mockito.mock(IcebergTableQueryInfo.class)).when(node).getSpecifiedSnapshot(); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } From b4909eb6c092776e4d4af88ffda539ceb2fdc3bc Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 19:31:07 +0800 Subject: [PATCH 06/10] [fix](iceberg) align pinned schemas and idless wrappers --- .../table/table_schema_change_helper.cpp | 87 +++++++++ be/src/format_v2/column_mapper.cpp | 16 +- .../table/iceberg/iceberg_reader_test.cpp | 168 ++++++++++++++++++ .../table/table_schema_change_helper_test.cpp | 69 +++++++ be/test/format_v2/column_mapper_test.cpp | 16 ++ .../format_v2/table/iceberg_reader_test.cpp | 63 ++++++- .../iceberg/source/IcebergScanNode.java | 17 +- .../iceberg/source/IcebergScanNodeTest.java | 66 +++++++ 8 files changed, 490 insertions(+), 12 deletions(-) diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 50c2b8eabd37db..b93c55459c997b 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -108,6 +108,80 @@ bool find_file_field_idx_by_name_mapping( return table_field.__isset.name && try_match(table_field.name); } +bool table_subtree_contains_field_id(const schema::external::TField& field, int32_t field_id) { + if (field.id == field_id) { + return true; + } + if (!field.__isset.nestedField) { + return false; + } + switch (field.type.type) { + case TPrimitiveType::STRUCT: + if (field.nestedField.__isset.struct_field) { + for (const auto& child : field.nestedField.struct_field.fields) { + if (child.field_ptr != nullptr && + table_subtree_contains_field_id(*child.field_ptr, field_id)) { + return true; + } + } + } + break; + case TPrimitiveType::ARRAY: + if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field && + field.nestedField.array_field.item_field.field_ptr != nullptr) { + return table_subtree_contains_field_id( + *field.nestedField.array_field.item_field.field_ptr, field_id); + } + break; + case TPrimitiveType::MAP: + if (field.nestedField.__isset.map_field) { + const auto& map = field.nestedField.map_field; + if (map.__isset.key_field && map.key_field.field_ptr != nullptr && + table_subtree_contains_field_id(*map.key_field.field_ptr, field_id)) { + return true; + } + if (map.__isset.value_field && map.value_field.field_ptr != nullptr) { + return table_subtree_contains_field_id(*map.value_field.field_ptr, field_id); + } + } + break; + default: + break; + } + return false; +} + +bool has_shared_descendant_field_id(const schema::external::TField& table_field, + const FieldSchema& file_field) { + for (const auto& file_child : file_field.children) { + if ((file_child.field_id != -1 && + table_subtree_contains_field_id(table_field, file_child.field_id)) || + has_shared_descendant_field_id(table_field, file_child)) { + return true; + } + } + return false; +} + +std::optional find_unique_idless_wrapper( + const schema::external::TField& table_field, + const std::vector& parquet_fields_schema) { + std::optional match; + for (size_t idx = 0; idx < parquet_fields_schema.size(); ++idx) { + const auto& candidate = parquet_fields_schema[idx]; + if (candidate.field_id != -1 || candidate.children.empty() || + !has_shared_descendant_field_id(table_field, candidate)) { + continue; + } + if (match.has_value()) { + return std::nullopt; + } + match = idx; + } + return match; +} + } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -611,6 +685,13 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (auto wrapper = find_unique_idless_wrapper(*table_field.field_ptr, + parquet_fields_schema); + wrapper.has_value()) { + // Parquet may retain a selected struct without its own ID; a unique descendant-ID + // match is authoritative even when Iceberg name mapping intentionally has no alias. + file_column_idx = *wrapper; + matched = true; } } else { matched = find_file_field_idx_by_name_mapping( @@ -716,6 +797,12 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (auto wrapper = find_unique_idless_wrapper(*table_field.field_ptr, + parquet_field.children); + wrapper.has_value()) { + // Apply the same Parquet wrapper invariant at every struct recursion depth. + file_column_idx = *wrapper; + matched = true; } } else { matched = find_file_field_idx_by_name_mapping( diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 542608f77b00d8..9fbe05e7cfe4b4 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2166,13 +2166,19 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { return matched; } - const auto* wrapper = find_column_by_name(table_column, file_schema); - if (wrapper == nullptr || wrapper->has_identifier_field_id() || wrapper->children.empty() || - !has_shared_descendant_field_id(table_column, *wrapper)) { - return nullptr; + const ColumnDefinition* wrapper = nullptr; + for (const auto& candidate : file_schema) { + if (candidate.has_identifier_field_id() || candidate.children.empty() || + !has_shared_descendant_field_id(table_column, candidate)) { + continue; + } + if (wrapper != nullptr) { + return nullptr; + } + wrapper = &candidate; } // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is - // selected. ORC instead synthesizes the missing parent, so this fallback is opt-in by format. + // selected. Descendant IDs, not aliases, identify that wrapper; ambiguity remains unmapped. return wrapper; } diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 9590c9d3f336c5..97f1507fd35c70 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -183,6 +183,29 @@ void write_iceberg_three_int_parquet_file( properties.build())); } +void write_iceberg_id_and_idless_struct_parquet_file(const std::string& file_path, int32_t id, + int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = + arrow::field("a", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = arrow::field("s", arrow::struct_({child_field}), false); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, const std::string& field_name, int32_t field_id, int32_t value) { @@ -245,6 +268,56 @@ Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::st return Status::OK(); } +Status create_single_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TStructField child_field; + child_field.__set_name("a"); + child_field.__set_contains_null(true); + TTypeNode struct_node; + struct_node.__set_type(TTypeNodeType::STRUCT); + struct_node.__set_struct_fields({child_field}); + TTypeNode child_node; + child_node.__set_type(TTypeNodeType::SCALAR); + TScalarType child_scalar; + child_scalar.__set_type(TPrimitiveType::INT); + child_node.__set_scalar_type(child_scalar); + TTypeDesc type; + type.__set_types({struct_node, child_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(10); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(-1); + slot_descriptor.__set_colName("s"); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(16); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1346,6 +1419,101 @@ TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default std::filesystem::remove_all(test_dir); } +TEST_F(IcebergReaderTest, v1_parquet_reads_idless_wrapper_with_authoritative_empty_mapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_idless_struct_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":42}"); + + std::filesystem::remove_all(test_dir); +} + TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 1895fee300b89c..1380b185000131 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -595,6 +595,75 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistin EXPECT_TRUE(default_value->is_base64); } +TEST(MockTableSchemaChangeHelper, + IcebergParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_fields; + struct_fields.__set_fields({child_ptr}); + + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_fields); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "a"; + file_child.field_id = 2; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = -1; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 761f371d3382e0..567f44dea2bb56 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -274,6 +274,22 @@ TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); } +TEST(ColumnMapperTest, ParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + auto table_struct = struct_col("s", 10, {field_id_col("a", 2, i32())}); + table_struct.has_name_mapping = true; + auto file_struct = struct_name_col("s", {field_id_col("a", 2, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + EXPECT_TRUE(mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); +} + TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); auto table_outer = struct_col("outer", 10, {table_inner}); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index b28c179e391b16..c3430125ac5826 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -495,7 +495,8 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } -void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value) { +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value, + bool outer_has_field_id = true) { const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); @@ -503,9 +504,11 @@ void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, i auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); ASSERT_TRUE(outer_result.ok()) << outer_result.status(); - const auto outer_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"10"}); - auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false) - ->WithMetadata(outer_metadata); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false); + if (outer_has_field_id) { + outer_field = + outer_field->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + } auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); auto file_result = arrow::io::FileOutputStream::Open(file_path); @@ -2523,6 +2526,58 @@ TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafI std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42, false); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + inner.has_name_mapping = true; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + outer.has_name_mapping = true; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index a75226a9c065cb..df641d149581d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -40,12 +40,15 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.profile.IcebergMetricsReporter; import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.persist.gson.GsonUtils; @@ -554,9 +557,17 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti } IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); if (selectedSnapshot == null) { - // A schema-only update does not create a data snapshot. Ordinary scans expose current - // table columns, so their default metadata must come from that same current schema. - return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = icebergTable.schema(); + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } + // The statement snapshot produced the target columns; cache invalidation must not + // let initial-default metadata advance to a different schema during the same scan. + return IcebergUtils.getBase64EncodedInitialDefaults( + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } TableScan tableScan = createTableScan(); Snapshot snapshot = tableScan.snapshot(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 7eb74451eeeee7..92f4a04f06ebfa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -22,12 +22,23 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileFormatType; @@ -206,11 +217,60 @@ public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(TableIf.class)); + setIcebergSource(node, source); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); Assert.assertTrue(defaults.isEmpty()); } + @Test + public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { + Schema pinnedSchema = new Schema(11, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Schema refreshedSchema = new Schema(12, + List.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); + Mockito.when(refreshedTable.schemas()).thenReturn(Map.of( + pinnedSchema.schemaId(), pinnedSchema, + refreshedSchema.schemaId(), refreshedSchema)); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, pinnedSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") @@ -265,6 +325,12 @@ private static void setIcebergTable(IcebergScanNode node, Table table) throws Ex icebergTableField.set(node, table); } + private static void setIcebergSource(IcebergScanNode node, IcebergSource source) throws Exception { + Field sourceField = IcebergScanNode.class.getDeclaredField("source"); + sourceField.setAccessible(true); + sourceField.set(node, source); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); From f1675c692328649863e402cba3646f5b156813d5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 20:54:37 +0800 Subject: [PATCH 07/10] [fix](iceberg) pin name mapping and parquet id mode --- .../table/table_schema_change_helper.cpp | 46 ++++--- .../format/table/table_schema_change_helper.h | 4 + .../table/iceberg/iceberg_reader_test.cpp | 118 ++++++++++++++++++ .../table/table_schema_change_helper_test.cpp | 64 ++++++++++ .../iceberg/IcebergExternalMetaCache.java | 3 +- .../iceberg/IcebergSnapshotCacheValue.java | 20 +++ .../datasource/iceberg/IcebergUtils.java | 44 ++++++- .../iceberg/source/IcebergScanNode.java | 43 +------ .../iceberg/source/IcebergScanNodeTest.java | 39 ++++++ 9 files changed, 324 insertions(+), 57 deletions(-) diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index b93c55459c997b..841b008d99b9e1 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -182,6 +182,13 @@ std::optional find_unique_idless_wrapper( return match; } +bool parquet_subtree_has_field_id(const FieldSchema& field) { + if (field.field_id != -1) { + return true; + } + return std::ranges::any_of(field.children, parquet_subtree_has_field_id); +} + } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -661,18 +668,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& parquet_fields_schema = parquet_field_desc.get_fields_schema(); std::map file_column_id_idx_map; - // Iceberg considers the schema ID-bearing when any field has an ID; requiring all IDs would - // discard authoritative matches merely because an unrelated sibling is ID-less. - bool has_field_id = false; + // Iceberg's hasIds predicate is file-wide. Keep this decision through recursion so an ID-less + // nested struct cannot switch an ID-bearing file back to alias/name matching. + const bool use_field_id = + std::ranges::any_of(parquet_fields_schema, parquet_subtree_has_field_id); for (size_t idx = 0; idx < parquet_fields_schema.size(); idx++) { if (parquet_fields_schema[idx].field_id != -1) { - has_field_id = true; file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } } std::map file_column_name_idx_map; - if (!has_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_fields_schema); } @@ -680,7 +687,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (has_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; @@ -706,7 +713,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam std::shared_ptr field_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( - *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node)); + *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node, + use_field_id)); struct_node->add_children(table_column_name, parquet_fields_schema[file_column_idx].name, field_node); } @@ -718,6 +726,13 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node) { + return by_parquet_field_id_with_name_mapping(table_schema, parquet_field, node, + parquet_subtree_has_field_id(parquet_field)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (parquet_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] { @@ -737,10 +752,10 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, parquet_field.children[0], - key_node)); + key_node, use_field_id)); RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, - parquet_field.children[1], value_node)); + parquet_field.children[1], value_node, use_field_id)); node = std::make_shared(key_node, value_node); break; @@ -759,7 +774,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, - parquet_field.children[0], element_node)); + parquet_field.children[0], element_node, use_field_id)); node = std::make_shared(element_node); break; @@ -774,17 +789,14 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - // Apply the same any-ID rule recursively so nested mixed-ID structs retain ID matches. - bool has_field_id = false; for (size_t idx = 0; idx < parquet_field.children.size(); idx++) { if (parquet_field.children[idx].field_id != -1) { - has_field_id = true; file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } } std::map file_column_name_idx_map; - if (!has_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_field.children); } @@ -792,7 +804,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (has_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; @@ -817,8 +829,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& file_field = parquet_field.children.at(file_column_idx); std::shared_ptr field_node = nullptr; - RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping(*table_field.field_ptr, - file_field, field_node)); + RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( + *table_field.field_ptr, file_field, field_node, use_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } node = struct_node; diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 6b1adb138fc962..c148f9aa8b1346 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -352,6 +352,10 @@ class TableSchemaChangeHelper { const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node); + static Status by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id); + // for iceberg orc : Use the field id in the `table schema` and the orc file to match columns. static Status by_orc_field_id(const schema::external::TStructField& table_schema, const orc::Type* orc_root, diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 97f1507fd35c70..f45a8b40d1aa4f 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -206,6 +206,29 @@ void write_iceberg_id_and_idless_struct_parquet_file(const std::string& file_pat 1, properties.build())); } +void write_iceberg_id_and_struct_with_idless_child_parquet_file(const std::string& file_path, + int32_t id, int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = arrow::field("legacy_a", arrow::int32(), false); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, const std::string& field_name, int32_t field_id, int32_t value) { @@ -1514,6 +1537,101 @@ TEST_F(IcebergReaderTest, v1_parquet_reads_idless_wrapper_with_authoritative_emp std::filesystem::remove_all(test_dir); } +TEST_F(IcebergReaderTest, v1_parquet_keeps_file_id_mode_inside_nested_struct) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_whole_file_id_mode_nested_struct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_struct_with_idless_child_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + child_field->__set_initial_default_value("7"); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":7}"); + + std::filesystem::remove_all(test_dir); +} + TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 1380b185000131..96c32f3ca55248 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -664,6 +664,70 @@ TEST(MockTableSchemaChangeHelper, " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetKeepsWholeFileIdModeInNestedStruct) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "legacy_a"; + file_child.field_id = -1; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = 10; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index bd16057dfd9569..90b0fd90e5b97b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -231,7 +231,8 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTabl icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 95c9a6f26cc5c5..a17ff87b1131c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,14 +17,30 @@ package org.apache.doris.datasource.iceberg; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; + private final Optional>> nameMapping; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { + this(partitionInfo, snapshot, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.nameMapping = nameMapping.map(mapping -> { + Map> copy = new HashMap<>(); + mapping.forEach((id, names) -> copy.put(id, List.copyOf(names))); + return Map.copyOf(copy); + }); } public IcebergPartitionInfo getPartitionInfo() { @@ -34,4 +50,8 @@ public IcebergPartitionInfo getPartitionInfo() { public IcebergSnapshot getSnapshot() { return snapshot; } + + public Optional>> getNameMapping() { + return nameMapping; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 62150c564004f3..6000fd2efaf9f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -108,6 +108,10 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mapping.MappedField; +import org.apache.iceberg.mapping.MappedFields; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; @@ -1830,7 +1834,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( } return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId())); + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(icebergTable)); } return getLatestSnapshotCacheValue(dorisTable); } @@ -1911,12 +1916,47 @@ private static IcebergSnapshotCacheValue loadSnapshotCacheValue(ExternalTable do icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } } + /** + * Extract the Iceberg name mapping while retaining the distinction between an absent property + * and a valid empty mapping. + */ + public static Optional>> getNameMapping(Table icebergTable) { + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } + try { + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); + } + Map> result = new HashMap<>(); + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); + } catch (Exception e) { + LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); + } + } + + private static void extractMappingsFromNameMapping( + MappedFields mappingFields, Map> result) { + if (mappingFields == null) { + return; + } + for (MappedField mappedField : mappingFields.fields()) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + } + } + private static Table loadIcebergTableWithSession(ExternalTable dorisTable) { IcebergExternalCatalog catalog = (IcebergExternalCatalog) dorisTable.getCatalog(); IcebergMetadataOps ops = (IcebergMetadataOps) catalog.getMetadataOps(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index df641d149581d9..29de378ee247a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -96,7 +96,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Binder; import org.apache.iceberg.expressions.Expression; @@ -106,10 +105,6 @@ import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.mapping.MappedField; -import org.apache.iceberg.mapping.MappedFields; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.NestedField; @@ -271,40 +266,14 @@ protected void doInitialize() throws UserException { super.doInitialize(); } - /** - * Extract name mapping from Iceberg table properties. - * Returns a present map, possibly empty, only when the property parsed successfully. - */ private Optional>> extractNameMapping() { - Map> result = new HashMap<>(); - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson == null || nameMappingJson.isEmpty()) { - return Optional.empty(); - } - try { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping == null) { - return Optional.empty(); - } - // Optional presence is semantic here: a valid [] still disables current-name fallback. - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - return Optional.of(result); - } catch (Exception e) { - // If name mapping parsing fails, continue without it - LOG.warn("Failed to parse name mapping from Iceberg table properties", e); - return Optional.empty(); + Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + // The mapping must come from the same metadata generation as the pinned schema; a + // property-only refresh can otherwise change alias semantics within one statement. + return ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getNameMapping(); } - } - - private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { - if (mappingFields == null) { - return; - } - for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); - extractMappingsFromNameMapping(mappedField.nestedMapping(), result); - } - + return IcebergUtils.getNameMapping(icebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 92f4a04f06ebfa..53136d04d66c75 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -134,6 +134,9 @@ public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); Table table = Mockito.mock(Table.class); setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + setIcebergSource(node, source); Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); Assert.assertFalse(extractNameMapping(node).isPresent()); @@ -145,6 +148,42 @@ public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception Assert.assertTrue(emptyMapping.get().isEmpty()); } + @Test + public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 11L), Optional.empty()))); + try { + Assert.assertFalse(extractNameMapping(node).isPresent()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { Schema systemTableSchema = new Schema( From 512c431cfd079101e129d2eec307c265f0e35dc6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 10:19:05 +0800 Subject: [PATCH 08/10] [fix](iceberg) address review feedback and FE UT --- ...eberg_position_delete_sys_table_reader.cpp | 21 +- be/src/format_v2/table/iceberg_reader.h | 26 +-- be/src/format_v2/table/iceberg_schema_utils.h | 40 ++++ ..._position_delete_sys_table_reader_test.cpp | 185 ++++++++++++++++++ .../datasource/iceberg/IcebergUtils.java | 6 +- .../iceberg/IcebergDDLAndDMLPlanTest.java | 5 + .../iceberg/source/IcebergScanNodeTest.java | 22 +++ 7 files changed, 269 insertions(+), 36 deletions(-) create mode 100644 be/src/format_v2/table/iceberg_schema_utils.h diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..2e5cf41286bef3 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -35,6 +35,7 @@ #include "core/types.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -132,24 +133,14 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) { } } -bool has_field_id(const std::vector& schema) { - for (const auto& field : schema) { - if (!field.has_identifier_field_id()) { - return false; - } - if (!has_field_id(field.children)) { - return false; - } - } - return true; -} - class PositionDeleteFileTableReader final : public format::TableReader { protected: format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + if (!_data_reader.file_schema.empty() && + schema_has_any_field_id(_data_reader.file_schema)) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } }; diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 1f776df8dba3e5..afd3ee53913f9d 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -27,6 +27,7 @@ #include "core/block/block.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format_v2/file_reader.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" @@ -57,9 +58,11 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_any_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + if (!_data_reader.file_schema.empty() && + schema_has_any_field_id(_data_reader.file_schema)) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } protected: @@ -80,23 +83,6 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - - bool _has_any_field_id(const std::vector& schema) const { - for (const auto& field : schema) { - // Iceberg's hasIds contract is existential. Ignore synthesized columns and keep ID - // projection as soon as any real field (including a nested field) carries an ID. - if (field.column_type != format::ColumnType::DATA_COLUMN) { - continue; - } - if (field.has_identifier_field_id()) { - return true; - } - if (_has_any_field_id(field.children)) { - return true; - } - } - return false; - } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; static constexpr int EQUALITY_DELETE = 2; diff --git a/be/src/format_v2/table/iceberg_schema_utils.h b/be/src/format_v2/table/iceberg_schema_utils.h new file mode 100644 index 00000000000000..ff91e8018447e8 --- /dev/null +++ b/be/src/format_v2/table/iceberg_schema_utils.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/column_data.h" + +namespace doris::format::iceberg { + +inline bool schema_has_any_field_id(const std::vector& schema) { + for (const auto& field : schema) { + // Iceberg's hasIds contract is existential for the whole file. Ignore synthesized columns + // and retain ID projection when any real field, including a nested field, carries an ID. + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (field.has_identifier_field_id() || schema_has_any_field_id(field.children)) { + return true; + } + } + return false; +} + +} // namespace doris::format::iceberg diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index bc3f916281de7d..da4f52484c788f 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -17,9 +17,15 @@ #include "format/table/iceberg_position_delete_sys_table_reader.h" +#include +#include #include +#include +#include +#include #include +#include #include #include #include @@ -33,6 +39,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "format/orc/orc_memory_stream_test.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -43,6 +50,8 @@ namespace doris { namespace { +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file); + class ProfileTrackingReader final : public GenericReader { public: int collect_calls = 0; @@ -115,6 +124,173 @@ Int64 struct_int_at(const Block& block, const std::string& name, size_t child_in return child.get_nested_column().get_int(row); } +bool struct_child_is_null_at(const Block& block, const std::string& name, size_t child_index, + size_t row) { + const auto& struct_column = assert_cast(nested_column(block, name)); + const auto& child = assert_cast(struct_column.get_column(child_index)); + return child.is_null_at(row); +} + +std::shared_ptr int32_array(int32_t value) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(value).ok()); + auto result = builder.Finish(); + EXPECT_TRUE(result.ok()) << result.status(); + return result.ok() ? *result : nullptr; +} + +void write_mixed_id_position_delete_parquet(const std::string& path) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false)->WithMetadata(id(1)); + auto idless_b = arrow::field("b", arrow::int32(), false); + auto row_array = + arrow::StructArray::Make({int32_array(42), int32_array(99)}, {legacy_a, idless_b}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + arrow::field("row", arrow::struct_({legacy_a, idless_b}), false) + ->WithMetadata(id(2147483544)), + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_mixed_id_position_delete_orc(const std::string& path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>")); + type->getSubtype(0)->setAttribute("iceberg.id", "2147483546"); + type->getSubtype(1)->setAttribute("iceberg.id", "2147483545"); + type->getSubtype(2)->setAttribute("iceberg.id", "2147483544"); + type->getSubtype(2)->getSubtype(0)->setAttribute("iceberg.id", "1"); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& file_path = dynamic_cast<::orc::StringVectorBatch&>(*root.fields[0]); + auto& pos = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[1]); + auto& row = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[2]); + auto& legacy_a = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[0]); + auto& idless_b = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[1]); + std::string data_path = "s3://bucket/data.orc"; + file_path.data[0] = data_path.data(); + file_path.length[0] = data_path.size(); + pos.data[0] = 5; + legacy_a.data[0] = 42; + idless_b.data[0] = 99; + root.numElements = file_path.numElements = pos.numElements = row.numElements = + legacy_a.numElements = idless_b.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +format::ColumnDefinition table_column(int32_t id, std::string name, DataTypePtr type) { + format::ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = std::move(name); + column.type = std::move(type); + return column; +} + +void run_mixed_id_position_delete_test(format::FileFormat file_format, + TFileFormatType::type thrift_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_iceberg_position_delete_mixed_id_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == format::FileFormat::PARQUET) { + write_mixed_id_position_delete_parquet(path); + } else { + write_mixed_id_position_delete_orc(path); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + auto a = table_column(1, "a", nullable_int32); + a.name_mapping = {"legacy_a"}; + a.has_name_mapping = true; + auto b = table_column(2, "b", nullable_int32); + auto row = table_column(2147483544, "row", row_type); + row.children = {a, b}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(thrift_format); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = file_format, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(thrift_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = file_format; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + ASSERT_FALSE(struct_child_is_null_at(block, "row", 0, 0)); + EXPECT_EQ(42, struct_int_at(block, "row", 0, 0)); + // Once any file field has an Iceberg id, the id-less sibling is absent rather than name-bound. + EXPECT_TRUE(struct_child_is_null_at(block, "row", 1, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { TIcebergFileDesc iceberg_desc; iceberg_desc.__set_delete_files({delete_file}); @@ -603,4 +779,13 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVect EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "parquet"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 6000fd2efaf9f1..755bf83afdd959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1952,7 +1952,11 @@ private static void extractMappingsFromNameMapping( return; } for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + // Iceberg permits id-less wrapper entries; only their nested ID-bearing aliases can + // participate in Doris field-id lookup and in the immutable snapshot cache. + if (mappedField.id() != null) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + } extractMappingsFromNameMapping(mappedField.nestedMapping(), result); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 0a798faff93fef..b4a1c6a5d1b235 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -188,6 +188,11 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + // The scan now resolves initial defaults from the statement-pinned schema id, so the + // mocked table must expose the same historical-schema lookup as a real Iceberg table. + Mockito.doAnswer(invocation -> ImmutableMap.of( + mockedIcebergTable.schema().schemaId(), mockedIcebergTable.schema())) + .when(mockedIcebergTable).schemas(); // Mock newScan() chain used by IcebergScanNode.createTableScan() TableScan mockedTableScan = Mockito.mock(TableScan.class, Mockito.RETURNS_DEEP_STUBS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 53136d04d66c75..b2529bf9673857 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -34,6 +34,7 @@ import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; import org.apache.doris.datasource.iceberg.IcebergSnapshot; import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.mvcc.MvccTableInfo; import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; @@ -148,6 +149,27 @@ public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception Assert.assertTrue(emptyMapping.get().isEmpty()); } + @Test + public void testSnapshotCacheIgnoresIdlessNameMappingWrapper() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap( + TableProperties.DEFAULT_NAME_MAPPING, + "[{\"names\":[\"legacy_wrapper\"],\"fields\":[" + + "{\"field-id\":7,\"names\":[\"legacy_child\"]}]}]")); + + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), + IcebergUtils.getNameMapping(table)); + + Assert.assertTrue(snapshotCacheValue.getNameMapping().isPresent()); + Assert.assertEquals(Collections.singletonList("legacy_child"), + snapshotCacheValue.getNameMapping().get().get(7)); + Assert.assertEquals(Collections.singleton(7), + snapshotCacheValue.getNameMapping().get().keySet()); + } + @Test public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { Table refreshedTable = Mockito.mock(Table.class); From c4f800c85117c718e36b85e0edcefe791bf8ab7e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 12:03:43 +0800 Subject: [PATCH 09/10] [fix](iceberg) preserve file-wide schema projection --- ...eberg_position_delete_sys_table_reader.cpp | 84 +++---- .../table/table_schema_change_helper.cpp | 39 ++- .../format/table/table_schema_change_helper.h | 10 + ...eberg_position_delete_sys_table_reader.cpp | 6 + ..._position_delete_sys_table_reader_test.cpp | 236 +++++++++++++++++- .../table/iceberg/iceberg_reader_test.cpp | 230 +++++++++++++++++ .../iceberg/source/IcebergScanNode.java | 35 ++- .../iceberg/source/IcebergScanNodeTest.java | 65 ++++- 8 files changed, 616 insertions(+), 89 deletions(-) diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp index c6e97465e2dd44..0ade539cf241c0 100644 --- a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -99,15 +99,7 @@ const ColumnInt64* get_int64_column(const Block& block, const std::string& name) return check_and_get_column(block.get_by_position(pos).column.get()); } -const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { - if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { - return nullptr; - } - return field_ptr.field_ptr.get(); -} - -const schema::external::TField* find_current_schema_field(const TFileScanRangeParams* params, - const std::string& name) { +const schema::external::TSchema* find_current_schema(const TFileScanRangeParams* params) { if (params == nullptr || !params->__isset.history_schema_info || params->history_schema_info.empty()) { return nullptr; @@ -121,16 +113,7 @@ const schema::external::TField* find_current_schema_field(const TFileScanRangePa } } } - if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { - return nullptr; - } - for (const auto& field_ptr : schema->root_field.fields) { - const auto* field = get_field_ptr(field_ptr); - if (field != nullptr && field->__isset.name && field->name == name) { - return field; - } - } - return nullptr; + return schema; } template @@ -248,13 +231,23 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { &_state->timezone_obj(), _io_ctx, _state, _meta_cache); const FieldDescriptor* schema = nullptr; - int row_index = -1; + std::shared_ptr mapped_file_schema; if (row_requested) { RETURN_IF_ERROR(parquet_reader->get_file_metadata_schema(&schema)); DORIS_CHECK(schema != nullptr); - row_index = schema->get_column_index(kRowColumn); + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + // Position-delete mapping mode is file-wide: file_path/pos IDs must prevent an + // ID-less physical row from being rebound by name in either reader generation. + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: + by_parquet_field_id_with_name_mapping( + table_schema->root_field, *schema, mapped_file_schema)); } - const bool read_row = row_requested && row_index >= 0; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -270,18 +263,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { pctx.range = &_range; pctx.filter_groups = false; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - const auto* file_row_field = schema->get_column(static_cast(row_index)); - std::shared_ptr row_node; - RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: - by_parquet_field_id_with_name_mapping( - *table_row_field, *file_row_field, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, file_row_field->name, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); pctx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(parquet_reader.get())->init_reader(&pctx)); @@ -294,19 +279,25 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { OrcReader::create_unique(_profile, _state, *_range_params, _range, _batch_size, _state->timezone(), _io_ctx, _meta_cache); - const orc::Type* row_type = nullptr; + std::shared_ptr mapped_file_schema; if (row_requested) { const orc::Type* root_type = nullptr; RETURN_IF_ERROR(orc_reader->get_file_type(&root_type)); DORIS_CHECK(root_type != nullptr); - for (uint64_t i = 0; i < root_type->getSubtypeCount(); ++i) { - if (root_type->getFieldName(i) == kRowColumn) { - row_type = root_type->getSubtype(i); - break; - } + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); } + // Resolve row against the complete delete-file type so top-level IDs keep ORC in ID + // projection throughout the nested row subtree. + RETURN_IF_ERROR( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + table_schema->root_field, root_type, kIcebergOrcAttribute, + mapped_file_schema)); } - const bool read_row = row_requested && row_type != nullptr; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -321,17 +312,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { octx.params = _range_params; octx.range = &_range; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - std::shared_ptr row_node; - RETURN_IF_ERROR( - TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( - *table_row_field, row_type, kIcebergOrcAttribute, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, kRowColumn, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); octx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(orc_reader.get())->init_reader(&octx)); diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 841b008d99b9e1..b8a140dfacc5e8 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -955,12 +955,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TStructField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping( + table_schema, orc_root, field_id_attribute_key, node, + orc_subtree_has_field_id(orc_root, field_id_attribute_key)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id) { auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - // Iceberg ORC builds projection by ID for the entire subtree. An ID-less wrapper containing - // an ID-bearing child is therefore absent rather than rebound by its physical name. - bool has_field_id = orc_subtree_has_field_id(orc_root, field_id_attribute_key); for (size_t idx = 0; idx < orc_root->getSubtypeCount(); idx++) { if (orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { auto field_id = @@ -970,7 +976,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } std::map file_column_name_idx_map; - if (!has_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_orc_field_name_idx_map(orc_root); } @@ -978,7 +984,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const auto& table_column_name = table_field.field_ptr->name; size_t file_field_idx = 0; bool matched = false; - if (has_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_field_idx = id_it->second; @@ -998,7 +1004,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const auto& file_field = orc_root->getSubtype(file_field_idx); std::shared_ptr field_node = nullptr; RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(*table_field.field_ptr, file_field, - field_id_attribute_key, field_node)); + field_id_attribute_key, field_node, + use_field_id)); struct_node->add_children(table_column_name, orc_root->getFieldName(file_field_idx), field_node); } @@ -1010,6 +1017,15 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping( + table_schema, orc_root, field_id_attribute_key, node, + orc_subtree_has_field_id(orc_root, field_id_attribute_key)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (orc_root->getKind() != orc::TypeKind::MAP) [[unlikely]] { @@ -1029,10 +1045,10 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, key_node)); + field_id_attribute_key, key_node, use_field_id)); RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, orc_root->getSubtype(1), - field_id_attribute_key, value_node)); + field_id_attribute_key, value_node, use_field_id)); node = std::make_shared(key_node, value_node); break; @@ -1051,7 +1067,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, element_node)); + field_id_attribute_key, element_node, use_field_id)); node = std::make_shared(element_node); break; @@ -1062,8 +1078,11 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } MOCK_REMOVE(DCHECK(table_schema.__isset.nestedField)); MOCK_REMOVE(DCHECK(table_schema.nestedField.__isset.struct_field)); + // Iceberg chooses ID projection once for the complete ORC file; container recursion must + // not re-enable name matching merely because a nested struct has no local IDs. RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(table_schema.nestedField.struct_field, - orc_root, field_id_attribute_key, node)); + orc_root, field_id_attribute_key, node, + use_field_id)); break; } default: { diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index c148f9aa8b1346..45cce284014701 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -377,11 +377,21 @@ class TableSchemaChangeHelper { const std::string& field_id_attribute_key, std::shared_ptr& node); + static Status by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id); + // for iceberg orc static Status by_orc_field_id_with_name_mapping( const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node); + + static Status by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id); }; }; diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index 2e5cf41286bef3..f3491fd2d237a1 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -142,6 +142,12 @@ class PositionDeleteFileTableReader final : public format::TableReader { } return format::TableColumnMappingMode::BY_NAME; } + + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + // Parquet may preserve a selected complex wrapper without its own ID; position-delete row + // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. + options->allow_idless_complex_wrapper_projection = _format == format::FileFormat::PARQUET; + } }; } // namespace diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index da4f52484c788f..5efc7f6b6a9050 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -139,11 +139,16 @@ std::shared_ptr int32_array(int32_t value) { return result.ok() ? *result : nullptr; } -void write_mixed_id_position_delete_parquet(const std::string& path) { +void write_mixed_id_position_delete_parquet(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { auto id = [](int32_t value) { return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); }; - auto legacy_a = arrow::field("legacy_a", arrow::int32(), false)->WithMetadata(id(1)); + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false); + if (include_child_id) { + legacy_a = legacy_a->WithMetadata(id(1)); + } auto idless_b = arrow::field("b", arrow::int32(), false); auto row_array = arrow::StructArray::Make({int32_array(42), int32_array(99)}, {legacy_a, idless_b}); @@ -158,11 +163,14 @@ void write_mixed_id_position_delete_parquet(const std::string& path) { auto pos_array = pos_builder.Finish(); ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + auto row_field = arrow::field("row", arrow::struct_({legacy_a, idless_b}), false); + if (row_id.has_value()) { + row_field = row_field->WithMetadata(id(*row_id)); + } auto schema = arrow::schema({ arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), - arrow::field("row", arrow::struct_({legacy_a, idless_b}), false) - ->WithMetadata(id(2147483544)), + row_field, }); auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); auto output = arrow::io::FileOutputStream::Open(path); @@ -173,13 +181,53 @@ void write_mixed_id_position_delete_parquet(const std::string& path) { 1, properties.build())); } -void write_mixed_id_position_delete_orc(const std::string& path) { +void write_nested_wrapper_position_delete_parquet(const std::string& path) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false)->WithMetadata(id(1)); + auto s_array = arrow::StructArray::Make({int32_array(42)}, {legacy_a}); + ASSERT_TRUE(s_array.ok()) << s_array.status(); + auto idless_s = arrow::field("s", arrow::struct_({legacy_a}), false); + auto row_array = arrow::StructArray::Make({*s_array}, {idless_s}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + arrow::field("row", arrow::struct_({idless_s}), false)->WithMetadata(id(2147483544)), + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_mixed_id_position_delete_orc(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( "struct>")); type->getSubtype(0)->setAttribute("iceberg.id", "2147483546"); type->getSubtype(1)->setAttribute("iceberg.id", "2147483545"); - type->getSubtype(2)->setAttribute("iceberg.id", "2147483544"); - type->getSubtype(2)->getSubtype(0)->setAttribute("iceberg.id", "1"); + if (row_id.has_value()) { + type->getSubtype(2)->setAttribute("iceberg.id", std::to_string(*row_id)); + } + if (include_child_id) { + type->getSubtype(2)->getSubtype(0)->setAttribute("iceberg.id", "1"); + } MemoryOutputStream memory_stream(1024 * 1024); ::orc::WriterOptions options; @@ -216,6 +264,101 @@ format::ColumnDefinition table_column(int32_t id, std::string name, DataTypePtr return column; } +schema::external::TFieldPtr external_scalar_field(const std::string& name, int32_t id, + TPrimitiveType::type type) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + TColumnType column_type; + column_type.__set_type(type); + field->__set_type(column_type); + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; +} + +schema::external::TStructField position_delete_table_schema() { + schema::external::TStructField row_children; + row_children.__set_fields({external_scalar_field("a", 1, TPrimitiveType::INT), + external_scalar_field("b", 2, TPrimitiveType::INT)}); + auto row = std::make_shared(); + row->__set_name("row"); + row->__set_id(2147483544); + TColumnType row_type; + row_type.__set_type(TPrimitiveType::STRUCT); + row->__set_type(row_type); + row->nestedField.__set_struct_field(std::move(row_children)); + row->__isset.nestedField = true; + schema::external::TFieldPtr row_ptr; + row_ptr.__set_field_ptr(std::move(row)); + + schema::external::TStructField root; + root.__set_fields({external_scalar_field("file_path", 2147483546, TPrimitiveType::STRING), + external_scalar_field("pos", 2147483545, TPrimitiveType::BIGINT), + std::move(row_ptr)}); + return root; +} + +void run_v1_idless_row_position_delete_test(TFileFormatType::type file_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_v1_position_delete_idless_row_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == TFileFormatType::FORMAT_PARQUET) { + write_mixed_id_position_delete_parquet(path, std::nullopt, false); + } else { + write_mixed_id_position_delete_orc(path, std::nullopt, false); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("UTC"); + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(file_format); + scan_params.__set_current_schema_id(-1); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(position_delete_table_schema()); + scan_params.__set_history_schema_info({std::move(current_schema)}); + auto io_ctx = std::make_shared(); + io::FileReaderStats file_reader_stats; + io_ctx->file_reader_stats = &file_reader_stats; + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(file_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_size(static_cast(std::filesystem::file_size(path))); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, &scan_params, io_ctx, + nullptr); + ReaderInitContext context; + ASSERT_TRUE(reader.init_reader(&context).ok()); + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader.get_next_block(&block, &read_rows, &eof).ok()); + ASSERT_EQ(1, read_rows); + ASSERT_EQ(1, block.rows()); + // Top-level delete fields carry IDs, so an ID-less physical row must not bind by name. + EXPECT_TRUE(is_null_at(block, "row", 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + void run_mixed_id_position_delete_test(format::FileFormat file_format, TFileFormatType::type thrift_format, const std::string& extension) { @@ -291,6 +434,73 @@ void run_mixed_id_position_delete_test(format::FileFormat file_format, std::filesystem::remove_all(test_dir); } +void run_v2_nested_wrapper_position_delete_test() { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v2_position_delete_nested_wrapper_parquet"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / "delete.parquet").string(); + write_nested_wrapper_position_delete_parquet(path); + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto s_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"a"})); + const auto row_type = + make_nullable(std::make_shared(DataTypes {s_type}, Strings {"s"})); + auto a = table_column(1, "a", nullable_int32); + auto s = table_column(10, "s", s_type); + s.children = {a}; + auto row = table_column(2147483544, "row", row_type); + row.children = {s}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = format::FileFormat::PARQUET; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + EXPECT_EQ("{\"s\":{\"a\":42}}", row_type->to_string(*block.get_by_position(0).column, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { TIcebergFileDesc iceberg_desc; iceberg_desc.__set_delete_files({delete_file}); @@ -678,6 +888,14 @@ TEST(IcebergPositionDeleteSysTableReaderTest, AppendsNullMetadataAndUsesDeletePa EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableReaderTest, ParquetUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_PARQUET, "parquet"); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, OrcUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_ORC, "orc"); +} + TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) { io::FileReaderStats file_reader_stats; auto scanner_io_ctx = std::make_shared(); @@ -788,4 +1006,8 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { + run_v2_nested_wrapper_position_delete_test(); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index f45a8b40d1aa4f..0d5a6217f766f3 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -138,6 +138,46 @@ void write_iceberg_three_int_orc_file( output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } +void write_iceberg_orc_containers_with_idless_struct_children(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>,attrs:map>>")); + type->getSubtype(0)->setAttribute("iceberg.id", "10"); + type->getSubtype(1)->setAttribute("iceberg.id", "20"); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + auto& items = dynamic_cast<::orc::ListVectorBatch&>(*root.fields[0]); + auto& item = dynamic_cast<::orc::StructVectorBatch&>(*items.elements); + auto& item_a = dynamic_cast<::orc::LongVectorBatch&>(*item.fields[0]); + items.offsets[0] = 0; + items.offsets[1] = 1; + item_a.data[0] = 42; + items.numElements = item.numElements = item_a.numElements = 1; + + auto& attrs = dynamic_cast<::orc::MapVectorBatch&>(*root.fields[1]); + auto& attr_key = dynamic_cast<::orc::LongVectorBatch&>(*attrs.keys); + auto& attr_value = dynamic_cast<::orc::StructVectorBatch&>(*attrs.elements); + auto& attr_a = dynamic_cast<::orc::LongVectorBatch&>(*attr_value.fields[0]); + attrs.offsets[0] = 0; + attrs.offsets[1] = 1; + attr_key.data[0] = 1; + attr_a.data[0] = 43; + attrs.numElements = attr_key.numElements = attr_value.numElements = attr_a.numElements = 1; + + root.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + std::shared_ptr build_iceberg_int32_array(const std::vector& values) { arrow::Int32Builder builder; for (const auto value : values) { @@ -341,6 +381,78 @@ Status create_single_struct_tuple_descriptor(ObjectPool* object_pool, return Status::OK(); } +Status create_array_map_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(2); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + auto scalar_int_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar; + scalar.__set_type(TPrimitiveType::INT); + node.__set_scalar_type(scalar); + return node; + }; + auto struct_a_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::STRUCT); + TStructField child; + child.__set_name("a"); + child.__set_contains_null(true); + node.__set_struct_fields({child}); + return node; + }; + + TTypeNode array_node; + array_node.__set_type(TTypeNodeType::ARRAY); + array_node.__set_contains_nulls({true}); + TTypeDesc items_type; + items_type.__set_types({array_node, struct_a_node(), scalar_int_node()}); + + TTypeNode map_node; + map_node.__set_type(TTypeNodeType::MAP); + map_node.__set_contains_nulls({false, true}); + TTypeDesc attrs_type; + attrs_type.__set_types({map_node, scalar_int_node(), struct_a_node(), scalar_int_node()}); + + std::vector slots; + auto add_slot = [&slots](int id, const std::string& name, int field_id, const TTypeDesc& type) { + TSlotDescriptor slot; + slot.__set_id(id); + slot.__set_parent(0); + slot.__set_col_unique_id(field_id); + slot.__set_slotType(type); + slot.__set_columnPos(id); + slot.__set_byteOffset(0); + slot.__set_nullIndicatorByte(0); + slot.__set_nullIndicatorBit(-1); + slot.__set_colName(name); + slot.__set_slotIdx(id); + slot.__set_isMaterialized(true); + slots.emplace_back(std::move(slot)); + }; + add_slot(0, "items", 10, items_type); + add_slot(1, "attrs", 20, attrs_type); + thrift_table.__set_slotDescriptors(std::move(slots)); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(32); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1632,6 +1744,124 @@ TEST_F(IcebergReaderTest, v1_parquet_keeps_file_id_mode_inside_nested_struct) { std::filesystem::remove_all(test_dir); } +TEST_F(IcebergReaderTest, v1_orc_keeps_file_id_mode_inside_array_and_map) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_orc_whole_file_id_mode_containers_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + write_iceberg_orc_containers_with_idless_struct_children(data_file); + + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto make_struct_field = [&struct_type](const std::string& name, int32_t id, int32_t child_id, + const std::string& default_value) { + schema::external::TStructField children; + children.__set_fields({make_external_int_field("a", child_id, default_value)}); + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(struct_type); + field->nestedField.__set_struct_field(std::move(children)); + field->__isset.nestedField = true; + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; + }; + + TColumnType array_type; + array_type.__set_type(TPrimitiveType::ARRAY); + auto items = std::make_shared(); + items->__set_name("items"); + items->__set_id(10); + items->__set_type(array_type); + schema::external::TArrayField array_field; + array_field.__set_item_field(make_struct_field("element", 11, 12, "7")); + items->nestedField.__set_array_field(std::move(array_field)); + items->__isset.nestedField = true; + schema::external::TFieldPtr items_ptr; + items_ptr.__set_field_ptr(std::move(items)); + + TColumnType map_type; + map_type.__set_type(TPrimitiveType::MAP); + auto attrs = std::make_shared(); + attrs->__set_name("attrs"); + attrs->__set_id(20); + attrs->__set_type(map_type); + schema::external::TMapField map_field; + map_field.__set_key_field(make_external_int_field("key", 21, std::nullopt)); + map_field.__set_value_field(make_struct_field("value", 22, 23, "8")); + attrs->nestedField.__set_map_field(std::move(map_field)); + attrs->__isset.nestedField = true; + schema::external::TFieldPtr attrs_ptr; + attrs_ptr.__set_field_ptr(std::move(attrs)); + + schema::external::TStructField root_field; + root_field.__set_fields({std::move(items_ptr), std::move(attrs_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_array_map_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(2); + column_descriptors[0].name = "items"; + column_descriptors[1].name = "attrs"; + std::unordered_map block_positions = {{"items", 0}, {"attrs", 1}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + Block block; + for (const auto* slot : tuple_descriptor->slots()) { + const auto& type = slot->get_data_type_ptr(); + block.insert({type->create_column(), type, slot->col_name()}); + } + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(tuple_descriptor->slots()[0]->get_data_type_ptr()->to_string( + *block.get_by_position(0).column, 0), + "[{\"a\":7}]"); + EXPECT_EQ(tuple_descriptor->slots()[1]->get_data_type_ptr()->to_string( + *block.get_by_position(1).column, 0), + "{1:{\"a\":8}}"); + + std::filesystem::remove_all(test_dir); +} + TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 29de378ee247a9..e28f6bfa9fae1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -525,28 +525,21 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); - if (selectedSnapshot == null) { - Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); - Schema scanSchema = icebergTable.schema(); - if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { - long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) - .getSnapshotCacheValue().getSnapshot().getSchemaId(); - scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); - } - // The statement snapshot produced the target columns; cache invalidation must not - // let initial-default metadata advance to a different schema during the same scan. - return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); - } - TableScan tableScan = createTableScan(); - Snapshot snapshot = tableScan.snapshot(); - // Explicit time travel and ref scans expose the selected snapshot schema rather than the - // current table schema, so resolve its schema id instead of using TableScan.schema(). - Schema scanSchema = snapshot == null - ? tableScan.schema() - : tableScan.table().schemas().get(snapshot.schemaId()); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = null; + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } else { + scanSchema = selectedSnapshot == null + ? icebergTable.schema() + : icebergTable.schemas().get(selectedSnapshot.getSchemaId()); + } + // A branch can expose a schema newer than its data snapshot. The statement-pinned schema + // produced the target columns, so default markers must not be recomputed from that snapshot. return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index b2529bf9673857..6c47b4f76b2f4b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -347,15 +347,78 @@ public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() t Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + IcebergTableQueryInfo selectedSnapshot = Mockito.mock(IcebergTableQueryInfo.class); + Mockito.when(selectedSnapshot.getSchemaId()).thenReturn(11); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); node.setTableScan(snapshotScan); - Mockito.doReturn(Mockito.mock(IcebergTableQueryInfo.class)).when(node).getSpecifiedSnapshot(); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(selectedSnapshot).when(node).getSpecifiedSnapshot(); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } + @Test + public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { + Schema dataSnapshotSchema = new Schema(11, List.of(Types.NestedField.optional("string_default") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build())); + Schema branchSchema = new Schema(12, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Snapshot dataSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Map.of( + dataSnapshotSchema.schemaId(), dataSnapshotSchema, + branchSchema.schemaId(), branchSchema)); + TableScan branchScan = Mockito.mock(TableScan.class); + Mockito.when(branchScan.snapshot()).thenReturn(dataSnapshot); + Mockito.when(branchScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(branchScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, "branch", branchSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, branchSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") From 1984bf3d2f4a818601b17253ffcbe35de3776785 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 13:10:31 +0800 Subject: [PATCH 10/10] [fix](iceberg) retain full schema identity for projection --- be/src/format_v2/column_data.h | 3 + be/src/format_v2/column_mapper.cpp | 7 +- be/src/format_v2/table_reader.cpp | 85 +++++++++++++++++++ .../format_v2/table/iceberg_reader_test.cpp | 85 +++++++++++++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index aae67774bef5fb..d504305fad4014 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -258,6 +258,9 @@ struct ColumnDefinition { // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For // example, MAP children are key/value and ARRAY children contain only the element field. std::vector children {}; + // Full table-schema identity subtree before access-path pruning. ID-less physical complex + // wrappers must be discovered from this view without adding unrequested children to output. + std::vector identity_children {}; // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 9fbe05e7cfe4b4..4df9c051c21abd 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -101,7 +101,9 @@ Status build_initial_default_literal(const ColumnDefinition& column, VExprContex } bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { - for (const auto& table_child : table.children) { + const auto& table_children = + table.identity_children.empty() ? table.children : table.identity_children; + for (const auto& table_child : table_children) { if (!table_child.has_identifier_field_id()) { continue; } @@ -406,6 +408,9 @@ std::string ColumnDefinition::debug_string() const { << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", identity_children=" + << join_debug_strings(identity_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) << ", is_partition_key=" << is_partition_key << "}"; return out.str(); diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 182ff79ffaccd8..06534ffa80a7c6 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -143,6 +143,84 @@ const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& return field_ptr.field_ptr.get(); } +ColumnDefinition build_schema_identity_from_external_field(const schema::external::TField& field) { + ColumnDefinition identity; + if (field.__isset.id) { + identity.identifier = Field::create_field(field.id); + } + identity.name = field.__isset.name ? field.name : ""; + identity.name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + identity.has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; + if (!field.__isset.nestedField) { + return identity; + } + if (field.nestedField.__isset.struct_field && field.nestedField.struct_field.__isset.fields) { + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + if (const auto* child = get_field_ptr(child_ptr); child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + } + } + } else if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field) { + if (const auto* child = get_field_ptr(field.nestedField.array_field.item_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "element"; + } + } else if (field.nestedField.__isset.map_field) { + if (field.nestedField.map_field.__isset.key_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.key_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "key"; + } + } + if (field.nestedField.map_field.__isset.value_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.value_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "value"; + } + } + } + return identity; +} + +const ColumnDefinition* find_identity_child(const ColumnDefinition& projected_child, + const ColumnDefinition& identity_parent) { + const auto child_it = std::ranges::find_if( + identity_parent.children, [&](const ColumnDefinition& identity_child) { + if (projected_child.has_identifier_field_id() && + identity_child.has_identifier_field_id()) { + return projected_child.get_identifier_field_id() == + identity_child.get_identifier_field_id(); + } + if (to_lower(projected_child.name) == to_lower(identity_child.name)) { + return true; + } + return std::ranges::any_of( + identity_child.name_mapping, [&](const std::string& alias) { + return to_lower(projected_child.name) == to_lower(alias); + }); + }); + return child_it == identity_parent.children.end() ? nullptr : &*child_it; +} + +void attach_full_schema_identity(ColumnDefinition* projected, const ColumnDefinition& identity) { + DORIS_CHECK(projected != nullptr); + // Access-path children control materialization, but wrapper discovery needs sibling IDs that + // were pruned from that projection. Keep the complete identity tree on a separate channel. + projected->identity_children = identity.children; + for (auto& projected_child : projected->children) { + if (const auto* identity_child = find_identity_child(projected_child, identity); + identity_child != nullptr) { + attach_full_schema_identity(&projected_child, *identity_child); + } + } +} + bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { if (field.__isset.name && to_lower(field.name) == to_lower(name)) { return true; @@ -538,6 +616,13 @@ Status TableReader::init(TableReadOptions&& options) { _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; _projected_columns = std::move(options.projected_columns); + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity(&projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } _system_properties = create_system_properties(_scan_params); _mapper_options.mode = TableColumnMappingMode::BY_NAME; _conjuncts = std::move(options.conjuncts); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index c3430125ac5826..22642f20b68093 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -522,6 +522,27 @@ void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, i builder.build())); } +void write_nullable_idless_struct_with_sibling_id_parquet_file(const std::string& file_path) { + const auto sibling_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto sibling_field = arrow::field("b", arrow::int32(), false)->WithMetadata(sibling_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 42})}, {sibling_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = arrow::field("s", arrow::struct_({sibling_field}), true); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -2578,6 +2599,70 @@ TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_unprojected_sibling_id_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.initial_default_value = "7"; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + auto schema_a = + external_schema_field("a", 1, {}, "7", external_primitive_type(TPrimitiveType::INT)); + auto schema_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + schema::external::TStructField struct_children; + struct_children.__set_fields({schema_a, schema_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test";