diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..f0346abaaf300e 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,6 +87,13 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + // The presence bit is part of the mapping contract: an explicit empty mapping must remain + // authoritative after access-path pruning instead of enabling current-name fallback. + column->has_name_mapping = schema_column->has_name_mapping; + // Initial defaults describe the logical value of fields absent from older files. Nested + // access-path pruning must retain them just like it retains rename metadata. + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..728595bdad13a7 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -34,6 +34,7 @@ #include "exprs/vexpr.h" #include "exprs/vslot_ref.h" #include "exprs/vtopn_pred.h" +#include "util/url_coding.h" // IWYU pragma: no_include #include // IWYU pragma: keep @@ -105,6 +106,35 @@ #include "util/unaligned.h" namespace doris { +static Status build_orc_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + class RuntimeState; namespace io { struct IOContext; @@ -2145,15 +2175,28 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, for (int missing_field : missing_fields) { ColumnPtr& doris_field = doris_struct.get_column_ptr(missing_field); - if (!doris_field->is_nullable()) { + const auto& table_column_name = doris_struct_type->get_name_by_position(missing_field); + const auto& doris_type = doris_struct_type->get_element(missing_field); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_orc_initial_default_column( + root_node->children_initial_default_value(table_column_name), doris_type, + num_values, &initial_default)); + if (initial_default.get() != nullptr) { + // ORC projection may synthesize a missing nested field, but its Iceberg initial + // default remains the logical value for every row in the older file. + auto mutable_field = IColumn::mutate(std::move(doris_field)); + mutable_field->insert_range_from(*initial_default, 0, num_values); + doris_field = std::move(mutable_field); + } else if (!doris_field->is_nullable()) { return Status::InternalError( "Child field of '{}' is not nullable, but is missing in orc file", col_name); + } else { + auto mutable_field = IColumn::mutate(std::move(doris_field)); + reinterpret_cast(mutable_field.get()) + ->insert_many_defaults(num_values); + doris_field = std::move(mutable_field); } - auto mutable_field = IColumn::mutate(std::move(doris_field)); - reinterpret_cast(mutable_field.get()) - ->insert_many_defaults(num_values); - doris_field = std::move(mutable_field); } for (auto read_field : read_fields) { diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 2726044c193b4c..53e44babbd8297 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -40,8 +40,38 @@ #include "format/parquet/vparquet_column_chunk_reader.h" #include "io/fs/tracing_file_reader.h" #include "runtime/runtime_profile.h" +#include "util/url_coding.h" namespace doris { +static Status build_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, const std::vector& rep_levels, const std::vector& def_levels) { @@ -990,11 +1020,21 @@ Status StructColumnReader::read_column_data( for (auto idx : missing_column_idxs) { auto& doris_field = doris_struct.get_column_ptr(idx); auto& doris_type = doris_struct_type->get_element(idx); - DCHECK(doris_type->is_nullable()); - doris_field = IColumn::mutate(std::move(doris_field)); - auto mutable_column = doris_field->assert_mutable(); - auto* nullable_column = static_cast(mutable_column.get()); - nullable_column->insert_many_defaults(missing_column_sz); + auto mutable_field = IColumn::mutate(std::move(doris_field)); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_initial_default_column( + root_node->children_initial_default_value(doris_struct_type->get_element_name(idx)), + doris_type, missing_column_sz, &initial_default)); + if (initial_default.get() != nullptr) { + // Iceberg initial defaults are logical row values, including for nested fields absent + // from the physical file; append them instead of the type's generic NULL/default. + mutable_field->insert_range_from(*initial_default, 0, missing_column_sz); + } else { + DCHECK(doris_type->is_nullable()); + static_cast(mutable_field.get()) + ->insert_many_defaults(missing_column_sz); + } + doris_field = std::move(mutable_field); } if (null_map_ptr != nullptr) { diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp index c6e97465e2dd44..0ade539cf241c0 100644 --- a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -99,15 +99,7 @@ const ColumnInt64* get_int64_column(const Block& block, const std::string& name) return check_and_get_column(block.get_by_position(pos).column.get()); } -const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { - if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { - return nullptr; - } - return field_ptr.field_ptr.get(); -} - -const schema::external::TField* find_current_schema_field(const TFileScanRangeParams* params, - const std::string& name) { +const schema::external::TSchema* find_current_schema(const TFileScanRangeParams* params) { if (params == nullptr || !params->__isset.history_schema_info || params->history_schema_info.empty()) { return nullptr; @@ -121,16 +113,7 @@ const schema::external::TField* find_current_schema_field(const TFileScanRangePa } } } - if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { - return nullptr; - } - for (const auto& field_ptr : schema->root_field.fields) { - const auto* field = get_field_ptr(field_ptr); - if (field != nullptr && field->__isset.name && field->name == name) { - return field; - } - } - return nullptr; + return schema; } template @@ -248,13 +231,23 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { &_state->timezone_obj(), _io_ctx, _state, _meta_cache); const FieldDescriptor* schema = nullptr; - int row_index = -1; + std::shared_ptr mapped_file_schema; if (row_requested) { RETURN_IF_ERROR(parquet_reader->get_file_metadata_schema(&schema)); DORIS_CHECK(schema != nullptr); - row_index = schema->get_column_index(kRowColumn); + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + // Position-delete mapping mode is file-wide: file_path/pos IDs must prevent an + // ID-less physical row from being rebound by name in either reader generation. + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: + by_parquet_field_id_with_name_mapping( + table_schema->root_field, *schema, mapped_file_schema)); } - const bool read_row = row_requested && row_index >= 0; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -270,18 +263,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { pctx.range = &_range; pctx.filter_groups = false; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - const auto* file_row_field = schema->get_column(static_cast(row_index)); - std::shared_ptr row_node; - RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: - by_parquet_field_id_with_name_mapping( - *table_row_field, *file_row_field, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, file_row_field->name, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); pctx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(parquet_reader.get())->init_reader(&pctx)); @@ -294,19 +279,25 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { OrcReader::create_unique(_profile, _state, *_range_params, _range, _batch_size, _state->timezone(), _io_ctx, _meta_cache); - const orc::Type* row_type = nullptr; + std::shared_ptr mapped_file_schema; if (row_requested) { const orc::Type* root_type = nullptr; RETURN_IF_ERROR(orc_reader->get_file_type(&root_type)); DORIS_CHECK(root_type != nullptr); - for (uint64_t i = 0; i < root_type->getSubtypeCount(); ++i) { - if (root_type->getFieldName(i) == kRowColumn) { - row_type = root_type->getSubtype(i); - break; - } + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); } + // Resolve row against the complete delete-file type so top-level IDs keep ORC in ID + // projection throughout the nested row subtree. + RETURN_IF_ERROR( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + table_schema->root_field, root_type, kIcebergOrcAttribute, + mapped_file_schema)); } - const bool read_row = row_requested && row_type != nullptr; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -321,17 +312,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { octx.params = _range_params; octx.range = &_range; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - std::shared_ptr row_node; - RETURN_IF_ERROR( - TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( - *table_row_field, row_type, kIcebergOrcAttribute, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, kRowColumn, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); octx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(orc_reader.get())->init_reader(&octx)); diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index aee211aee019cd..918ace70972ae9 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -157,6 +157,30 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { protected: // ---- Hook implementations ---- + Status on_fill_missing_columns(Block* block, size_t rows, + const std::vector& cols) override { + std::vector generic_columns; + for (const auto& name : cols) { + const auto* field = _find_current_schema_field(name); + if (field == nullptr || !field->__isset.initial_default_value) { + generic_columns.push_back(name); + continue; + } + if (!this->col_name_to_block_idx_ref()->contains(name)) { + return Status::InternalError("Missing column: {} not found in block {}", name, + block->dump_structure()); + } + const auto position = this->col_name_to_block_idx_ref()->at(name); + ColumnPtr value; + RETURN_IF_ERROR(_build_owned_initial_default_column( + *field, block->get_by_position(position).type, rows, &value)); + // Iceberg metadata is authoritative for pre-add rows; a generic FE expression may be + // the Base64 carrier rather than the logical BINARY/FIXED/UUID value. + block->get_by_position(position).column = value->convert_to_full_column_if_const(); + } + return BaseReader::on_fill_missing_columns(block, rows, generic_columns); + } + // Called before reading a block: expand block for equality delete columns + detect row_id Status on_before_read_block(Block* block) override { RETURN_IF_ERROR(_expand_block_if_need(block)); @@ -270,6 +294,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { Status _expand_block_if_need(Block* block); Status _shrink_block_if_need(Block* block); + const schema::external::TStructField* _current_schema_root() const; + const schema::external::TField* _find_current_schema_field(const std::string& name) const; + Status _build_owned_initial_default_column(const schema::external::TField& field, + const DataTypePtr& type, size_t rows, + ColumnPtr* column) const; Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type); Status _materialize_missing_equality_delete_column(Block* block, const std::string& name, @@ -664,6 +693,76 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { return Status::OK(); } +template +const schema::external::TStructField* IcebergReaderMixin::_current_schema_root() const { + const auto& scan_params = this->get_scan_params(); + if (!scan_params.__isset.history_schema_info || scan_params.history_schema_info.empty()) { + return nullptr; + } + const schema::external::TSchema* current_schema = &scan_params.history_schema_info.front(); + if (scan_params.__isset.current_schema_id) { + const auto schema_it = std::ranges::find_if( + scan_params.history_schema_info, [&](const schema::external::TSchema& schema) { + return schema.__isset.schema_id && + schema.schema_id == scan_params.current_schema_id; + }); + if (schema_it == scan_params.history_schema_info.end()) { + return nullptr; + } + current_schema = &*schema_it; + } + return current_schema->__isset.root_field ? ¤t_schema->root_field : nullptr; +} + +template +const schema::external::TField* IcebergReaderMixin::_find_current_schema_field( + const std::string& name) const { + const auto* root = _current_schema_root(); + if (root == nullptr) { + return nullptr; + } + const auto field = std::ranges::find_if(root->fields, [&](const auto& field_ptr) { + return field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.name && field_ptr.field_ptr->name == name; + }); + return field == root->fields.end() ? nullptr : field->field_ptr.get(); +} + +template +Status IcebergReaderMixin::_build_owned_initial_default_column( + const schema::external::TField& field, const DataTypePtr& type, size_t rows, + ColumnPtr* column) const { + DORIS_CHECK(type != nullptr); + DORIS_CHECK(column != nullptr); + DORIS_CHECK(field.__isset.initial_default_value); + const auto nested_type = remove_nullable(type); + const bool is_base64 = + (field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64) || + (field.__isset.type && thrift_to_type(field.type.type) == TYPE_VARBINARY); + Field value; + if (is_base64) { + std::string decoded; + if (!base64_decode(field.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + field.name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded); + } + // Variable-width Fields borrow decoded. Materialize before it leaves scope so every V1 + // missing-column and equality-delete boundary keeps UUID/FIXED payloads alive. + *column = type->create_column_const(rows, value); + return Status::OK(); + } + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(field.initial_default_value, value)); + *column = type->create_column_const(rows, value); + return Status::OK(); +} + template Status IcebergReaderMixin::_register_missing_equality_delete_column( int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) { @@ -703,33 +802,15 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( "Missing Iceberg schema metadata for equality-delete field id {}", field_id); } - Field value; + ColumnPtr value_column; if (table_field->__isset.initial_default_value) { - const auto nested_type = remove_nullable(delete_key_type); - const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && - table_field->initial_default_value_is_base64) || - (table_field->__isset.type && - thrift_to_type(table_field->type.type) == TYPE_VARBINARY); - if (default_is_base64) { - std::string decoded_default; - if (!base64_decode(table_field->initial_default_value, &decoded_default)) { - return Status::InvalidArgument( - "Invalid Base64 Iceberg initial default for field {}", table_field->name); - } - if (nested_type->get_primitive_type() == TYPE_VARBINARY) { - value = Field::create_field(StringView(decoded_default)); - } else { - DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); - value = Field::create_field(decoded_default); - } - } else { - RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( - table_field->initial_default_value, value)); - } + RETURN_IF_ERROR(_build_owned_initial_default_column(*table_field, delete_key_type, 1, + &value_column)); + } else { + value_column = delete_key_type->create_column_const(1, Field()); } - const bool inserted = _missing_equality_delete_values - .emplace(name, delete_key_type->create_column_const(1, value)) - .second; + const bool inserted = + _missing_equality_delete_values.emplace(name, std::move(value_column)).second; DORIS_CHECK(inserted); this->register_synthesized_column_handler( name, [this, name](Block* block, size_t rows) -> Status { diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 3d4cb6b5a6c99c..b8a140dfacc5e8 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -55,6 +55,30 @@ std::map build_lowercase_orc_field_name_idx_map(const orc:: return file_column_name_idx_map; } +bool orc_subtree_has_field_id(const orc::Type* type, const std::string& attribute) { + if (type->hasAttributeKey(attribute)) { + return true; + } + for (uint64_t idx = 0; idx < type->getSubtypeCount(); ++idx) { + if (orc_subtree_has_field_id(type->getSubtype(idx), attribute)) { + return true; + } + } + return false; +} + +std::optional initial_default_value( + const schema::external::TField& field) { + if (!field.__isset.initial_default_value) { + return std::nullopt; + } + return TableSchemaChangeHelper::InitialDefaultValue { + .value = field.initial_default_value, + .is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, + }; +} + bool find_file_field_idx_by_name_mapping( const schema::external::TField& table_field, const std::map& file_column_name_idx_map, size_t* file_column_idx) { @@ -73,11 +97,98 @@ bool find_file_field_idx_by_name_mapping( return true; } } + if (table_field.__isset.name_mapping_is_authoritative && + table_field.name_mapping_is_authoritative) { + // Only a compatible FE can make the mapping authoritative; older FE plans must retain + // their legacy current-name fallback throughout a rolling BE upgrade. + return false; + } } return table_field.__isset.name && try_match(table_field.name); } +bool table_subtree_contains_field_id(const schema::external::TField& field, int32_t field_id) { + if (field.id == field_id) { + return true; + } + if (!field.__isset.nestedField) { + return false; + } + switch (field.type.type) { + case TPrimitiveType::STRUCT: + if (field.nestedField.__isset.struct_field) { + for (const auto& child : field.nestedField.struct_field.fields) { + if (child.field_ptr != nullptr && + table_subtree_contains_field_id(*child.field_ptr, field_id)) { + return true; + } + } + } + break; + case TPrimitiveType::ARRAY: + if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field && + field.nestedField.array_field.item_field.field_ptr != nullptr) { + return table_subtree_contains_field_id( + *field.nestedField.array_field.item_field.field_ptr, field_id); + } + break; + case TPrimitiveType::MAP: + if (field.nestedField.__isset.map_field) { + const auto& map = field.nestedField.map_field; + if (map.__isset.key_field && map.key_field.field_ptr != nullptr && + table_subtree_contains_field_id(*map.key_field.field_ptr, field_id)) { + return true; + } + if (map.__isset.value_field && map.value_field.field_ptr != nullptr) { + return table_subtree_contains_field_id(*map.value_field.field_ptr, field_id); + } + } + break; + default: + break; + } + return false; +} + +bool has_shared_descendant_field_id(const schema::external::TField& table_field, + const FieldSchema& file_field) { + for (const auto& file_child : file_field.children) { + if ((file_child.field_id != -1 && + table_subtree_contains_field_id(table_field, file_child.field_id)) || + has_shared_descendant_field_id(table_field, file_child)) { + return true; + } + } + return false; +} + +std::optional find_unique_idless_wrapper( + const schema::external::TField& table_field, + const std::vector& parquet_fields_schema) { + std::optional match; + for (size_t idx = 0; idx < parquet_fields_schema.size(); ++idx) { + const auto& candidate = parquet_fields_schema[idx]; + if (candidate.field_id != -1 || candidate.children.empty() || + !has_shared_descendant_field_id(table_field, candidate)) { + continue; + } + if (match.has_value()) { + return std::nullopt; + } + match = idx; + } + return match; +} + +bool parquet_subtree_has_field_id(const FieldSchema& field) { + if (field.field_id != -1) { + return true; + } + return std::ranges::any_of(field.children, parquet_subtree_has_field_id); +} + } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -534,7 +645,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( exist_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } else { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); } } node = struct_node; @@ -556,17 +668,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& parquet_fields_schema = parquet_field_desc.get_fields_schema(); std::map file_column_id_idx_map; - bool all_have_field_id = true; + // Iceberg's hasIds predicate is file-wide. Keep this decision through recursion so an ID-less + // nested struct cannot switch an ID-bearing file back to alias/name matching. + const bool use_field_id = + std::ranges::any_of(parquet_fields_schema, parquet_subtree_has_field_id); for (size_t idx = 0; idx < parquet_fields_schema.size(); idx++) { - if (parquet_fields_schema[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_fields_schema[idx].field_id != -1) { + file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_fields_schema); } @@ -574,11 +687,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (auto wrapper = find_unique_idless_wrapper(*table_field.field_ptr, + parquet_fields_schema); + wrapper.has_value()) { + // Parquet may retain a selected struct without its own ID; a unique descendant-ID + // match is authoritative even when Iceberg name mapping intentionally has no alias. + file_column_idx = *wrapper; + matched = true; } } else { matched = find_file_field_idx_by_name_mapping( @@ -586,13 +706,15 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } std::shared_ptr field_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( - *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node)); + *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node, + use_field_id)); struct_node->add_children(table_column_name, parquet_fields_schema[file_column_idx].name, field_node); } @@ -604,6 +726,13 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node) { + return by_parquet_field_id_with_name_mapping(table_schema, parquet_field, node, + parquet_subtree_has_field_id(parquet_field)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (parquet_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] { @@ -623,10 +752,10 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, parquet_field.children[0], - key_node)); + key_node, use_field_id)); RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, - parquet_field.children[1], value_node)); + parquet_field.children[1], value_node, use_field_id)); node = std::make_shared(key_node, value_node); break; @@ -645,7 +774,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, - parquet_field.children[0], element_node)); + parquet_field.children[0], element_node, use_field_id)); node = std::make_shared(element_node); break; @@ -660,17 +789,14 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - bool all_have_field_id = true; for (size_t idx = 0; idx < parquet_field.children.size(); idx++) { - if (parquet_field.children[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_field.children[idx].field_id != -1) { + file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_field.children); } @@ -678,11 +804,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (auto wrapper = find_unique_idless_wrapper(*table_field.field_ptr, + parquet_field.children); + wrapper.has_value()) { + // Apply the same Parquet wrapper invariant at every struct recursion depth. + file_column_idx = *wrapper; + matched = true; } } else { matched = find_file_field_idx_by_name_mapping( @@ -690,14 +822,15 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } const auto& file_field = parquet_field.children.at(file_column_idx); std::shared_ptr field_node = nullptr; - RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping(*table_field.field_ptr, - file_field, field_node)); + RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( + *table_field.field_ptr, file_field, field_node, use_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } node = struct_node; @@ -822,22 +955,28 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TStructField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping( + table_schema, orc_root, field_id_attribute_key, node, + orc_subtree_has_field_id(orc_root, field_id_attribute_key)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id) { auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - bool all_have_field_id = true; for (size_t idx = 0; idx < orc_root->getSubtypeCount(); idx++) { - if (!orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { - all_have_field_id = false; - break; + if (orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { + auto field_id = + std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); + file_column_id_idx_map.emplace(field_id, idx); } - auto field_id = - std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); - file_column_id_idx_map.emplace(field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_orc_field_name_idx_map(orc_root); } @@ -845,7 +984,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const auto& table_column_name = table_field.field_ptr->name; size_t file_field_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_field_idx = id_it->second; @@ -857,14 +996,16 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); continue; } const auto& file_field = orc_root->getSubtype(file_field_idx); std::shared_ptr field_node = nullptr; RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(*table_field.field_ptr, file_field, - field_id_attribute_key, field_node)); + field_id_attribute_key, field_node, + use_field_id)); struct_node->add_children(table_column_name, orc_root->getFieldName(file_field_idx), field_node); } @@ -876,6 +1017,15 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping( + table_schema, orc_root, field_id_attribute_key, node, + orc_subtree_has_field_id(orc_root, field_id_attribute_key)); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (orc_root->getKind() != orc::TypeKind::MAP) [[unlikely]] { @@ -895,10 +1045,10 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, key_node)); + field_id_attribute_key, key_node, use_field_id)); RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, orc_root->getSubtype(1), - field_id_attribute_key, value_node)); + field_id_attribute_key, value_node, use_field_id)); node = std::make_shared(key_node, value_node); break; @@ -917,7 +1067,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, element_node)); + field_id_attribute_key, element_node, use_field_id)); node = std::make_shared(element_node); break; @@ -928,8 +1078,11 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } MOCK_REMOVE(DCHECK(table_schema.__isset.nestedField)); MOCK_REMOVE(DCHECK(table_schema.nestedField.__isset.struct_field)); + // Iceberg chooses ID projection once for the complete ORC file; container recursion must + // not re-enable name matching merely because a nested struct has no local IDs. RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(table_schema.nestedField.struct_field, - orc_root, field_id_attribute_key, node)); + orc_root, field_id_attribute_key, node, + use_field_id)); break; } default: { diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 2d54b3216cac89..45cce284014701 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -19,6 +19,7 @@ #include #include +#include #include #include "common/status.h" @@ -41,6 +42,10 @@ namespace doris { class TableSchemaChangeHelper { public: + struct InitialDefaultValue { + std::string value; + bool is_base64 = false; + }; ~TableSchemaChangeHelper() = default; class Node { @@ -67,6 +72,11 @@ class TableSchemaChangeHelper { "children_column_exists should not be called on base TableInfoNode"); } + virtual std::optional children_initial_default_value( + std::string) const { + return std::nullopt; + } + virtual std::shared_ptr get_element_node() const { throw std::logic_error("get_element_node should not be called on base TableInfoNode"); } @@ -78,7 +88,9 @@ class TableSchemaChangeHelper { throw std::logic_error("get_value_node should not be called on base TableInfoNode"); } - virtual void add_not_exist_children(std::string table_column_name) { + virtual void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) { throw std::logic_error( "add_not_exist_children should not be called on base TableInfoNode"); }; @@ -131,6 +143,7 @@ class TableSchemaChangeHelper { const std::shared_ptr node; const std::string column_name; const bool exists; + const std::optional initial_default; }; // table column name -> { node, file_column_name, exists_in_file} @@ -167,14 +180,23 @@ class TableSchemaChangeHelper { return children.at(table_column_name).exists; } - void add_not_exist_children(std::string table_column_name) override { - children.emplace(table_column_name, StructChild {nullptr, "", false}); + std::optional children_initial_default_value( + std::string table_column_name) const override { + DCHECK(children.contains(table_column_name)); + return children.at(table_column_name).initial_default; + } + + void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) override { + children.emplace(table_column_name, + StructChild {nullptr, "", false, std::move(initial_default)}); } void add_children(std::string table_column_name, std::string file_column_name, std::shared_ptr children_node) override { children.emplace(table_column_name, - StructChild {children_node, file_column_name, true}); + StructChild {children_node, file_column_name, true, std::nullopt}); } const std::map& get_children() const { return children; } @@ -330,6 +352,10 @@ class TableSchemaChangeHelper { const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node); + static Status by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id); + // for iceberg orc : Use the field id in the `table schema` and the orc file to match columns. static Status by_orc_field_id(const schema::external::TStructField& table_schema, const orc::Type* orc_root, @@ -351,11 +377,21 @@ class TableSchemaChangeHelper { const std::string& field_id_attribute_key, std::shared_ptr& node); + static Status by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id); + // for iceberg orc static Status by_orc_field_id_with_name_mapping( const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node); + + static Status by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id); }; }; diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index ac510caaf07ac4..d504305fad4014 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -248,6 +248,9 @@ struct ColumnDefinition { // Historical or external names for the same logical field. Table formats such as Iceberg can // use this to resolve partition path keys after column rename. std::vector name_mapping {}; + // Distinguishes no table-level mapping from an explicit empty field mapping. The latter must + // not fall back to the current name when matching fields in legacy files. + bool has_name_mapping = false; DataTypePtr type; // Semantic nested children for this schema node. // @@ -255,6 +258,9 @@ struct ColumnDefinition { // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For // example, MAP children are key/value and ARRAY children contain only the element field. std::vector children {}; + // Full table-schema identity subtree before access-path pruning. ID-less physical complex + // wrappers must be discovered from this view without adding unrequested children to output. + std::vector identity_children {}; // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 313bbf376f860e..4df9c051c21abd 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -52,11 +52,77 @@ #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "util/url_coding.h" namespace doris::format { namespace { +Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { + DORIS_CHECK(value != nullptr); + *value = nullptr; + if (!column.initial_default_value.has_value()) { + return Status::OK(); + } + const auto nested_type = remove_nullable(column.type); + Field parsed; + if (column.initial_default_value_is_base64 || + nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(*column.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + column.name); + } + parsed = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // Variable-width Fields borrow their input. Materialize while decoded is alive so the + // resulting column owns the payload before it crosses a mapping/literal boundary. + *value = column.type->create_column_const(1, parsed); + return Status::OK(); + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); + } + *value = column.type->create_column_const(1, parsed); + return Status::OK(); +} + +Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { + DORIS_CHECK(literal != nullptr); + ColumnPtr owned_value; + RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); + DORIS_CHECK(static_cast(owned_value)); + Field value; + owned_value->get(0, value); + // VLiteral copies the borrowed Field into its own column while owned_value is still alive. + *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); + return Status::OK(); +} + +bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { + const auto& table_children = + table.identity_children.empty() ? table.children : table.identity_children; + for (const auto& table_child : table_children) { + if (!table_child.has_identifier_field_id()) { + continue; + } + const auto file_child = + std::ranges::find_if(file.children, [&](const ColumnDefinition& candidate) { + return candidate.has_identifier_field_id() && + candidate.get_identifier_field_id() == + table_child.get_identifier_field_id(); + }); + if (file_child != file.children.end() || + std::ranges::any_of(file.children, [&](const ColumnDefinition& candidate) { + return has_shared_descendant_field_id(table_child, candidate); + })) { + return true; + } + } + return false; +} + std::string mapping_mode_to_string(TableColumnMappingMode mode) { switch (mode) { case TableColumnMappingMode::BY_FIELD_ID: @@ -82,12 +148,16 @@ bool column_has_name(const ColumnDefinition& column, const std::string& name) { } bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { - if (column_has_name(rhs, lhs.name)) { - return true; - } - if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { - return true; + if (!lhs.has_name_mapping) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } } + // Explicit Iceberg name mapping is authoritative: an empty alias list represents a field that + // did not exist in the imported file, so only transported aliases may match. return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { return column_has_name(rhs, alias); }); @@ -323,7 +393,9 @@ static bool is_binary_comparison_predicate(const VExprSPtr& expr) { std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; - out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection + << "}"; return out.str(); } @@ -332,9 +404,13 @@ std::string ColumnDefinition::debug_string() const { out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) << ", name_mapping=" << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) - << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << ", has_name_mapping=" << has_name_mapping << ", local_id=" << local_id + << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", identity_children=" + << join_debug_strings(identity_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) << ", is_partition_key=" << is_partition_key << "}"; return out.str(); @@ -1762,6 +1838,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.initial_default_value.has_value()) { + VExprContextSPtr initial_default; + RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); + // Iceberg metadata is the authoritative logical value for files written before the field + // existed; the generic FE expression may still contain its Base64 transport text. + _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); @@ -2084,7 +2166,25 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( }); return field_it == file_schema.end() ? nullptr : &*field_it; } - return matcher_for_mode(_options.mode).find(table_column, file_schema); + const auto* matched = matcher_for_mode(_options.mode).find(table_column, file_schema); + if (matched != nullptr || _options.mode != TableColumnMappingMode::BY_FIELD_ID || + !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { + return matched; + } + const ColumnDefinition* wrapper = nullptr; + for (const auto& candidate : file_schema) { + if (candidate.has_identifier_field_id() || candidate.children.empty() || + !has_shared_descendant_field_id(table_column, candidate)) { + continue; + } + if (wrapper != nullptr) { + return nullptr; + } + wrapper = &candidate; + } + // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is + // selected. Descendant IDs, not aliases, identify that wrapper; ambiguity remains unmapped. + return wrapper; } Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, @@ -2122,6 +2222,13 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c const auto* file_child = find_file_child_for_mapping(table_child, file_field, _options.mode, table_child_idx, synthesized_table_children); + if (file_child == nullptr && !synthesized_table_children && + _options.mode == TableColumnMappingMode::BY_FIELD_ID && + _options.allow_idless_complex_wrapper_projection) { + // Parquet can retain an ID-less wrapper at any depth when a selected descendant + // has an ID; apply the same opt-in fallback used for root lookup recursively. + file_child = _find_file_field(table_child, file_field.children); + } if (synthesized_table_children && file_child != nullptr) { const auto file_child_id = file_child->file_local_id(); if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != @@ -2147,6 +2254,10 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + // A missing nested field still has its Iceberg initial-default value in every row + // written before the field was added; carry it into recursive materialization. + RETURN_IF_ERROR(build_initial_default_column( + table_child, &child_mapping.initial_default_column)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 4f417a680aad60..0b42ac39a5cbea 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -150,12 +150,16 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; + // One-row constant owns variable-width payloads; Field is only a borrowed + // StringView and cannot safely outlive the Base64 decode buffer used to construct it. + ColumnPtr initial_default_column; std::string debug_string() const; }; struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + bool allow_idless_complex_wrapper_projection = false; std::string debug_string() const; }; diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..f3491fd2d237a1 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -35,6 +35,7 @@ #include "core/types.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -132,24 +133,20 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) { } } -bool has_field_id(const std::vector& schema) { - for (const auto& field : schema) { - if (!field.has_identifier_field_id()) { - return false; - } - if (!has_field_id(field.children)) { - return false; - } - } - return true; -} - class PositionDeleteFileTableReader final : public format::TableReader { protected: format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + if (!_data_reader.file_schema.empty() && + schema_has_any_field_id(_data_reader.file_schema)) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; + } + + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + // Parquet may preserve a selected complex wrapper without its own ID; position-delete row + // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. + options->allow_idless_complex_wrapper_projection = _format == format::FileFormat::PARQUET; } }; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 962a414a4c23f1..40fe41783fecd5 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -91,7 +91,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; + VExprSPtr literal; if (table_field.initial_default_value_is_base64 || table_field.type->get_primitive_type() == TYPE_VARBINARY) { // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its @@ -104,19 +104,26 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit table_field.name); } if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); + const auto initial_default = + Field::create_field(StringView(decoded_default)); + // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and + // long FIXED defaults otherwise retain a pointer into freed decode storage. + literal = VLiteral::create_shared(table_field.type, initial_default); } else { DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); + literal = VLiteral::create_shared(table_field.type, + Field::create_field(decoded_default)); } } else { // An added field's initial default is its logical value in every older data file that lacks // the physical column. FE normalizes the string for the current Doris table type. + Field initial_default; RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( *table_field.initial_default_value, initial_default)); + literal = VLiteral::create_shared(table_field.type, initial_default); } - auto literal = VLiteral::create_shared(table_field.type, initial_default); + DORIS_CHECK(literal != nullptr); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index d328f574f2b22b..afd3ee53913f9d 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -27,6 +27,7 @@ #include "core/block/block.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format_v2/file_reader.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" @@ -57,12 +58,18 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + if (!_data_reader.file_schema.empty() && + schema_has_any_field_id(_data_reader.file_schema)) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } protected: + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->allow_idless_complex_wrapper_projection = _format == FileFormat::PARQUET; + } + Status materialize_virtual_columns(Block* table_block) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; @@ -76,26 +83,6 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - - bool _has_field_id(const std::vector& schema) const { - for (const auto& field : schema) { - // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the - // first-phase scan. That virtual column is not an Iceberg data field and therefore has - // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, - // otherwise old data files whose physical names predate a rename (for example, - // table column `new_new_id` stored as file column `id`) are materialized as defaults. - if (field.column_type != format::ColumnType::DATA_COLUMN) { - continue; - } - if (!field.has_identifier_field_id()) { - return false; - } - if (!_has_field_id(field.children)) { - return false; - } - } - return true; - } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; static constexpr int EQUALITY_DELETE = 2; diff --git a/be/src/format_v2/table/iceberg_schema_utils.h b/be/src/format_v2/table/iceberg_schema_utils.h new file mode 100644 index 00000000000000..ff91e8018447e8 --- /dev/null +++ b/be/src/format_v2/table/iceberg_schema_utils.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/column_data.h" + +namespace doris::format::iceberg { + +inline bool schema_has_any_field_id(const std::vector& schema) { + for (const auto& field : schema) { + // Iceberg's hasIds contract is existential for the whole file. Ignore synthesized columns + // and retain ID projection when any real field, including a nested field, carries an ID. + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (field.has_identifier_field_id() || schema_has_any_field_id(field.children)) { + return true; + } + } + return false; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 10109839e6987d..440211607d633d 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,6 +77,8 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; + column->has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5f37538c1f2639..06534ffa80a7c6 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -143,6 +143,84 @@ const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& return field_ptr.field_ptr.get(); } +ColumnDefinition build_schema_identity_from_external_field(const schema::external::TField& field) { + ColumnDefinition identity; + if (field.__isset.id) { + identity.identifier = Field::create_field(field.id); + } + identity.name = field.__isset.name ? field.name : ""; + identity.name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + identity.has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; + if (!field.__isset.nestedField) { + return identity; + } + if (field.nestedField.__isset.struct_field && field.nestedField.struct_field.__isset.fields) { + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + if (const auto* child = get_field_ptr(child_ptr); child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + } + } + } else if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field) { + if (const auto* child = get_field_ptr(field.nestedField.array_field.item_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "element"; + } + } else if (field.nestedField.__isset.map_field) { + if (field.nestedField.map_field.__isset.key_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.key_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "key"; + } + } + if (field.nestedField.map_field.__isset.value_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.value_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "value"; + } + } + } + return identity; +} + +const ColumnDefinition* find_identity_child(const ColumnDefinition& projected_child, + const ColumnDefinition& identity_parent) { + const auto child_it = std::ranges::find_if( + identity_parent.children, [&](const ColumnDefinition& identity_child) { + if (projected_child.has_identifier_field_id() && + identity_child.has_identifier_field_id()) { + return projected_child.get_identifier_field_id() == + identity_child.get_identifier_field_id(); + } + if (to_lower(projected_child.name) == to_lower(identity_child.name)) { + return true; + } + return std::ranges::any_of( + identity_child.name_mapping, [&](const std::string& alias) { + return to_lower(projected_child.name) == to_lower(alias); + }); + }); + return child_it == identity_parent.children.end() ? nullptr : &*child_it; +} + +void attach_full_schema_identity(ColumnDefinition* projected, const ColumnDefinition& identity) { + DORIS_CHECK(projected != nullptr); + // Access-path children control materialization, but wrapper discovery needs sibling IDs that + // were pruned from that projection. Keep the complete identity tree on a separate channel. + projected->identity_children = identity.children; + for (auto& projected_child : projected->children) { + if (const auto* identity_child = find_identity_child(projected_child, identity); + identity_child != nullptr) { + attach_full_schema_identity(&projected_child, *identity_child); + } + } +} + bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { if (field.__isset.name && to_lower(field.name) == to_lower(name)) { return true; @@ -189,6 +267,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .has_name_mapping = field.__isset.name_mapping_is_authoritative && + field.name_mapping_is_authoritative, .type = std::move(type), .children = {}, .default_expr = nullptr, @@ -488,6 +568,12 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; + column->has_name_mapping = context->schema_column->has_name_mapping; + // Projected roots already carry a generic FE default expression, but Iceberg binary defaults + // need the raw Base64 marker so missing-file materialization can decode rather than copy text. + column->initial_default_value = context->schema_column->initial_default_value; + column->initial_default_value_is_base64 = + context->schema_column->initial_default_value_is_base64; return Status::OK(); } @@ -530,6 +616,13 @@ Status TableReader::init(TableReadOptions&& options) { _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; _projected_columns = std::move(options.projected_columns); + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity(&projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } _system_properties = create_system_properties(_scan_params); _mapper_options.mode = TableColumnMappingMode::BY_NAME; _conjuncts = std::move(options.conjuncts); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 73bade81bc4c65..6718a79eee6634 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -360,6 +360,7 @@ class TableReader { Status create_next_reader(bool* eos); virtual Status create_file_reader(std::unique_ptr* reader); virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { DORIS_CHECK(file_schema != nullptr); return Status::OK(); @@ -376,6 +377,7 @@ class TableReader { RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); + configure_mapper_options(&_mapper_options); _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); DORIS_CHECK(_data_reader.column_mapper != nullptr); @@ -1319,7 +1321,10 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) + (child_mapping->initial_default_column + ? child_mapping->initial_default_column->clone_resized(rows) + : child_mapping->table_type + ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); continue; } diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..eb4198df640592 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -59,11 +59,13 @@ TColumnAccessPath meta_access_path() { format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, std::vector children = {}, - std::vector aliases = {}) { + std::vector aliases = {}, + bool has_name_mapping = false) { return { .identifier = Field::create_field(id), .name = std::move(name), .name_mapping = std::move(aliases), + .has_name_mapping = has_name_mapping, .type = std::move(type), .children = std::move(children), }; @@ -167,7 +169,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { .type = struct_type, .children = { - field(101, "a", int_type), + field(101, "a", int_type, {}, {}, true), field(205, "b", int_type, {}, {"old_b"}), }, }; @@ -199,6 +201,15 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { expect_child(column.children[0], 205, "b"); EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "a"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); + } { auto column = root_column(100, "s", struct_type); auto status = AccessPathParser::build_nested_children( @@ -206,6 +217,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); expect_child(column.children[1], 205, "b"); } @@ -235,7 +247,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { field(201, "element", element_type, { field(202, "item", string_type, {}, {"old_item"}), - field(203, "quantity", int_type), + field(203, "quantity", int_type, {}, {}, true), }), }, }; @@ -254,6 +266,18 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { EXPECT_EQ(column.children[0].children[0].name_mapping, std::vector({"old_item"})); } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "quantity"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[0].has_name_mapping); + } { auto column = root_column(200, "items", array_type); auto status = AccessPathParser::build_nested_children( @@ -264,6 +288,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { ASSERT_EQ(column.children[0].children.size(), 2); expect_child(column.children[0].children[0], 202, "item"); expect_child(column.children[0].children[1], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[1].has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -293,7 +318,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { field(302, "value", value_type, { field(303, "full_name", string_type, {}, {"name"}), - field(304, "age", int_type), + field(304, "age", int_type, {}, {}, true), field(305, "gender", string_type), }), }, @@ -329,6 +354,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { expect_child(column.children[1], 302, "value"); ASSERT_EQ(column.children[1].children.size(), 1); expect_child(column.children[1].children[0], 304, "age"); + EXPECT_TRUE(column.children[1].children[0].has_name_mapping); } { auto column = root_column(300, "m", map_type); @@ -357,6 +383,9 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); ASSERT_EQ(column.children[1].children.size(), 3); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + EXPECT_TRUE(age->has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -368,4 +397,21 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { } } +TEST(AccessPathParserTest, PreservesNestedInitialDefaultMetadata) { + auto binary_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {binary_type}, Strings {"data"}); + auto defaulted_child = field(101, "data", binary_type); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto schema = field(100, "s", struct_type, {defaulted_child}); + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "data"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].initial_default_value, std::optional("AAEC/w==")); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index bc3f916281de7d..5efc7f6b6a9050 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -17,9 +17,15 @@ #include "format/table/iceberg_position_delete_sys_table_reader.h" +#include +#include #include +#include +#include +#include #include +#include #include #include #include @@ -33,6 +39,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "format/orc/orc_memory_stream_test.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -43,6 +50,8 @@ namespace doris { namespace { +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file); + class ProfileTrackingReader final : public GenericReader { public: int collect_calls = 0; @@ -115,6 +124,383 @@ Int64 struct_int_at(const Block& block, const std::string& name, size_t child_in return child.get_nested_column().get_int(row); } +bool struct_child_is_null_at(const Block& block, const std::string& name, size_t child_index, + size_t row) { + const auto& struct_column = assert_cast(nested_column(block, name)); + const auto& child = assert_cast(struct_column.get_column(child_index)); + return child.is_null_at(row); +} + +std::shared_ptr int32_array(int32_t value) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(value).ok()); + auto result = builder.Finish(); + EXPECT_TRUE(result.ok()) << result.status(); + return result.ok() ? *result : nullptr; +} + +void write_mixed_id_position_delete_parquet(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false); + if (include_child_id) { + legacy_a = legacy_a->WithMetadata(id(1)); + } + auto idless_b = arrow::field("b", arrow::int32(), false); + auto row_array = + arrow::StructArray::Make({int32_array(42), int32_array(99)}, {legacy_a, idless_b}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto row_field = arrow::field("row", arrow::struct_({legacy_a, idless_b}), false); + if (row_id.has_value()) { + row_field = row_field->WithMetadata(id(*row_id)); + } + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + row_field, + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_nested_wrapper_position_delete_parquet(const std::string& path) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false)->WithMetadata(id(1)); + auto s_array = arrow::StructArray::Make({int32_array(42)}, {legacy_a}); + ASSERT_TRUE(s_array.ok()) << s_array.status(); + auto idless_s = arrow::field("s", arrow::struct_({legacy_a}), false); + auto row_array = arrow::StructArray::Make({*s_array}, {idless_s}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + arrow::field("row", arrow::struct_({idless_s}), false)->WithMetadata(id(2147483544)), + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_mixed_id_position_delete_orc(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>")); + type->getSubtype(0)->setAttribute("iceberg.id", "2147483546"); + type->getSubtype(1)->setAttribute("iceberg.id", "2147483545"); + if (row_id.has_value()) { + type->getSubtype(2)->setAttribute("iceberg.id", std::to_string(*row_id)); + } + if (include_child_id) { + type->getSubtype(2)->getSubtype(0)->setAttribute("iceberg.id", "1"); + } + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& file_path = dynamic_cast<::orc::StringVectorBatch&>(*root.fields[0]); + auto& pos = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[1]); + auto& row = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[2]); + auto& legacy_a = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[0]); + auto& idless_b = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[1]); + std::string data_path = "s3://bucket/data.orc"; + file_path.data[0] = data_path.data(); + file_path.length[0] = data_path.size(); + pos.data[0] = 5; + legacy_a.data[0] = 42; + idless_b.data[0] = 99; + root.numElements = file_path.numElements = pos.numElements = row.numElements = + legacy_a.numElements = idless_b.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +format::ColumnDefinition table_column(int32_t id, std::string name, DataTypePtr type) { + format::ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = std::move(name); + column.type = std::move(type); + return column; +} + +schema::external::TFieldPtr external_scalar_field(const std::string& name, int32_t id, + TPrimitiveType::type type) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + TColumnType column_type; + column_type.__set_type(type); + field->__set_type(column_type); + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; +} + +schema::external::TStructField position_delete_table_schema() { + schema::external::TStructField row_children; + row_children.__set_fields({external_scalar_field("a", 1, TPrimitiveType::INT), + external_scalar_field("b", 2, TPrimitiveType::INT)}); + auto row = std::make_shared(); + row->__set_name("row"); + row->__set_id(2147483544); + TColumnType row_type; + row_type.__set_type(TPrimitiveType::STRUCT); + row->__set_type(row_type); + row->nestedField.__set_struct_field(std::move(row_children)); + row->__isset.nestedField = true; + schema::external::TFieldPtr row_ptr; + row_ptr.__set_field_ptr(std::move(row)); + + schema::external::TStructField root; + root.__set_fields({external_scalar_field("file_path", 2147483546, TPrimitiveType::STRING), + external_scalar_field("pos", 2147483545, TPrimitiveType::BIGINT), + std::move(row_ptr)}); + return root; +} + +void run_v1_idless_row_position_delete_test(TFileFormatType::type file_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_v1_position_delete_idless_row_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == TFileFormatType::FORMAT_PARQUET) { + write_mixed_id_position_delete_parquet(path, std::nullopt, false); + } else { + write_mixed_id_position_delete_orc(path, std::nullopt, false); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("UTC"); + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(file_format); + scan_params.__set_current_schema_id(-1); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(position_delete_table_schema()); + scan_params.__set_history_schema_info({std::move(current_schema)}); + auto io_ctx = std::make_shared(); + io::FileReaderStats file_reader_stats; + io_ctx->file_reader_stats = &file_reader_stats; + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(file_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_size(static_cast(std::filesystem::file_size(path))); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, &scan_params, io_ctx, + nullptr); + ReaderInitContext context; + ASSERT_TRUE(reader.init_reader(&context).ok()); + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader.get_next_block(&block, &read_rows, &eof).ok()); + ASSERT_EQ(1, read_rows); + ASSERT_EQ(1, block.rows()); + // Top-level delete fields carry IDs, so an ID-less physical row must not bind by name. + EXPECT_TRUE(is_null_at(block, "row", 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +void run_mixed_id_position_delete_test(format::FileFormat file_format, + TFileFormatType::type thrift_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_iceberg_position_delete_mixed_id_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == format::FileFormat::PARQUET) { + write_mixed_id_position_delete_parquet(path); + } else { + write_mixed_id_position_delete_orc(path); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + auto a = table_column(1, "a", nullable_int32); + a.name_mapping = {"legacy_a"}; + a.has_name_mapping = true; + auto b = table_column(2, "b", nullable_int32); + auto row = table_column(2147483544, "row", row_type); + row.children = {a, b}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(thrift_format); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = file_format, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(thrift_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = file_format; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + ASSERT_FALSE(struct_child_is_null_at(block, "row", 0, 0)); + EXPECT_EQ(42, struct_int_at(block, "row", 0, 0)); + // Once any file field has an Iceberg id, the id-less sibling is absent rather than name-bound. + EXPECT_TRUE(struct_child_is_null_at(block, "row", 1, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +void run_v2_nested_wrapper_position_delete_test() { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v2_position_delete_nested_wrapper_parquet"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / "delete.parquet").string(); + write_nested_wrapper_position_delete_parquet(path); + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto s_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"a"})); + const auto row_type = + make_nullable(std::make_shared(DataTypes {s_type}, Strings {"s"})); + auto a = table_column(1, "a", nullable_int32); + auto s = table_column(10, "s", s_type); + s.children = {a}; + auto row = table_column(2147483544, "row", row_type); + row.children = {s}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = format::FileFormat::PARQUET; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + EXPECT_EQ("{\"s\":{\"a\":42}}", row_type->to_string(*block.get_by_position(0).column, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { TIcebergFileDesc iceberg_desc; iceberg_desc.__set_delete_files({delete_file}); @@ -502,6 +888,14 @@ TEST(IcebergPositionDeleteSysTableReaderTest, AppendsNullMetadataAndUsesDeletePa EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableReaderTest, ParquetUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_PARQUET, "parquet"); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, OrcUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_ORC, "orc"); +} + TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) { io::FileReaderStats file_reader_stats; auto scanner_io_ctx = std::make_shared(); @@ -603,4 +997,17 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVect EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "parquet"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { + run_v2_nested_wrapper_position_delete_test(); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6c504e0b49a46b..0d5a6217f766f3 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -138,6 +138,46 @@ void write_iceberg_three_int_orc_file( output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } +void write_iceberg_orc_containers_with_idless_struct_children(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>,attrs:map>>")); + type->getSubtype(0)->setAttribute("iceberg.id", "10"); + type->getSubtype(1)->setAttribute("iceberg.id", "20"); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + auto& items = dynamic_cast<::orc::ListVectorBatch&>(*root.fields[0]); + auto& item = dynamic_cast<::orc::StructVectorBatch&>(*items.elements); + auto& item_a = dynamic_cast<::orc::LongVectorBatch&>(*item.fields[0]); + items.offsets[0] = 0; + items.offsets[1] = 1; + item_a.data[0] = 42; + items.numElements = item.numElements = item_a.numElements = 1; + + auto& attrs = dynamic_cast<::orc::MapVectorBatch&>(*root.fields[1]); + auto& attr_key = dynamic_cast<::orc::LongVectorBatch&>(*attrs.keys); + auto& attr_value = dynamic_cast<::orc::StructVectorBatch&>(*attrs.elements); + auto& attr_a = dynamic_cast<::orc::LongVectorBatch&>(*attr_value.fields[0]); + attrs.offsets[0] = 0; + attrs.offsets[1] = 1; + attr_key.data[0] = 1; + attr_a.data[0] = 43; + attrs.numElements = attr_key.numElements = attr_value.numElements = attr_a.numElements = 1; + + root.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + std::shared_ptr build_iceberg_int32_array(const std::vector& values) { arrow::Int32Builder builder; for (const auto value : values) { @@ -183,6 +223,52 @@ void write_iceberg_three_int_parquet_file( properties.build())); } +void write_iceberg_id_and_idless_struct_parquet_file(const std::string& file_path, int32_t id, + int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = + arrow::field("a", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = arrow::field("s", arrow::struct_({child_field}), false); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_iceberg_id_and_struct_with_idless_child_parquet_file(const std::string& file_path, + int32_t id, int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = arrow::field("legacy_a", arrow::int32(), false); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, const std::string& field_name, int32_t field_id, int32_t value) { @@ -245,6 +331,128 @@ Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::st return Status::OK(); } +Status create_single_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TStructField child_field; + child_field.__set_name("a"); + child_field.__set_contains_null(true); + TTypeNode struct_node; + struct_node.__set_type(TTypeNodeType::STRUCT); + struct_node.__set_struct_fields({child_field}); + TTypeNode child_node; + child_node.__set_type(TTypeNodeType::SCALAR); + TScalarType child_scalar; + child_scalar.__set_type(TPrimitiveType::INT); + child_node.__set_scalar_type(child_scalar); + TTypeDesc type; + type.__set_types({struct_node, child_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(10); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(-1); + slot_descriptor.__set_colName("s"); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(16); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + +Status create_array_map_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(2); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + auto scalar_int_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar; + scalar.__set_type(TPrimitiveType::INT); + node.__set_scalar_type(scalar); + return node; + }; + auto struct_a_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::STRUCT); + TStructField child; + child.__set_name("a"); + child.__set_contains_null(true); + node.__set_struct_fields({child}); + return node; + }; + + TTypeNode array_node; + array_node.__set_type(TTypeNodeType::ARRAY); + array_node.__set_contains_nulls({true}); + TTypeDesc items_type; + items_type.__set_types({array_node, struct_a_node(), scalar_int_node()}); + + TTypeNode map_node; + map_node.__set_type(TTypeNodeType::MAP); + map_node.__set_contains_nulls({false, true}); + TTypeDesc attrs_type; + attrs_type.__set_types({map_node, scalar_int_node(), struct_a_node(), scalar_int_node()}); + + std::vector slots; + auto add_slot = [&slots](int id, const std::string& name, int field_id, const TTypeDesc& type) { + TSlotDescriptor slot; + slot.__set_id(id); + slot.__set_parent(0); + slot.__set_col_unique_id(field_id); + slot.__set_slotType(type); + slot.__set_columnPos(id); + slot.__set_byteOffset(0); + slot.__set_nullIndicatorByte(0); + slot.__set_nullIndicatorBit(-1); + slot.__set_colName(name); + slot.__set_slotIdx(id); + slot.__set_isMaterialized(true); + slots.emplace_back(std::move(slot)); + }; + add_slot(0, "items", 10, items_type); + add_slot(1, "attrs", 20, attrs_type); + thrift_table.__set_slotDescriptors(std::move(slots)); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(32); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1076,12 +1284,12 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul }; schema::external::TStructField root_field; - root_field.__set_fields( - {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, - "2024-01-01 00:00:00.123456", -1, 6, false), - make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), - make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, - true)}); + root_field.__set_fields({make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, + "Ej5FZ+ibEtOkVkJmFBdAAA==", 16, -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, + "AAEC/w==", -1, -1, true)}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); @@ -1102,7 +1310,7 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul &runtime_state, cache.get()); const auto timestamp_type = make_nullable(std::make_shared(6)); - const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto varbinary_type = make_nullable(std::make_shared(16)); const auto string_type = make_nullable(std::make_shared()); Block block; block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); @@ -1126,7 +1334,7 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), "2024-01-01 00:00:00.123456"); EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), - std::string("\x00\x01\x02\xff", 4)); + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), std::string("\x00\x01\x02\xff", 4)); EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); @@ -1134,6 +1342,53 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); } +TEST_F(IcebergReaderTest, v1_top_level_missing_binary_prefers_iceberg_initial_default) { + auto field = std::make_shared(); + field->__set_name("added_uuid"); + field->__set_id(1); + field->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + field->__set_initial_default_value_is_base64(true); + TColumnType thrift_type; + thrift_type.__set_type(TPrimitiveType::VARBINARY); + thrift_type.__set_len(16); + field->__set_type(thrift_type); + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + schema::external::TStructField root_field; + root_field.__set_fields({std::move(field_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto varbinary_type = make_nullable(std::make_shared(16)); + Block block; + block.insert({varbinary_type->create_column(), varbinary_type, "added_uuid"}); + std::unordered_map positions = {{"added_uuid", 0}}; + reader.TEST_set_column_name_to_block_index(&positions); + + ASSERT_TRUE(reader.on_fill_missing_columns(&block, 2, {"added_uuid"}).ok()); + + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(0).column, 0), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_default) { schema::external::TStructField root_field; root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), @@ -1299,17 +1554,324 @@ TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_parquet_reads_idless_wrapper_with_authoritative_empty_mapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_idless_struct_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":42}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_keeps_file_id_mode_inside_nested_struct) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_whole_file_id_mode_nested_struct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_struct_with_idless_child_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + child_field->__set_initial_default_value("7"); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":7}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_orc_keeps_file_id_mode_inside_array_and_map) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_orc_whole_file_id_mode_containers_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + write_iceberg_orc_containers_with_idless_struct_children(data_file); + + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto make_struct_field = [&struct_type](const std::string& name, int32_t id, int32_t child_id, + const std::string& default_value) { + schema::external::TStructField children; + children.__set_fields({make_external_int_field("a", child_id, default_value)}); + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(struct_type); + field->nestedField.__set_struct_field(std::move(children)); + field->__isset.nestedField = true; + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; + }; + + TColumnType array_type; + array_type.__set_type(TPrimitiveType::ARRAY); + auto items = std::make_shared(); + items->__set_name("items"); + items->__set_id(10); + items->__set_type(array_type); + schema::external::TArrayField array_field; + array_field.__set_item_field(make_struct_field("element", 11, 12, "7")); + items->nestedField.__set_array_field(std::move(array_field)); + items->__isset.nestedField = true; + schema::external::TFieldPtr items_ptr; + items_ptr.__set_field_ptr(std::move(items)); + + TColumnType map_type; + map_type.__set_type(TPrimitiveType::MAP); + auto attrs = std::make_shared(); + attrs->__set_name("attrs"); + attrs->__set_id(20); + attrs->__set_type(map_type); + schema::external::TMapField map_field; + map_field.__set_key_field(make_external_int_field("key", 21, std::nullopt)); + map_field.__set_value_field(make_struct_field("value", 22, 23, "8")); + attrs->nestedField.__set_map_field(std::move(map_field)); + attrs->__isset.nestedField = true; + schema::external::TFieldPtr attrs_ptr; + attrs_ptr.__set_field_ptr(std::move(attrs)); + + schema::external::TStructField root_field; + root_field.__set_fields({std::move(items_ptr), std::move(attrs_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_array_map_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(2); + column_descriptors[0].name = "items"; + column_descriptors[1].name = "attrs"; + std::unordered_map block_positions = {{"items", 0}, {"attrs", 1}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + Block block; + for (const auto* slot : tuple_descriptor->slots()) { + const auto& type = slot->get_data_type_ptr(); + block.insert({type->create_column(), type, slot->col_name()}); + } + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(tuple_descriptor->slots()[0]->get_data_type_ptr()->to_string( + *block.get_by_position(0).column, 0), + "[{\"a\":7}]"); + EXPECT_EQ(tuple_descriptor->slots()[1]->get_data_type_ptr()->to_string( + *block.get_by_position(1).column, 0), + "{1:{\"a\":8}}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.parquet").string(); const auto delete_file = (test_dir / "equality-delete.parquet").string(); - // The id-less columns put the complete Parquet file in BY_NAME mode. The unrelated - // stale_added column deliberately retains field id 1, which must not override the historical - // alias selected for the hidden added_column equality key. - write_iceberg_three_int_parquet_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + // Iceberg's hasIds rule is existential: once any physical field has an ID, equality keys bind + // to that authoritative ID instead of an ID-less historical alias. + write_iceberg_three_int_parquet_file(data_file, "id", 0, {1, 2, 3}, "legacy_added", std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); write_iceberg_int_equality_delete_parquet_file(delete_file, "added_column", 1, 7); @@ -1387,26 +1949,26 @@ TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_fi bool eof = false; const auto status = reader.get_next_block(&block, &read_rows, &eof); ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(read_rows, 2); - ASSERT_EQ(block.rows(), 2); + ASSERT_EQ(read_rows, 3); + ASSERT_EQ(block.rows(), 3); EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); - EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "2"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 2), "3"); std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_orc_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_orc_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.orc").string(); const auto delete_file = (test_dir / "equality-delete.orc").string(); - // One id-less column switches the whole file to BY_NAME. The hidden current key is physically - // stored as its id-less historical alias, while an unrelated migrated column still carries - // the key's stale field id. Equality-delete binding must select legacy_added, not stale_added. - write_iceberg_three_int_orc_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", - std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + // Mixed ORC schemas also stay in ID projection when any field has an ID; the key with ID 1 is + // authoritative even though an ID-less historical alias is present. + write_iceberg_three_int_orc_file(data_file, "id", 0, {1, 2, 3}, "legacy_added", std::nullopt, + {5, 7, 9}, "stale_added", 1, {70, 70, 70}); write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); schema::external::TStructField root_field; @@ -1478,10 +2040,11 @@ TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_ bool eof = false; const auto status = reader.get_next_block(&block, &read_rows, &eof); ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(read_rows, 2); - ASSERT_EQ(block.rows(), 2); + ASSERT_EQ(read_rows, 3); + ASSERT_EQ(block.rows(), 3); EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); - EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "2"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 2), "3"); std::filesystem::remove_all(test_dir); } diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 3fcb9d39878255..96c32f3ca55248 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -35,6 +36,54 @@ namespace doris { class MockTableSchemaChangeHelper : public TableSchemaChangeHelper {}; +namespace { + +schema::external::TStructField partial_name_mapping_root_field() { + TColumnType int_type; + int_type.type = TPrimitiveType::INT; + + schema::external::TStructField root_field; + for (const auto& [name, id, aliases] : + std::vector>> {{"a", 1, {"a"}}, + {"b", 2, {}}}) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(int_type); + field->__set_name_mapping(aliases); + field->__set_name_mapping_is_authoritative(true); + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + root_field.fields.emplace_back(std::move(field_ptr)); + } + return root_field; +} + +schema::external::TStructField nested_partial_name_mapping_root_field() { + TColumnType struct_type; + struct_type.type = TPrimitiveType::STRUCT; + + auto nested_fields = partial_name_mapping_root_field(); + nested_fields.fields[0].field_ptr->__set_name_mapping({}); + + auto field = std::make_shared(); + field->__set_name("s"); + field->__set_id(10); + field->__set_type(struct_type); + field->__set_name_mapping({}); + field->__set_name_mapping_is_authoritative(true); + field->nestedField.__set_struct_field(std::move(nested_fields)); + field->__isset.nestedField = true; + + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + schema::external::TStructField root_field; + root_field.fields.emplace_back(std::move(field_ptr)); + return root_field; +} + +} // namespace + TEST(PartitionColumnFillerTest, FillNullableStringPartitionValue) { SlotDescriptor slot; slot._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); @@ -425,6 +474,260 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetLegacyEmptyNameMappingFallsBack) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields.resize(1); + root_field.fields[0].field_ptr->__set_name_mapping({}); + root_field.fields[0].field_ptr->__isset.name_mapping_is_authoritative = false; + + FieldDescriptor parquet_field; + FieldSchema file_field; + file_field.name = "a"; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + FieldDescriptor parquet_field; + for (const auto& name : {"a", "b"}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + FieldDescriptor parquet_field; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = field_id; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); + + FieldSchema struct_field; + struct_field.name = "s"; + struct_field.field_id = 10; + std::vector child_types; + Strings child_names; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema child; + child.name = name; + child.field_id = field_id; + child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + child_types.emplace_back(child.data_type); + child_names.emplace_back(child.name); + struct_field.children.emplace_back(std::move(child)); + } + struct_field.data_type = std::make_shared(child_types, child_names); + + FieldDescriptor parquet_field; + parquet_field._fields.emplace_back(std::move(struct_field)); + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); +} + +TEST(MockTableSchemaChangeHelper, + IcebergParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_fields; + struct_fields.__set_fields({child_ptr}); + + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_fields); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "a"; + file_child.field_id = 2; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = -1; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetKeepsWholeFileIdModeInNestedStruct) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "legacy_a"; + file_child.field_id = -1; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = 10; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; @@ -536,6 +839,89 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergOrcPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + std::shared_ptr ans_node; + ASSERT_TRUE( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + orc_type->getSubtype(0)->setAttribute(IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); + + std::unique_ptr orc_type( + orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->setAttribute(attribute, "10"); + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcDoesNotBindIdlessWrapperByName) { + auto root_field = nested_partial_name_mapping_root_field(); + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, NestedMapArrayStruct) { // struct, struct>> SlotDescriptor slot1; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 3bca3c4152c888..567f44dea2bb56 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -253,6 +253,81 @@ TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { } } +TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { + auto table_child = field_id_col("a", 1, i32()); + auto table_struct = struct_col("s", 10, {table_child}); + auto file_child = field_id_col("legacy_a", 1, i32(), 0); + auto file_struct = struct_name_col("s", {file_child}, 0); + + TableColumnMapper parquet_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(parquet_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(parquet_mapper.mappings().size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(parquet_mapper.mappings()[0].child_mappings.size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].child_mappings[0].file_local_id, 0); + + TableColumnMapper orc_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(orc_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + auto table_struct = struct_col("s", 10, {field_id_col("a", 2, i32())}); + table_struct.has_name_mapping = true; + auto file_struct = struct_name_col("s", {field_id_col("a", 2, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + EXPECT_TRUE(mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { + auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); + auto table_outer = struct_col("outer", 10, {table_inner}); + auto file_inner = struct_name_col("inner", {field_id_col("leaf", 30, i32(), 0)}, 0); + auto file_outer = struct_col("outer", 10, {file_inner}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_outer}, {}, {file_outer}).ok()); + + const auto& outer_mapping = mapper.mappings()[0]; + ASSERT_TRUE(outer_mapping.file_local_id.has_value()); + ASSERT_EQ(outer_mapping.child_mappings.size(), 1); + const auto& inner_mapping = outer_mapping.child_mappings[0]; + ASSERT_TRUE(inner_mapping.file_local_id.has_value()); + ASSERT_EQ(inner_mapping.child_mappings.size(), 1); + EXPECT_TRUE(inner_mapping.child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { + auto defaulted_child = field_id_col("data", 2, varbinary()); + defaulted_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + defaulted_child.initial_default_value_is_base64 = true; + auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); + auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + const auto& missing = mapper.mappings()[0].child_mappings[1]; + ASSERT_TRUE(missing.initial_default_column); + Field value; + missing.initial_default_column->get(0, value); + EXPECT_EQ(value.get_type(), TYPE_VARBINARY); + EXPECT_EQ(std::string(value.get()), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + void expect_mapping(const ColumnMapping& mapping, size_t global_index, const std::string& table_name, int32_t file_local_id, const std::string& file_name, const DataTypePtr& file_type, diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 0831c98750ecbf..22642f20b68093 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -495,6 +495,54 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value, + bool outer_has_field_id = true) { + const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); + auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); + auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); + ASSERT_TRUE(inner_result.ok()) << inner_result.status(); + auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); + auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false); + if (outer_has_field_id) { + outer_field = + outer_field->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + } + auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_nullable_idless_struct_with_sibling_id_parquet_file(const std::string& file_path) { + const auto sibling_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto sibling_field = arrow::field("b", arrow::int32(), false)->WithMetadata(sibling_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 42})}, {sibling_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = arrow::field("s", arrow::struct_({sibling_field}), true); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -1843,9 +1891,9 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { TableColumnMappingMode::BY_FIELD_ID); } -// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data -// column without an Iceberg field id still disables field-id mapping. -TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { +// Iceberg treats a schema as ID-bearing when any physical data field has an ID. Keeping ID mode +// preserves authoritative matches even when a sibling was written without an ID. +TEST(IcebergV2ReaderTest, IcebergMappingModeUsesAnyDataColumnFieldId) { IcebergTableReaderMappingModeTestHelper reader; std::vector file_schema { make_file_column(1, "id", std::make_shared()), @@ -1854,8 +1902,13 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { }; file_schema[1].identifier = Field {}; + EXPECT_EQ(reader.mapping_mode_for_schema(file_schema), TableColumnMappingMode::BY_FIELD_ID); + + file_schema[0].identifier = Field {}; + file_schema[0].children.emplace_back( + make_file_column(3, "nested", std::make_shared())); EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), - TableColumnMappingMode::BY_NAME); + TableColumnMappingMode::BY_FIELD_ID); } TEST(IcebergV2ReaderTest, IcebergTableReaderDoesNotPushDownAggregateWithPositionDelete) { @@ -2240,7 +2293,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + const std::string binary_default( + "\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2249,10 +2303,10 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "Ej5FZ+ibEtOkVkJmFBdAAA==", + external_primitive_type(TPrimitiveType::VARBINARY, 16), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -2443,7 +2497,173 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { +TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_recursive_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42, false); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + inner.has_name_mapping = true; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + outer.has_name_mapping = true; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_unprojected_sibling_id_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.initial_default_value = "7"; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + auto schema_a = + external_schema_field("a", 1, {}, "7", external_primitive_type(TPrimitiveType::INT)); + auto schema_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + schema::external::TStructField struct_children; + struct_children.__set_fields({schema_a, schema_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; std::filesystem::remove_all(test_dir); @@ -2451,8 +2671,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); - // The real key has no id and is mapped by its historical name. A different physical column - // carries the stale id 0, forcing the entire split into BY_NAME mode. + // Iceberg's existential hasIds contract makes the physical field carrying ID 0 authoritative; + // an ID-less alias cannot switch a mixed schema back to name projection. write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", {100, 200, 300}, 0); write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); @@ -2477,7 +2697,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({100, 200, 300})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 05ef805ae71267..d4c52d5c1fbcb2 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -41,6 +41,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -48,6 +49,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" @@ -1942,6 +1944,108 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, IcebergInitialDefaultMetadataOverridesGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_top_level_binary_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto int_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); + auto id_column = make_table_column(-1, "id", int_type); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + ASSERT_TRUE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(binary_column.initial_default_value_is_base64); + + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { + auto unmapped_field = external_schema_field("b", 2); + unmapped_field.field_ptr->__set_name_mapping({}); + unmapped_field.field_ptr->__set_name_mapping_is_authoritative(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_TRUE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(TableReaderTest, LegacyFeEmptyNameMappingStillMatchesCurrentFileName) { + auto legacy_field = external_schema_field("b", 2); + legacy_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {legacy_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_FALSE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); @@ -4029,7 +4133,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -4039,10 +4143,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { write_struct_parquet_file(file_path, 7); const auto int_type = std::make_shared(); - const auto string_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); - auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + auto missing_child = make_table_column(99, "missing_child", varbinary_type); + missing_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + missing_child.initial_default_value_is_base64 = true; + auto struct_type = std::make_shared(DataTypes {int_type, varbinary_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); struct_column.children = {id_child, missing_child}; @@ -4076,7 +4182,10 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& defaults = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(1))); + EXPECT_EQ(defaults.get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 6a986da45303d6..37dbf4cdd390ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -124,8 +124,7 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { - initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), base64InitialDefaults); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); + tSchema.setRootField(getExternalSchemaForAllColumn( + columns, nameMapping, hasNameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -161,11 +170,13 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { - return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + return getExternalSchema(columnType, dorisColumn, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); @@ -178,9 +189,12 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { - // for iceberg set name mapping. - root.setNameMapping(new ArrayList<>(nameMapping.get(dorisColumn.getUniqueId()))); + if (hasNameMapping) { + // The explicit capability keeps old-FE plans on legacy fallback while making an empty + // per-field mapping authoritative for plans produced by a compatible FE. + root.setNameMapping(new ArrayList<>( + nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); + root.setNameMappingIsAuthoritative(true); } TNestedField nestedField = new TNestedField(); @@ -199,7 +213,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( - subField.getType(), subColumn, nameMapping, base64InitialDefaults)); + subField.getType(), subColumn, nameMapping, hasNameMapping, + base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -212,7 +227,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -223,13 +238,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index bd16057dfd9569..90b0fd90e5b97b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -231,7 +231,8 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTabl icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 95c9a6f26cc5c5..a17ff87b1131c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,14 +17,30 @@ package org.apache.doris.datasource.iceberg; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; + private final Optional>> nameMapping; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { + this(partitionInfo, snapshot, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.nameMapping = nameMapping.map(mapping -> { + Map> copy = new HashMap<>(); + mapping.forEach((id, names) -> copy.put(id, List.copyOf(names))); + return Map.copyOf(copy); + }); } public IcebergPartitionInfo getPartitionInfo() { @@ -34,4 +50,8 @@ public IcebergPartitionInfo getPartitionInfo() { public IcebergSnapshot getSnapshot() { return snapshot; } + + public Optional>> getNameMapping() { + return nameMapping; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index b6f1d22046d76e..755bf83afdd959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -108,6 +108,10 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mapping.MappedField; +import org.apache.iceberg.mapping.MappedFields; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; @@ -1115,8 +1119,18 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + String serializedDefault = serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz); + // Column constructs complex children without Iceberg field metadata. Copy through the + // public default-info API so recursive fields retain their logical pre-add value. + Column defaultCarrier = new Column(column.getName(), column.getType(), false, null, + column.isAllowNull(), serializedDefault, ""); + column.setDefaultValueInfo(defaultCarrier); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1135,7 +1149,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1192,7 +1207,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { @@ -1819,7 +1834,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( } return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId())); + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(icebergTable)); } return getLatestSnapshotCacheValue(dorisTable); } @@ -1900,12 +1916,51 @@ private static IcebergSnapshotCacheValue loadSnapshotCacheValue(ExternalTable do icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } } + /** + * Extract the Iceberg name mapping while retaining the distinction between an absent property + * and a valid empty mapping. + */ + public static Optional>> getNameMapping(Table icebergTable) { + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } + try { + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); + } + Map> result = new HashMap<>(); + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); + } catch (Exception e) { + LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); + } + } + + private static void extractMappingsFromNameMapping( + MappedFields mappingFields, Map> result) { + if (mappingFields == null) { + return; + } + for (MappedField mappedField : mappingFields.fields()) { + // Iceberg permits id-less wrapper entries; only their nested ID-bearing aliases can + // participate in Doris field-id lookup and in the immutable snapshot cache. + if (mappedField.id() != null) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + } + extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + } + } + private static Table loadIcebergTableWithSession(ExternalTable dorisTable) { IcebergExternalCatalog catalog = (IcebergExternalCatalog) dorisTable.getCatalog(); IcebergMetadataOps ops = (IcebergMetadataOps) catalog.getMetadataOps(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 972009fb1e7ccf..e28f6bfa9fae1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -40,12 +40,15 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.profile.IcebergMetricsReporter; import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.persist.gson.GsonUtils; @@ -93,7 +96,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Binder; import org.apache.iceberg.expressions.Expression; @@ -103,10 +105,6 @@ import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.mapping.MappedField; -import org.apache.iceberg.mapping.MappedFields; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.NestedField; @@ -268,38 +266,14 @@ protected void doInitialize() throws UserException { super.doInitialize(); } - /** - * Extract name mapping from Iceberg table properties. - * Returns a map from field ID to list of mapped names. - */ - private Map> extractNameMapping() { - Map> result = new HashMap<>(); - try { - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson != null && !nameMappingJson.isEmpty()) { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping != null) { - // Extract mappings from NameMapping - // NameMapping contains field mappings, we need to convert them to our format - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - } - } - } catch (Exception e) { - // If name mapping parsing fails, continue without it - LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + private Optional>> extractNameMapping() { + Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + // The mapping must come from the same metadata generation as the pinned schema; a + // property-only refresh can otherwise change alias semantics within one statement. + return ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getNameMapping(); } - return result; - } - - private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { - if (mappingFields == null) { - return; - } - for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); - extractMappingsFromNameMapping(mappedField.nestedMapping(), result); - } - + return IcebergUtils.getNameMapping(icebergTable); } @Override @@ -532,13 +506,14 @@ private List getOrderedPathPartitionKeys() { public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); // Extract name mapping from Iceberg table properties - Map> nameMapping = extractNameMapping(); + Optional>> nameMapping = extractNameMapping(); // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), - nameMapping, getBase64EncodedInitialDefaultsForScan()); + nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), + getBase64EncodedInitialDefaultsForScan()); } @VisibleForTesting @@ -549,17 +524,22 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } - TableScan tableScan = createTableScan(); - Snapshot snapshot = tableScan.snapshot(); - // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. - // Resolve the selected snapshot's schema id explicitly so this metadata describes the same - // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make - // BE decode a historical non-binary default as Base64, or fail to decode a binary default. - Schema scanSchema = snapshot == null - ? tableScan.schema() - : tableScan.table().schemas().get(snapshot.schemaId()); + IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = null; + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } else { + scanSchema = selectedSnapshot == null + ? icebergTable.schema() + : icebergTable.schemas().get(selectedSnapshot.getSchemaId()); + } + // A branch can expose a schema newer than its data snapshot. The statement-pinned schema + // produced the target columns, so default markers must not be recomputed from that snapshot. return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..9422f08812de2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -28,6 +28,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.schema.external.TArrayField; import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TNestedField; import org.apache.doris.thrift.schema.external.TSchema; import org.apache.doris.thrift.schema.external.TStructField; @@ -254,6 +255,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertTrue(field1.isNameMappingIsAuthoritative()); Assert.assertEquals("7", field1.getInitialDefaultValue()); Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); @@ -262,7 +264,72 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertTrue(field2.isNameMappingIsAuthoritative()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + + @Test + public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { + StructType structType = new StructType( + new StructField("added_int", Type.INT, "nested default", true)); + Column structColumn = new Column("s", structType, true); + structColumn.setUniqueId(10); + Column child = structColumn.getChildren().get(0); + child.setUniqueId(11); + child.setDefaultValueInfo(new Column("added_int", Type.INT, false, null, true, "7", "")); + TFileScanRangeParams params = new TFileScanRangeParams(); + + ExternalUtil.initSchemaInfoForAllColumn( + params, 1L, Collections.singletonList(structColumn), Collections.emptyMap()); + + TField childField = params.getHistorySchemaInfo().get(0).getRootField().getFields().get(0) + .getFieldPtr().getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals("7", childField.getInitialDefaultValue()); + Assert.assertFalse(childField.isSetInitialDefaultValueIsBase64()); + } + + @Test + public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column mappedColumn = new Column("a", Type.INT, true); + mappedColumn.setUniqueId(1); + Column unmappedColumn = new Column("b", Type.INT, true); + unmappedColumn.setUniqueId(2); + + Map> nameMapping = new HashMap<>(); + nameMapping.put(mappedColumn.getUniqueId(), Collections.singletonList("a")); + ExternalUtil.initSchemaInfoForAllColumn( + params, 600L, Arrays.asList(mappedColumn, unmappedColumn), nameMapping); + + List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); + Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(0).getFieldPtr().isNameMappingIsAuthoritative()); + Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + Assert.assertTrue(fields.get(1).getFieldPtr().isNameMappingIsAuthoritative()); + } + + @Test + public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping() { + Column column = new Column("a", Type.INT, true); + column.setUniqueId(1); + + TFileScanRangeParams absentParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn( + absentParams, 700L, Collections.singletonList(column), Collections.emptyMap()); + TField absentField = absentParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertFalse(absentField.isSetNameMapping()); + Assert.assertFalse(absentField.isSetNameMappingIsAuthoritative()); + + TFileScanRangeParams emptyParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, + Collections.singletonList(column), Collections.emptyMap(), true, Collections.emptyMap()); + TField emptyField = emptyParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertTrue(emptyField.isSetNameMapping()); + Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + Assert.assertTrue(emptyField.isNameMappingIsAuthoritative()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 0a798faff93fef..b4a1c6a5d1b235 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -188,6 +188,11 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + // The scan now resolves initial defaults from the statement-pinned schema id, so the + // mocked table must expose the same historical-schema lookup as a real Iceberg table. + Mockito.doAnswer(invocation -> ImmutableMap.of( + mockedIcebergTable.schema().schemaId(), mockedIcebergTable.schema())) + .when(mockedIcebergTable).schemas(); // Mock newScan() chain used by IcebergScanNode.createTableScan() TableScan mockedTableScan = Mockito.mock(TableScan.class, Mockito.RETURNS_DEEP_STUBS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 349e1e0bb82ab1..f29ccd6182bd9f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -364,6 +364,20 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); } + @Test + public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional("added_int") + .withId(11) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build()))); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index c823cb2cc89465..6c47b4f76b2f4b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -22,12 +22,24 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TFileFormatType; @@ -42,6 +54,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -59,11 +72,20 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; + @SuppressWarnings("unchecked") + private static Optional>> extractNameMapping( + IcebergScanNode node) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("extractNameMapping"); + method.setAccessible(true); + return (Optional>>) method.invoke(node); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private TableScan tableScan; @@ -108,6 +130,82 @@ void addSlot(int slotId, Column column) { } } + @Test + public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Table table = Mockito.mock(Table.class); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + setIcebergSource(node, source); + + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Assert.assertFalse(extractNameMapping(node).isPresent()); + + Mockito.when(table.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + Optional>> emptyMapping = extractNameMapping(node); + Assert.assertTrue(emptyMapping.isPresent()); + Assert.assertTrue(emptyMapping.get().isEmpty()); + } + + @Test + public void testSnapshotCacheIgnoresIdlessNameMappingWrapper() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap( + TableProperties.DEFAULT_NAME_MAPPING, + "[{\"names\":[\"legacy_wrapper\"],\"fields\":[" + + "{\"field-id\":7,\"names\":[\"legacy_child\"]}]}]")); + + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), + IcebergUtils.getNameMapping(table)); + + Assert.assertTrue(snapshotCacheValue.getNameMapping().isPresent()); + Assert.assertEquals(Collections.singletonList("legacy_child"), + snapshotCacheValue.getNameMapping().get().get(7)); + Assert.assertEquals(Collections.singleton(7), + snapshotCacheValue.getNameMapping().get().keySet()); + } + + @Test + public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 11L), Optional.empty()))); + try { + Assert.assertFalse(extractNameMapping(node).isPresent()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { Schema systemTableSchema = new Schema( @@ -156,7 +254,7 @@ public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws } @Test - public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") .withId(7) .ofType(Types.BinaryType.get()) @@ -171,6 +269,7 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Mockito.when(snapshot.schemaId()).thenReturn(11); Table table = Mockito.mock(Table.class); Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + Mockito.when(table.schema()).thenReturn(currentSchema); TableScan snapshotScan = Mockito.mock(TableScan.class); Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); @@ -178,11 +277,148 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(TableIf.class)); + setIcebergSource(node, source); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertTrue(defaults.isEmpty()); + } + + @Test + public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { + Schema pinnedSchema = new Schema(11, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Schema refreshedSchema = new Schema(12, + List.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); + Mockito.when(refreshedTable.schemas()).thenReturn(Map.of( + pinnedSchema.schemaId(), pinnedSchema, + refreshedSchema.schemaId(), refreshedSchema)); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, pinnedSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + IcebergTableQueryInfo selectedSnapshot = Mockito.mock(IcebergTableQueryInfo.class); + Mockito.when(selectedSnapshot.getSchemaId()).thenReturn(11); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(snapshotScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(selectedSnapshot).when(node).getSpecifiedSnapshot(); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } + @Test + public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { + Schema dataSnapshotSchema = new Schema(11, List.of(Types.NestedField.optional("string_default") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build())); + Schema branchSchema = new Schema(12, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Snapshot dataSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Map.of( + dataSnapshotSchema.schemaId(), dataSnapshotSchema, + branchSchema.schemaId(), branchSchema)); + TableScan branchScan = Mockito.mock(TableScan.class); + Mockito.when(branchScan.snapshot()).thenReturn(dataSnapshot); + Mockito.when(branchScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(branchScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, "branch", branchSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, branchSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") @@ -213,6 +449,12 @@ private static void setIcebergTable(IcebergScanNode node, Table table) throws Ex icebergTableField.set(node, table); } + private static void setIcebergSource(IcebergScanNode node, IcebergSource source) throws Exception { + Field sourceField = IcebergScanNode.class.getDeclaredField("source"); + sourceField.setAccessible(true); + sourceField.set(node, source); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 46ae54781ae825..86915e46d28bfb 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -58,7 +58,10 @@ struct TField { // True when initial_default_value is Base64 and must be decoded before constructing the Doris // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. - 8: optional bool initial_default_value_is_base64 + 8: optional bool initial_default_value_is_base64, + // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the + // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. + 9: optional bool name_mapping_is_authoritative }