From 45e749e39d20f4442757dca44b313d0613d74f3e Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 10 Aug 2026 17:26:46 +0200 Subject: [PATCH 1/5] add tests, update doc, core logic Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 15 +- docs/en/antalya/partition_export.md | 15 +- src/Core/Settings.cpp | 5 +- src/Core/SettingsEnums.h | 1 + src/Storages/MergeTree/ExportPartTask.cpp | 20 +-- .../MergeTree/ExportPartitionUtils.cpp | 123 ++++++++++++--- src/Storages/MergeTree/ExportPartitionUtils.h | 14 +- .../tests/gtest_export_partition_ordering.cpp | 110 +++++++++++++ .../test.py | 144 +++++++++++++++--- .../test.py | 55 ++++++- .../test.py | 61 +++++--- .../test.py | 71 ++++++++- 12 files changed, 542 insertions(+), 92 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index ef7a7e25f5f7..fc4cefdafb51 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -140,11 +140,20 @@ In case a table function is used as the destination, the schema can be omitted a - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index a3b9e3dfc4f2..a57f38ba6ac9 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -132,11 +132,20 @@ Notes: - **Type**: `MergeTreePartExportSchemaMismatchMode` - **Default**: `strict` -- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. Possible values: - - `strict` - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: + - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. + - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. - The extra trailing source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. + Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. + + Error behavior: + + - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. + - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. + - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 4109ce49ac12..52ce73c6bf24 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7628,10 +7628,11 @@ Allow `EXPORT PART`/`EXPORT PARTITION` to apply lossy (non-value-preserving) cas When exporting to Apache Iceberg, the partition value written to the metadata is derived from the source partition columns by casting them to the destination partition-field types and applying the destination partition transform — the same computation the exported data files use, so the metadata stays consistent with the data. A lossy cast on a partition column remains semantically truncating: both the data files and the metadata contain the truncated value, and such casts require this setting to be enabled. )", 0) \ DECLARE(MergeTreePartExportSchemaMismatchMode, export_merge_tree_part_schema_mismatch_mode, MergeTreePartExportSchemaMismatchMode::strict, R"( -Controls whether `EXPORT PART`/`EXPORT PARTITION` allows a column-count mismatch between the source `MergeTree` table and the destination table. Columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`. +Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched. Possible values: -- `strict` (default) - the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. +- `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. +- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 0c82d40345cd..f5c48509e0b0 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -578,6 +578,7 @@ enum class MergeTreePartExportSchemaMismatchMode : uint8_t { strict, ignore_extra_source_columns_by_position, + ignore_extra_source_columns_by_name, }; DECLARE_SETTING_ENUM(MergeTreePartExportSchemaMismatchMode) diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 5ef537bbef1c..8aa3511afad4 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -113,15 +113,6 @@ namespace } } - /// Mirrors `InterpreterInsertQuery::addInsertToSelectPipeline`: positional match, - /// destination header = `getSampleBlockNonMaterialized()`, all type bridging is done - /// by the CAST inside `makeConvertingActions`. No pre-validation, no per-column - /// lossy/non-lossy classification — restrictions are exactly what INSERT SELECT enforces. - /// - /// Exception: when `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'` - /// and the source has more columns than the destination, the extra trailing source - /// columns (by position) are dropped by a preliminary projection step before the - /// positional convert, so `makeConvertingActions` always sees equal-sized inputs. void addExportConvertingActions( QueryPlan & plan_for_part, const IStorage & destination_storage, @@ -131,9 +122,10 @@ namespace = destination_storage.getInMemoryMetadataPtr()->getSampleBlockNonMaterialized(); const auto & destination_columns = destination_header.getColumnsWithTypeAndName(); + const auto schema_mismatch_mode = + local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - local_context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); @@ -169,7 +161,9 @@ namespace auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, local_context); auto expression_step = std::make_unique( @@ -353,8 +347,6 @@ bool ExportPartTask::executeStep() /// This is a hack that materializes the columns before the export so they can be exported to tables that have matching columns materializeSpecialColumns(plan_for_part.getCurrentHeader(), metadata_snapshot, local_context, plan_for_part); - /// Align the pipeline header with the destination's non-materialized sample block, - /// using the same `makeConvertingActions(Position)` call INSERT SELECT performs. addExportConvertingActions(plan_for_part, *destination_storage, local_context); QueryPlanOptimizationSettings optimization_settings(local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 0c69c24511d3..ef0aa3d6fc4f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -58,6 +58,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int THERE_IS_NO_COLUMN; extern const int INCOMPATIBLE_COLUMNS; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int FILE_ALREADY_EXISTS; @@ -112,6 +113,7 @@ namespace ExportPartitionUtils ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, ErrorCodes::ILLEGAL_COLUMN, ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + ErrorCodes::THERE_IS_NO_COLUMN, ErrorCodes::INCOMPATIBLE_COLUMNS, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, ErrorCodes::NOT_IMPLEMENTED, @@ -733,6 +735,24 @@ namespace ExportPartitionUtils source_column.type->getName(), destination_column.type->getName()); } + + void verifyExportColumnCastIsSafe( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + const StorageID & destination_storage_id) + { + if (canBeSafelyCast(source_column.type, destination_column.type)) + return; + + throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, + "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " + "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " + "to allow lossy casts during export.", + destination_storage_id.getFullTableName(), + destination_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } } void assertPartitionKeyASTAreEqual( @@ -748,6 +768,46 @@ namespace ExportPartitionUtils throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); } + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id) + { + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + { + std::unordered_map source_columns_by_name; + source_columns_by_name.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + source_columns_by_name.emplace(source_column.name, &source_column); + + for (const auto & destination_column : destination_columns) + { + const auto source_it = source_columns_by_name.find(destination_column.name); + if (source_it == source_columns_by_name.end()) + throw Exception( + ErrorCodes::THERE_IS_NO_COLUMN, + "Cannot find column `{}` in source stream", + destination_column.name); + + verifyExportColumnCastIsSafe(*source_it->second, destination_column, destination_storage_id); + } + return; + } + + if (source_columns.size() < destination_columns.size() + || (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::strict + && source_columns.size() != destination_columns.size())) + throw Exception( + ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, + "Number of columns doesn't match (source: {} and result: {})", + source_columns.size(), + destination_columns.size()); + + for (size_t i = 0; i < destination_columns.size(); ++i) + verifyExportColumnCastIsSafe(source_columns[i], destination_columns[i], destination_storage_id); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, @@ -770,9 +830,10 @@ namespace ExportPartitionUtils /// the trimming `ExportPartTask::addExportConvertingActions` applies to the real data. /// The reverse (destination has more columns than source) is always rejected below by /// `makeConvertingActions`, in both modes. + const auto schema_mismatch_mode = + context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; const bool ignore_extra_source_columns_by_position = - context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode] - == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) { @@ -788,7 +849,9 @@ namespace ExportPartitionUtils (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - ActionsDAG::MatchColumnsMode::Position, + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ? ActionsDAG::MatchColumnsMode::Name + : ActionsDAG::MatchColumnsMode::Position, context); const auto & source_columns_description = source_metadata->getColumns(); @@ -805,29 +868,47 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); - for (size_t i = 0; i < num_columns; ++i) + /// Partition-key columns must keep a stable identity across the conversion, regardless of + /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. + /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` + /// above), so look the source column up by name instead of assuming the same position. + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { - const auto & source_column = source_columns[i]; - const auto & destination_column = destination_columns[i]; + std::unordered_map source_positions_by_name; + source_positions_by_name.reserve(source_columns.size()); + for (size_t i = 0; i < source_columns.size(); ++i) + source_positions_by_name.emplace(source_columns[i].name, i); - if (partition_key_owner_columns.contains(source_column.name)) - verifyPartitionKeyColumn(source_column, destination_column, i, destination_storage_id); + for (const auto & destination_column : destination_columns) + { + if (!partition_key_owner_columns.contains(destination_column.name)) + continue; - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (allow_lossy_cast) - continue; + const auto source_it = source_positions_by_name.find(destination_column.name); + if (source_it == source_positions_by_name.end()) + continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. - if (!canBeSafelyCast(source_column.type, destination_column.type)) - throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, - "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " - "which may change values. Set `export_merge_tree_part_allow_lossy_cast = 1` " - "to allow lossy casts during export.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); + verifyPartitionKeyColumn( + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + } } + else + { + const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); + for (size_t i = 0; i < num_columns; ++i) + if (partition_key_owner_columns.contains(source_columns[i].name)) + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + } + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (allow_lossy_cast) + return; + + verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + schema_mismatch_mode, + destination_storage_id); } } diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 652ec9223394..303193072284 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include "Storages/IStorage.h" @@ -93,15 +95,23 @@ namespace ExportPartitionUtils const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); + void verifyExportColumnCastsAreSafe( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & destination_columns, + MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, + const StorageID & destination_storage_id); + /// Validates that source columns can be exported into the destination with the - /// same positional CAST matching as `INSERT INTO dest SELECT * FROM src`. Lossy - /// casts are rejected unless `export_merge_tree_part_allow_lossy_cast` is set. + /// configured positional or name-based CAST matching. Lossy casts are rejected + /// unless `export_merge_tree_part_allow_lossy_cast` is set. /// /// By default the source and destination must have the same number of columns. /// If `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, a /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. + /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are + /// matched to source columns by their exact names and may appear in a different order. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index df20755ba590..8186db87d944 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -3,7 +3,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -11,6 +14,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCOMPATIBLE_COLUMNS; + extern const int THERE_IS_NO_COLUMN; +} + namespace Setting { extern const SettingsMergeTreePartExportSchemaMismatchMode export_merge_tree_part_schema_mismatch_mode; @@ -18,6 +27,27 @@ namespace Setting namespace { + template + ColumnWithTypeAndName makeColumn(const String & name) + { + auto type = std::make_shared(); + return {type->createColumn(), type, name}; + } + + template + void expectExceptionCode(Function && function, int expected_code) + { + try + { + function(); + FAIL() << "Expected exception code " << expected_code; + } + catch (const Exception & exception) + { + EXPECT_EQ(exception.code(), expected_code) << exception.message(); + } + } + ExportReplicatedMergeTreePartitionManifest makeValidManifest() { ExportReplicatedMergeTreePartitionManifest manifest; @@ -171,4 +201,84 @@ TEST_F(ExportPartitionManifestBackCompatTest, SchemaMismatchModeAppliedToWorkerC } } +TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + destination_storage_id)); +} + +TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); +} + +TEST(ExportColumnCastsTest, RejectsMissingDestinationColumnAfterMatchingByName) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("extra"), + }; + const ColumnsWithTypeAndName destination_columns = { + makeColumn("renamed_id"), + makeColumn("year"), + }; + + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, + destination_columns, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name, + StorageID{"test", "destination"}); + }, + ErrorCodes::THERE_IS_NO_COLUMN); +} + } diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 34590115d7de..b711b4734d39 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -37,6 +37,12 @@ ) +EXTRA_SOURCE_COLUMN_MODES = [ + pytest.param("ignore_extra_source_columns_by_position", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", id="by-name"), +] + + # --------------------------------------------------------------------------- # Cluster fixture # --------------------------------------------------------------------------- @@ -757,13 +763,8 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster): - """ - Source has 3 columns (id, year, extra), destination has 2 (id, year). - With `export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'`, - the export must succeed: the trailing `extra` source column is dropped - (matched positionally) and only `id`/`year` land in the destination. - """ +@pytest.mark.parametrize("mismatch_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(cluster, mismatch_mode): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_{sfx}" @@ -777,7 +778,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust export_part( node=node, table=mt, part=part_2020, dest=iceberg, - extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'", + extra_settings=f"export_merge_tree_part_schema_mismatch_mode = '{mismatch_mode}'", ) wait_for_export_part(node=node, table=mt, part=part_2020) @@ -793,13 +794,16 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting(cluster): - """ - `ignore_extra_source_columns_by_position` only relaxes the source-has-more-columns - direction. Source has 2 columns (id, year), destination has 3 (id, year, extra): - the destination cannot be filled from the source, so this must still be - rejected synchronously even with the relaxed setting. - """ +@pytest.mark.parametrize( + "mismatch_mode,expected_error", + [ + pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), + pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + ], +) +def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( + cluster, mismatch_mode, expected_error +): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_ignore_extra_fewer_{sfx}" @@ -815,12 +819,9 @@ def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_igno f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " f"SETTINGS allow_experimental_export_merge_tree_part = 1, " f"allow_experimental_insert_into_iceberg = 1, " - f"export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_position'" - ) - assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source Date: Mon, 17 Aug 2026 14:01:21 +0200 Subject: [PATCH 2/5] remove comments --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index ef0aa3d6fc4f..86dbda30abb5 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -868,10 +868,6 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - /// Partition-key columns must keep a stable identity across the conversion, regardless of - /// `allow_lossy_cast`, so that `PARTITION BY` keeps selecting the same values after export. - /// `ignore_extra_source_columns_by_name` matches columns by name (see `makeConvertingActions` - /// above), so look the source column up by name instead of assuming the same position. if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; @@ -886,7 +882,7 @@ namespace ExportPartitionUtils const auto source_it = source_positions_by_name.find(destination_column.name); if (source_it == source_positions_by_name.end()) - continue; /// Reported as THERE_IS_NO_COLUMN by verifyExportColumnCastsAreSafe below. + continue; verifyPartitionKeyColumn( source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); From ee83ddd9876a71e9693f7e145341d4bb64cab75b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 10:59:46 +0200 Subject: [PATCH 3/5] update text Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 16 ++++++++++------ src/Storages/MergeTree/MergeTreeData.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 1 - 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 86dbda30abb5..992c0c817ca2 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -710,19 +710,21 @@ namespace ExportPartitionUtils const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, size_t position, - const StorageID & destination_storage_id) + const StorageID & destination_storage_id, + bool match_by_name) { if (source_column.name != destination_column.name) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot export to {}: partition key column '{}' is at position {} in the source " "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", + "PART/PARTITION {} so partition key columns must be declared {} in both tables.", destination_storage_id.getFullTableName(), source_column.name, position, - destination_column.name); + destination_column.name, + match_by_name ? "matches columns by name" : "matches columns by position", + match_by_name ? "with the same name" : "at the same position"); if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) throw Exception( @@ -885,7 +887,8 @@ namespace ExportPartitionUtils continue; verifyPartitionKeyColumn( - source_columns[source_it->second], destination_column, source_it->second, destination_storage_id); + source_columns[source_it->second], destination_column, source_it->second, destination_storage_id, + /*match_by_name=*/ true); } } else @@ -893,7 +896,8 @@ namespace ExportPartitionUtils const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) if (partition_key_owner_columns.contains(source_columns[i].name)) - verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id); + verifyPartitionKeyColumn(source_columns[i], destination_columns[i], i, destination_storage_id, + /*match_by_name=*/ false); } /// Lossy casts may silently change values, so reject them unless the user opts in. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0519730a5ee8..dfc49f1bf7bb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6782,7 +6782,6 @@ void MergeTreeData::exportPartToTable( #endif } - /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. ExportPartitionUtils::verifyExportSchemaCastable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9d40c5b35df1..ed4a82c69d7f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8412,7 +8412,6 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & auto src_snapshot = getInMemoryMetadataPtr(); auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(); - /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. ExportPartitionUtils::verifyExportSchemaCastable( src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); From a89b0196c74aa3a42f972fcff1348b163ef3d512 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:15:12 +0200 Subject: [PATCH 4/5] by_name only for extra columns Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 6 +-- docs/en/antalya/partition_export.md | 6 +-- src/Core/Settings.cpp | 2 +- src/Storages/MergeTree/ExportPartTask.cpp | 9 +++- .../MergeTree/ExportPartitionUtils.cpp | 16 +++++-- src/Storages/MergeTree/ExportPartitionUtils.h | 6 ++- .../tests/gtest_export_partition_ordering.cpp | 45 +++++++++++++++++++ .../test.py | 2 +- .../test.py | 2 +- .../test.py | 2 +- 10 files changed, 78 insertions(+), 18 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index fc4cefdafb51..10b15b79b9c8 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -143,7 +143,7 @@ In case a table function is used as the destination, the schema can be omitted a - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -151,8 +151,8 @@ In case a table function is used as the destination, the schema can be omitted a - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index a57f38ba6ac9..934e61418c36 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -135,7 +135,7 @@ Notes: - **Description**: Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` table to have more columns than the destination table and how destination columns are matched in that case. Possible values: - `strict` - columns are matched positionally, like `INSERT INTO dest SELECT * FROM src`, and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. - - `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. Every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and additional source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. + - `ignore_extra_source_columns_by_name` - only takes effect when the source has more columns than the destination. In that case, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position. Unmatched source columns are not exported. If a destination column is absent from the source, including when it was renamed, the export throws `THERE_IS_NO_COLUMN`; there is no positional fallback. When the source does **not** have more columns than the destination, this mode behaves exactly like `strict` - columns are matched positionally, and a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. Use `ignore_extra_source_columns_by_position` when renamed columns should be matched by position. Extra source columns are still read and evaluated (including `MATERIALIZED`/`ALIAS` columns, and any column another kept column's `ALIAS`/`MATERIALIZED` expression depends on) before being dropped, so this setting only changes which columns end up in the destination, not what is computed while reading the part. Type conversion and `export_merge_tree_part_allow_lossy_cast` are applied after columns are matched. @@ -143,8 +143,8 @@ Notes: - In `strict` mode, a different number of source and destination columns throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - In `ignore_extra_source_columns_by_position` mode, the destination having more columns than the source throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - - In `ignore_extra_source_columns_by_name` mode, a destination column which is absent from the source throws `THERE_IS_NO_COLUMN`. - - Renaming a destination column in `ignore_extra_source_columns_by_name` mode throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching. + - In `ignore_extra_source_columns_by_name` mode, when the source has more columns than the destination, a destination column absent from the source (including a renamed one) throws `THERE_IS_NO_COLUMN`; the mode does not fall back to positional matching in that case. + - In `ignore_extra_source_columns_by_name` mode, when the source does not have more columns than the destination, a column-count mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`, same as `strict`. - After columns have been matched successfully, a cast rejected by the export type-safety check throws `INCOMPATIBLE_COLUMNS`. ## Examples diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 52ce73c6bf24..c9f364eeaf51 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7632,7 +7632,7 @@ Controls whether `EXPORT PART`/`EXPORT PARTITION` allows the source `MergeTree` Possible values: - `strict` (default) - columns are matched positionally and the source and destination must have the same number of columns. A mismatch in either direction throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. - `ignore_extra_source_columns_by_position` - the source may have more columns than the destination. The extra trailing source columns (by position) are dropped and not exported. The destination having more columns than the source is still rejected in this mode. -- `ignore_extra_source_columns_by_name` - every destination column is matched to a source column with the same exact, case-sensitive name. Destination columns may be reordered and additional source columns may occur in any position. A destination column absent from the source throws `THERE_IS_NO_COLUMN`; there is no positional fallback. +- `ignore_extra_source_columns_by_name` - the source may have more columns than the destination. If it does, every destination column is matched to a source column with the same exact, case-sensitive name, so destination columns may be reordered and the extra source columns may occur in any position; a destination column absent from the source throws `THERE_IS_NO_COLUMN`, with no positional fallback. When the source does not have more columns than the destination, this mode behaves exactly like `strict`: columns are matched positionally, and a mismatch throws `NUMBER_OF_COLUMNS_DOESNT_MATCH`. )", 0) \ \ /* ####################################################### */ \ diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 8aa3511afad4..8f33e48646cb 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -128,8 +128,9 @@ namespace schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; auto source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartTask"), "Source has {} columns while destination has {} columns, " @@ -158,10 +159,14 @@ namespace source_columns = plan_for_part.getCurrentHeader()->getColumnsWithTypeAndName(); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + auto dag = ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, local_context); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 992c0c817ca2..1f817afc7d1f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -776,7 +776,10 @@ namespace ExportPartitionUtils MergeTreePartExportSchemaMismatchMode schema_mismatch_mode, const StorageID & destination_storage_id) { - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); + + if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns) { std::unordered_map source_columns_by_name; source_columns_by_name.reserve(source_columns.size()); @@ -834,10 +837,11 @@ namespace ExportPartitionUtils /// `makeConvertingActions`, in both modes. const auto schema_mismatch_mode = context->getSettingsRef()[Setting::export_merge_tree_part_schema_mismatch_mode].value; + const bool src_has_extra_columns = source_columns.size() > destination_columns.size(); const bool ignore_extra_source_columns_by_position = schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position; - if (ignore_extra_source_columns_by_position && source_columns.size() > destination_columns.size()) + if (ignore_extra_source_columns_by_position && src_has_extra_columns) { LOG_DEBUG(getLogger("ExportPartitionUtils"), "Source has {} columns while destination has {} columns, " @@ -848,10 +852,14 @@ namespace ExportPartitionUtils source_columns.resize(destination_columns.size()); } + const bool ignore_extra_source_columns_by_name = + schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + && src_has_extra_columns; + (void) ActionsDAG::makeConvertingActions( source_columns, destination_columns, - schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name + ignore_extra_source_columns_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, context); @@ -870,7 +878,7 @@ namespace ExportPartitionUtils const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; - if (schema_mismatch_mode == MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name) + if (ignore_extra_source_columns_by_name) { std::unordered_map source_positions_by_name; source_positions_by_name.reserve(source_columns.size()); diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 303193072284..4b8a484f7dc9 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -110,8 +110,10 @@ namespace ExportPartitionUtils /// source with more columns than the destination is allowed: the extra trailing /// source columns (by position) are excluded from the comparison here, matching /// what `ExportPartTask::addExportConvertingActions` drops from the actual data. - /// If the mode is `ignore_extra_source_columns_by_name`, destination columns are - /// matched to source columns by their exact names and may appear in a different order. + /// If the mode is `ignore_extra_source_columns_by_name` and the source has more columns + /// than the destination, destination columns are matched to source columns by their exact + /// names and may appear in a different order. With an equal (or smaller) column count this + /// mode has no extra source columns to ignore and behaves exactly like `strict`. /// /// Throws BAD_ARGUMENTS on any violation. void verifyExportSchemaCastable( diff --git a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp index 8186db87d944..0dde781e4871 100644 --- a/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp +++ b/src/Storages/MergeTree/tests/gtest_export_partition_ordering.cpp @@ -207,6 +207,7 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) makeColumn("id"), makeColumn("year"), makeColumn("payload"), + makeColumn("extra"), }; const ColumnsWithTypeAndName destination_columns = { makeColumn("payload"), @@ -233,6 +234,50 @@ TEST(ExportColumnCastsTest, UsesSelectedMatchingMode) destination_storage_id)); } +TEST(ExportColumnCastsTest, MatchingByNameNoOpWithoutExtraSourceColumns) +{ + const ColumnsWithTypeAndName source_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + const ColumnsWithTypeAndName reordered_destination_columns = { + makeColumn("payload"), + makeColumn("year"), + makeColumn("id"), + }; + const StorageID destination_storage_id{"test", "destination"}; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + expectExceptionCode( + [&] + { + ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, reordered_destination_columns, mode, destination_storage_id); + }, + ErrorCodes::INCOMPATIBLE_COLUMNS); + } + + const ColumnsWithTypeAndName same_order_destination_columns = { + makeColumn("id"), + makeColumn("year"), + makeColumn("payload"), + }; + + for (const auto mode : + {MergeTreePartExportSchemaMismatchMode::strict, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_position, + MergeTreePartExportSchemaMismatchMode::ignore_extra_source_columns_by_name}) + { + EXPECT_NO_THROW(ExportPartitionUtils::verifyExportColumnCastsAreSafe( + source_columns, same_order_destination_columns, mode, destination_storage_id)); + } +} + TEST(ExportColumnCastsTest, RejectsLossyCastAfterMatchingByName) { const ColumnsWithTypeAndName source_columns = { diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index b711b4734d39..c3a193edbd80 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -798,7 +798,7 @@ def test_export_part_source_more_columns_allowed_with_ignore_extra_setting(clust "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 324521561616..24fe43c2806f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1040,7 +1040,7 @@ def test_export_part_column_count_mismatch_source_fewer_is_rejected(cluster): "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_part_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 6f40f2e61373..dcccaef66009 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1371,7 +1371,7 @@ def test_export_partition_source_more_columns_allowed_with_ignore_extra_setting( "mismatch_mode,expected_error", [ pytest.param("ignore_extra_source_columns_by_position", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-position"), - pytest.param("ignore_extra_source_columns_by_name", "THERE_IS_NO_COLUMN", id="by-name"), + pytest.param("ignore_extra_source_columns_by_name", "NUMBER_OF_COLUMNS_DOESNT_MATCH", id="by-name"), ], ) def test_export_partition_column_count_mismatch_source_fewer_still_rejected_with_ignore_extra_setting( From 374fd8df359fbbf5b11581a579c93ea8d0893894 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 18 Aug 2026 13:19:43 +0200 Subject: [PATCH 5/5] update tests Signed-off-by: Konstantin Morozov --- .../test.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index c3a193edbd80..157832f56547 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1101,27 +1101,16 @@ def test_export_part_reordered_subset_requires_matching_by_name(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -@pytest.mark.parametrize( - "source_columns,values", - [ - pytest.param("id Int32, year Int32", "(1, 2020), (2, 2020)", id="same-column-count"), - pytest.param( - "id Int32, year Int32, extra String", - "(1, 2020, 'first'), (2, 2020, 'second')", - id="extra-source-column", - ), - ], -) -def test_export_part_match_by_name_requires_every_destination_column(cluster, source_columns, values): +def test_export_part_match_by_name_requires_every_destination_column(cluster): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_match_by_name_missing_{sfx}" iceberg = f"iceberg_match_by_name_missing_{sfx}" - make_mt(node, mt, source_columns, "year") + make_mt(node, mt, "id Int32, year Int32, extra String", "year") make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") - node.query(f"INSERT INTO {mt} VALUES {values}") + node.query(f"INSERT INTO {mt} VALUES (1, 2020, 'first'), (2, 2020, 'second')") part_2020 = get_part(node, mt, "2020") error = node.query_and_get_error( @@ -1140,6 +1129,32 @@ def test_export_part_match_by_name_requires_every_destination_column(cluster, so node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_match_by_name_falls_back_to_positional_without_extra_source_columns(cluster): + # No extra source column here, so `ignore_extra_source_columns_by_name` behaves like `strict`. + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_match_by_name_fallback_{sfx}" + iceberg = f"iceberg_match_by_name_fallback_{sfx}" + + make_mt(node, mt, "id Int32, year Int32", "year") + make_iceberg_s3(node, iceberg, "renamed_id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") + part_2020 = get_part(node, mt, "2020") + + export_part( + node, mt, part_2020, iceberg, + extra_settings="export_merge_tree_part_schema_mismatch_mode = 'ignore_extra_source_columns_by_name'", + ) + wait_for_export_part(node, mt, part_2020) + + result = node.query(f"SELECT renamed_id, year FROM {iceberg} ORDER BY renamed_id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data:\n{result}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_castable_widening(cluster): """ Source column is Int32, destination expects Int64. makeConvertingActions