diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 03266539c4dc8e..1dde4ae40eb2dc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1719,6 +1719,11 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB /** Iceberg sink configurations **/ DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB +/** Paimon sink configurations **/ +DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB +DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, + [](int64_t bytes) -> bool { return bytes > 0; }); + // URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. // Each entry uses the format "=", and file_type must be one of: // local, hdfs, s3, http, broker. diff --git a/be/src/common/config.h b/be/src/common/config.h index 436a1878ef424a..a4478bbc76356f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1785,6 +1785,10 @@ DECLARE_mInt64(hive_sink_max_file_size); /** Iceberg sink configurations **/ DECLARE_mInt64(iceberg_sink_max_file_size); +/** Paimon sink configurations **/ +// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. +DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); + /** Paimon file system configurations **/ DECLARE_Strings(paimon_file_system_scheme_mappings); diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp index e5926b2470aef1..a00735457e6be2 100644 --- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp @@ -18,6 +18,7 @@ #include "core/data_type_serde/data_type_variant_v2_serde.h" #include +#include #include #include @@ -182,6 +183,133 @@ void preflight_json(const IColumn& column, size_t start, size_t end, }); } +void validate_paimon_variant_primitive(VariantPrimitiveId primitive_id) { + switch (primitive_id) { + case VariantPrimitiveId::NULL_VALUE: + case VariantPrimitiveId::TRUE_VALUE: + case VariantPrimitiveId::FALSE_VALUE: + case VariantPrimitiveId::INT8: + case VariantPrimitiveId::INT16: + case VariantPrimitiveId::INT32: + case VariantPrimitiveId::INT64: + case VariantPrimitiveId::DOUBLE: + case VariantPrimitiveId::DECIMAL4: + case VariantPrimitiveId::DECIMAL8: + case VariantPrimitiveId::DECIMAL16: + case VariantPrimitiveId::DATE: + case VariantPrimitiveId::TIMESTAMP_MICROS: + case VariantPrimitiveId::TIMESTAMP_NTZ_MICROS: + case VariantPrimitiveId::FLOAT: + case VariantPrimitiveId::BINARY: + case VariantPrimitiveId::STRING: + case VariantPrimitiveId::UUID: + return; + case VariantPrimitiveId::TIME_NTZ_MICROS: + case VariantPrimitiveId::TIMESTAMP_NANOS: + case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS: + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "Paimon does not support Variant primitive id {}", + static_cast(primitive_id)); + } + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "Paimon does not support unknown Variant primitive id {}", + static_cast(primitive_id)); +} + +void validate_paimon_variant_value(VariantRef value, uint32_t depth = 0) { + if (depth > VARIANT_MAX_NESTING_DEPTH) { + throw Exception(ErrorCode::CORRUPTION, "Variant value exceeds maximum nesting depth {}", + VARIANT_MAX_NESTING_DEPTH); + } + const size_t encoded_size = value.value_size(); + if (encoded_size != value.value.size) { + throw Exception(ErrorCode::CORRUPTION, + "Variant value has {} trailing bytes after the encoded value", + value.value.size - encoded_size); + } + + switch (value.basic_type()) { + case VariantBasicType::PRIMITIVE: + validate_paimon_variant_primitive(value.primitive_id()); + return; + case VariantBasicType::SHORT_STRING: + return; + case VariantBasicType::OBJECT: + for (uint32_t i = 0; i < value.num_elements(); ++i) { + uint32_t field_id = 0; + VariantRef child = value.object_value_at(i, &field_id); + value.metadata.key_at(field_id); + validate_paimon_variant_value(child, depth + 1); + } + return; + case VariantBasicType::ARRAY: + for (uint32_t i = 0; i < value.num_elements(); ++i) { + validate_paimon_variant_value(value.array_at(i), depth + 1); + } + return; + } +} + +void require_variant_arrow_status(const arrow::Status& status) { + if (!status.ok()) { + throw Exception(ErrorCode::INTERNAL_ERROR, "Variant V2 Arrow append failed: {}", + status.ToString()); + } +} + +Status write_binary_variant_arrow(const IColumn& column, const NullMap* null_map, + arrow::StructBuilder& builder, size_t start, size_t end) { + // StructBuilder::type() returns a shared_ptr by value. Keep that owner alive while using the + // cast reference; otherwise the reference would dangle as soon as the temporary is destroyed. + const auto builder_type = builder.type(); + const auto& struct_type = assert_cast(*builder_type); + if (struct_type.num_fields() != 2 || struct_type.field(0)->name() != "value" || + struct_type.field(1)->name() != "metadata" || + struct_type.field(0)->type()->id() != arrow::Type::BINARY || + struct_type.field(1)->type()->id() != arrow::Type::BINARY) { + return Status::InvalidArgument( + "Binary Variant V2 Arrow type must be " + "struct, got {}", + struct_type.ToString()); + } + auto* value_builder = dynamic_cast(builder.field_builder(0)); + auto* metadata_builder = dynamic_cast(builder.field_builder(1)); + if (value_builder == nullptr || metadata_builder == nullptr) { + return Status::InvalidArgument("Binary Variant V2 Arrow child builders must be binary"); + } + + // GenericVariant assumes its input is valid, and Paimon's unshredded writer copies these two + // buffers without inspecting them. Validate once at the Doris-to-Paimon boundary so a write + // cannot commit bytes which Paimon is unable to read later. + const auto outer_nulls = forced_nulls(null_map); + visit_variant_v2_values( + column, start, end, outer_nulls, + [&](size_t) { require_variant_arrow_status(builder.AppendNull()); }, + [&](size_t row, VariantRef value) { + try { + constexpr size_t PAIMON_VARIANT_SIZE_LIMIT = 128 * 1024 * 1024; + if (value.value.size > PAIMON_VARIANT_SIZE_LIMIT || + value.metadata.size > PAIMON_VARIANT_SIZE_LIMIT) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "exceeds the 128 MiB value/metadata limit"); + } + value.metadata.validate(); + validate_paimon_variant_value(value); + } catch (const Exception& e) { + throw Exception(e.code(), "Paimon Variant V2 row {} is incompatible: {}", row, + e.what()); + } + require_variant_arrow_status(builder.Append()); + require_variant_arrow_status( + value_builder->Append(reinterpret_cast(value.value.data), + cast_set(value.value.size))); + require_variant_arrow_status(metadata_builder->Append( + reinterpret_cast(value.metadata.data), + cast_set(value.metadata.size))); + }); + return Status::OK(); +} + } // namespace DataTypeVariantV2SerDe::DataTypeVariantV2SerDe(int nesting_level) : DataTypeSerDe(nesting_level) {} @@ -553,6 +681,11 @@ Status DataTypeVariantV2SerDe::write_column_to_arrow(const IColumn& column, cons assert_cast(*array_builder), first, last, options); } + if (array_builder->type()->id() == arrow::Type::STRUCT) { + return write_binary_variant_arrow(column, null_map, + assert_cast(*array_builder), + first, last); + } return Status::InvalidArgument("Unsupported arrow type for variant column: {}", array_builder->type()->name()); }); diff --git a/be/src/exec/operator/exchange_sink_operator.cpp b/be/src/exec/operator/exchange_sink_operator.cpp index 449c71e3339281..e84ec49a54f655 100644 --- a/be/src/exec/operator/exchange_sink_operator.cpp +++ b/be/src/exec/operator/exchange_sink_operator.cpp @@ -28,6 +28,7 @@ #include #include +#include "agent/be_exec_version_manager.h" #include "common/status.h" #include "core/column/column_const.h" #include "exec/exchange/exchange_writer.h" @@ -35,9 +36,9 @@ #include "exec/operator/exchange_sink_buffer.h" #include "exec/operator/operator.h" #include "exec/operator/sort_source_operator.h" +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" #include "exec/pipeline/dependency.h" #include "exec/pipeline/pipeline_fragment_context.h" -#include "exec/sink/scale_writer_partitioning_exchanger.hpp" #include "exec/sink/tablet_sink_hash_partitioner.h" #include "exprs/vexpr.h" #include "format/transformer/merge_partitioner.h" @@ -98,7 +99,7 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf _part_type = p._part_type; // Shuffle the channels randomly if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM || - _part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) { + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) { std::random_device rd; std::mt19937 g(rd()); shuffle(channels.begin(), channels.end(), g); @@ -153,28 +154,32 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf p._tablet_sink_partition, p._tablet_sink_location, p._tablet_sink_tuple_id, this); RETURN_IF_ERROR(_partitioner->init({})); RETURN_IF_ERROR(_partitioner->prepare(state, {})); - } else if (_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED) { - _partition_count = - channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer; - _partitioner = std::make_unique( - channels.size(), _partition_count, channels.size(), 1, - config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold / - state->task_num() == - 0 - ? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold - : config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold / - state->task_num(), - config::table_sink_partition_write_min_data_processed_rebalance_threshold / - state->task_num() == - 0 - ? config::table_sink_partition_write_min_data_processed_rebalance_threshold - : config::table_sink_partition_write_min_data_processed_rebalance_threshold / - state->task_num()); - + } else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED) { + if (state->be_exec_version() < SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION) { + return Status::NotSupported( + "External table sink hash exchange requires BE execution version {}, actual {}", + SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION, state->be_exec_version()); + } + if (!p._has_external_table_sink_hash_partition_info) { + return Status::InternalError("External table sink hash partition info is missing"); + } + _partition_count = channels.size(); + const bool use_crc32c = _state->query_options().__isset.enable_new_shuffle_hash_method && + _state->query_options().enable_new_shuffle_hash_method; + const ShuffleHashMethod hash_method = + use_crc32c ? ShuffleHashMethod::CRC32C : ShuffleHashMethod::CRC32; + _partitioner = std::make_unique( + _partition_count, hash_method, p._external_table_sink_hash_partition_info); RETURN_IF_ERROR(_partitioner->init(p._texprs)); RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc)); custom_profile()->add_info_string( - "Partitioner", fmt::format("ScaleWriterPartitioner({})", _partition_count)); + "Partitioner", + fmt::format("ExternalTableSinkHashPartitioner({})", _partition_count)); + custom_profile()->add_info_string( + "WriterAssignment", p._external_table_sink_hash_partition_info.writer_assignment == + TExternalTableSinkWriterAssignment::IDENTITY + ? "IDENTITY" + : "SKEWED"); } else if (_part_type == TPartitionType::MERGE_PARTITIONED) { if (!p._has_merge_partition_info) { return Status::InternalError("Merge partition info is missing"); @@ -271,7 +276,7 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) { if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || - _part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::MERGE_PARTITIONED) { RETURN_IF_ERROR(_partitioner->open(state)); @@ -320,8 +325,8 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX( sink.output_partition.type == TPartitionType::RANGE_PARTITIONED || sink.output_partition.type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || sink.output_partition.type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || - sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || - sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED || + sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || + sink.output_partition.type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED || sink.output_partition.type == TPartitionType::MERGE_PARTITIONED); #endif _name = "ExchangeSinkOperatorX"; @@ -333,6 +338,11 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX( _merge_partition_info = sink.output_partition.merge_partition_info; _has_merge_partition_info = true; } + if (sink.output_partition.__isset.external_table_sink_hash_partition_info) { + _external_table_sink_hash_partition_info = + sink.output_partition.external_table_sink_hash_partition_info; + _has_external_table_sink_hash_partition_info = true; + } if (_part_type != TPartitionType::UNPARTITIONED) { // if the destinations only one dest, we need to use broadcast @@ -533,11 +543,11 @@ Status ExchangeSinkOperatorX::sink_impl(RuntimeState* state, Block* block, bool (local_state.current_channel_idx + 1) % local_state.channels.size(); } else if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED || + _part_type == TPartitionType::EXTERNAL_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED || - _part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED || _part_type == TPartitionType::MERGE_PARTITIONED) { RETURN_IF_ERROR(local_state._writer->write(state, block, eos)); - } else if (_part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) { + } else if (_part_type == TPartitionType::EXTERNAL_TABLE_SINK_UNPARTITIONED) { // Control the number of channels according to the flow, thereby controlling the number of table sink writers. RETURN_IF_ERROR(send_to_current_channel()); _data_processed += block->bytes(); diff --git a/be/src/exec/operator/exchange_sink_operator.h b/be/src/exec/operator/exchange_sink_operator.h index 10351154d1d8cd..12911e448043e9 100644 --- a/be/src/exec/operator/exchange_sink_operator.h +++ b/be/src/exec/operator/exchange_sink_operator.h @@ -243,6 +243,8 @@ class ExchangeSinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX _texprs; + TExternalTableSinkHashPartitionInfo _external_table_sink_hash_partition_info; + bool _has_external_table_sink_hash_partition_info = false; TMergePartitionInfo _merge_partition_info; bool _has_merge_partition_info = false; diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index 796cc1169691d9..358e43cdb42d75 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -64,6 +64,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -846,6 +847,7 @@ DECLARE_OPERATOR(OlapTableSinkV2LocalState) DECLARE_OPERATOR(HiveTableSinkLocalState) DECLARE_OPERATOR(TVFTableSinkLocalState) DECLARE_OPERATOR(IcebergTableSinkLocalState) +DECLARE_OPERATOR(PaimonTableSinkLocalState) DECLARE_OPERATOR(SpillIcebergTableSinkLocalState) DECLARE_OPERATOR(IcebergDeleteSinkLocalState) DECLARE_OPERATOR(IcebergMergeSinkLocalState) diff --git a/be/src/exec/operator/paimon_table_sink_operator.cpp b/be/src/exec/operator/paimon_table_sink_operator.cpp new file mode 100644 index 00000000000000..dace179ba2d9fb --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.cpp @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/operator/paimon_table_sink_operator.h" + +#include "common/logging.h" + +namespace doris { + +Status PaimonTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { + RETURN_IF_ERROR(Base::init(state, info)); + _writer = std::make_unique(info.tsink, _output_vexpr_ctxs); + return Status::OK(); +} + +Status PaimonTableSinkLocalState::open(RuntimeState* state) { + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_open_timer); + RETURN_IF_ERROR(Base::open(state)); + + auto& parent = _parent->cast(); + _output_vexpr_ctxs.resize(parent._output_vexpr_ctxs.size()); + for (size_t i = 0; i < _output_vexpr_ctxs.size(); ++i) { + RETURN_IF_ERROR(parent._output_vexpr_ctxs[i]->clone(state, _output_vexpr_ctxs[i])); + } + return _writer->open(state, operator_profile()); +} + +Status PaimonTableSinkLocalState::close(RuntimeState* state, Status exec_status) { + if (_closed) { + return Status::OK(); + } + + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_close_timer); + + Status final_status = exec_status; + if (_writer) { + Status writer_status = _writer->close(exec_status); + if (final_status.ok() && !writer_status.ok()) { + final_status = writer_status; + } + _writer.reset(); + } + + Status base_status = Base::close(state, final_status); + if (final_status.ok() && !base_status.ok()) { + final_status = base_status; + } + return final_status; +} + +Status PaimonTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool /*eos*/) { + auto& local_state = get_local_state(state); + SCOPED_TIMER(local_state.exec_time_counter()); + COUNTER_UPDATE(local_state.rows_input_counter(), static_cast(in_block->rows())); + + if (in_block->rows() == 0) { + return Status::OK(); + } + + // This is a synchronous SDK call. The LocalState is marked blockable, so + // the whole pipeline task (including open and close) runs on the blocking + // scheduler instead of occupying a regular pipeline worker. + DCHECK(local_state._writer); + return local_state._writer->write(state, *in_block); +} + +} // namespace doris diff --git a/be/src/exec/operator/paimon_table_sink_operator.h b/be/src/exec/operator/paimon_table_sink_operator.h new file mode 100644 index 00000000000000..9d07dc45390e24 --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/operator/operator.h" +#include "exec/sink/writer/paimon/paimon_table_writer.h" +#include "runtime/runtime_state.h" + +namespace doris { + +/// Paimon table sink operator. +/// +/// Each pipeline instance (LocalState) owns one PaimonTableWriter, which in +/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism +/// determines the number of concurrent Paimon writer sessions per table. +/// Paimon writes are synchronous: sink_impl() returns only after the SDK has +/// consumed the input Block. The LocalState is therefore always blockable so +/// that open, write, and close run on the pipeline blocking scheduler. +/// Doris-owned Arrow buffers remain under the query MemTracker, while Paimon +/// pages are allocated lazily under DorisMemorySegmentPool's fixed cap. The +/// sink uses the generic pipeline minimum reservation only as an admission +/// guard; it does not try to predict Paimon's future page demand. +/// +/// The upstream sink Exchange may reproduce Paimon's stateless HASH_FIXED +/// selector to establish unique writer ownership. The writer still passes +/// complete Blocks to the SDK, which independently computes partition and +/// bucket values for file writing; no routing column is appended to the row. +class PaimonTableSinkOperatorX; + +class PaimonTableSinkLocalState final : public PipelineXSinkLocalState { +public: + using Base = PipelineXSinkLocalState; + using Parent = PaimonTableSinkOperatorX; + ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState); + PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) + : Base(parent, state) {} + + Status init(RuntimeState* state, LocalSinkStateInfo& info) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state, Status exec_status) override; + + [[nodiscard]] bool is_blockable() const override { return true; } + +private: + friend class PaimonTableSinkOperatorX; + + VExprContextSPtrs _output_vexpr_ctxs; + std::unique_ptr _writer; +}; + +class PaimonTableSinkOperatorX final : public DataSinkOperatorX { +public: + using Base = DataSinkOperatorX; + PaimonTableSinkOperatorX(int operator_id, const RowDescriptor& row_desc, + const std::vector& t_output_expr) + : Base(operator_id, 0, 0), _row_desc(row_desc), _t_output_expr(t_output_expr) {} + + Status init(const TDataSink& thrift_sink) override { + RETURN_IF_ERROR(Base::init(thrift_sink)); + DCHECK(thrift_sink.__isset.paimon_table_sink); + RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); + return Status::OK(); + } + + Status prepare(RuntimeState* state) override { + RETURN_IF_ERROR(Base::prepare(state)); + RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); + return VExpr::open(_output_vexpr_ctxs, state); + } + + Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; + +private: + friend class PaimonTableSinkLocalState; + + const RowDescriptor& _row_desc; + VExprContextSPtrs _output_vexpr_ctxs; + const std::vector& _t_output_expr; +}; + +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_partition_function_factory.cpp b/be/src/exec/partitioner/external/external_partition_function_factory.cpp new file mode 100644 index 00000000000000..457c9974566a61 --- /dev/null +++ b/be/src/exec/partitioner/external/external_partition_function_factory.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/external_partition_function_factory.h" + +#include "common/status.h" +#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h" +#include "format/transformer/iceberg_partition_function.h" + +namespace doris { + +namespace { +bool has_partition_transform_metadata(const TExternalTableSinkHashPartitionInfo& info) { + return info.__isset.partition_transforms; +} + +bool has_paimon_metadata(const TExternalTableSinkHashPartitionInfo& info) { + return info.__isset.paimon_fixed_bucket_info; +} + +Status create_direct_hash_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (has_partition_transform_metadata(info) || has_paimon_metadata(info)) { + return Status::InvalidArgument("Direct external sink hash contains incompatible metadata"); + } + auto function = std::make_unique(logical_partition_count, hash_method); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); +} + +Status create_iceberg_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (has_paimon_metadata(info)) { + return Status::InvalidArgument( + "Iceberg external sink routing contains incompatible Paimon metadata"); + } + if (!info.__isset.partition_transforms) { + return Status::InvalidArgument("Iceberg external sink partition transforms are missing"); + } + if (info.partition_transforms.size() != partition_exprs.size()) { + return Status::InvalidArgument( + "External sink partition transform count {} does not match expression count {}", + info.partition_transforms.size(), partition_exprs.size()); + } + std::vector fields; + fields.reserve(partition_exprs.size()); + for (size_t index = 0; index < partition_exprs.size(); ++index) { + TIcebergPartitionField field; + field.__set_transform(info.partition_transforms[index]); + field.__set_source_expr(partition_exprs[index]); + fields.emplace_back(std::move(field)); + } + auto function = std::make_unique( + logical_partition_count, hash_method, std::vector {}, std::move(fields)); + RETURN_IF_ERROR(function->init({})); + *partition_function = std::move(function); + return Status::OK(); +} + +Status create_paimon_fixed_bucket_function(const TExternalTableSinkHashPartitionInfo& info, + PartitionerBase::HashValType logical_partition_count, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (!info.__isset.paimon_fixed_bucket_info) { + return Status::InvalidArgument("Paimon fixed-bucket routing metadata is missing"); + } + if (has_partition_transform_metadata(info)) { + return Status::InvalidArgument( + "Paimon fixed-bucket routing contains incompatible metadata"); + } + auto function = std::make_unique( + logical_partition_count, info.paimon_fixed_bucket_info); + RETURN_IF_ERROR(function->init(partition_exprs)); + *partition_function = std::move(function); + return Status::OK(); +} + +} // namespace + +Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function) { + if (partition_function == nullptr) { + return Status::InvalidArgument("External partition function output is null"); + } + switch (partition_info.algorithm) { + case TExternalTableSinkHashAlgorithm::DIRECT_HASH: + return create_direct_hash_function(partition_info, logical_partition_count, hash_method, + partition_exprs, partition_function); + case TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM: + return create_iceberg_function(partition_info, logical_partition_count, hash_method, + partition_exprs, partition_function); + case TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET: + return create_paimon_fixed_bucket_function(partition_info, logical_partition_count, + partition_exprs, partition_function); + default: + return Status::InvalidArgument("Unsupported external sink hash algorithm {}", + static_cast(partition_info.algorithm)); + } +} + +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_partition_function_factory.h b/be/src/exec/partitioner/external/external_partition_function_factory.h new file mode 100644 index 00000000000000..07528fab163989 --- /dev/null +++ b/be/src/exec/partitioner/external/external_partition_function_factory.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +#include "exec/partitioner/partitioner.h" + +namespace doris { + +// Validates connector-specific routing metadata and creates the matching logical partition +// function. Writer assignment remains the responsibility of ExternalTableSinkHashPartitioner. +Status create_external_partition_function(const TExternalTableSinkHashPartitionInfo& partition_info, + PartitionerBase::HashValType logical_partition_count, + ShuffleHashMethod hash_method, + const std::vector& partition_exprs, + std::unique_ptr* partition_function); + +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp new file mode 100644 index 00000000000000..74d4f9ccd22785 --- /dev/null +++ b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.cpp @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "exec/partitioner/external/external_partition_function_factory.h" + +namespace doris { + +namespace { +Status logical_partition_count(uint32_t writer_count, + TExternalTableSinkWriterAssignment::type assignment, + uint32_t* result) { + if (writer_count == 0) { + return Status::InvalidArgument("External sink writer count must be positive"); + } + if (assignment != TExternalTableSinkWriterAssignment::SKEWED) { + *result = writer_count; + return Status::OK(); + } + const uint32_t partitions_per_writer = static_cast( + std::max(1, config::table_sink_partition_write_max_partition_nums_per_writer)); + if (writer_count > std::numeric_limits::max() / partitions_per_writer) { + return Status::InvalidArgument("External sink logical partition count overflows"); + } + *result = writer_count * partitions_per_writer; + return Status::OK(); +} +} // namespace + +ExternalTableSinkHashPartitioner::ExternalTableSinkHashPartitioner( + HashValType partition_count, ShuffleHashMethod hash_method, + TExternalTableSinkHashPartitionInfo partition_info) + : PartitionerBase(partition_count), + _hash_method(hash_method), + _partition_info(std::move(partition_info)), + _logical_partition_count(partition_count) {} + +Status ExternalTableSinkHashPartitioner::init(const std::vector& texprs) { + if (!_partition_info.__isset.writer_assignment) { + return Status::InvalidArgument("External sink writer assignment is missing"); + } + if (_partition_info.writer_assignment != TExternalTableSinkWriterAssignment::IDENTITY && + _partition_info.writer_assignment != TExternalTableSinkWriterAssignment::SKEWED) { + return Status::InvalidArgument("Unsupported external sink writer assignment {}", + static_cast(_partition_info.writer_assignment)); + } + RETURN_IF_ERROR(logical_partition_count(_partition_count, _partition_info.writer_assignment, + &_logical_partition_count)); + const bool requires_identity = + _partition_info.algorithm == TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET; + if (requires_identity && + _partition_info.writer_assignment != TExternalTableSinkWriterAssignment::IDENTITY) { + return Status::InvalidArgument("Paimon bucket routing requires identity writer assignment"); + } + return create_external_partition_function(_partition_info, _logical_partition_count, + _hash_method, texprs, &_partition_function); +} + +Status ExternalTableSinkHashPartitioner::prepare(RuntimeState* state, + const RowDescriptor& row_desc) { + return _partition_function->prepare(state, row_desc); +} + +Status ExternalTableSinkHashPartitioner::open(RuntimeState* state) { + RETURN_IF_ERROR(_partition_function->open(state)); + + if (_partition_info.writer_assignment == TExternalTableSinkWriterAssignment::IDENTITY) { + _writer_assigner = std::make_unique(_partition_count); + } else { + const int task_num = state == nullptr ? 0 : state->task_num(); + _writer_assigner = std::make_unique( + cast_set(_logical_partition_count), cast_set(_partition_count), 1, + scale_writer_threshold_by_task( + config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold, + task_num), + scale_writer_threshold_by_task( + config::table_sink_partition_write_min_data_processed_rebalance_threshold, + task_num)); + } + return Status::OK(); +} + +Status ExternalTableSinkHashPartitioner::close(RuntimeState* state) { + return _partition_function->close(state); +} + +Status ExternalTableSinkHashPartitioner::do_partitioning(RuntimeState* state, Block* block) const { + if (_writer_assigner == nullptr) { + return Status::InternalError("External sink writer assigner is not open"); + } + const size_t rows = block->rows(); + const size_t block_bytes = block->bytes(); + if (rows == 0) { + _logical_partition_ids.clear(); + _channel_ids.clear(); + return Status::OK(); + } + RETURN_IF_ERROR(_partition_function->get_partitions(state, block, _logical_partition_count, + _logical_partition_ids)); + + return _writer_assigner->assign(_logical_partition_ids, nullptr, rows, block_bytes, + _channel_ids); +} + +const std::vector& +ExternalTableSinkHashPartitioner::get_channel_ids() const { + return _channel_ids; +} + +Status ExternalTableSinkHashPartitioner::clone(RuntimeState* state, + std::unique_ptr& partitioner) { + auto cloned = std::make_unique(_partition_count, _hash_method, + _partition_info); + RETURN_IF_ERROR(_partition_function->clone(state, cloned->_partition_function)); + partitioner = std::move(cloned); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h new file mode 100644 index 00000000000000..c627b756bf4b19 --- /dev/null +++ b/be/src/exec/partitioner/external/external_table_sink_hash_partitioner.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "exec/partitioner/partitioner.h" +#include "exec/partitioner/writer_assigner.h" + +namespace doris { + +// Computes external sink logical partitions and maps them to Doris exchange channels. +// Optional partition transforms are evaluated transiently and never appended to the sink row. +class ExternalTableSinkHashPartitioner final : public PartitionerBase { +public: + ExternalTableSinkHashPartitioner(HashValType partition_count, ShuffleHashMethod hash_method, + TExternalTableSinkHashPartitionInfo partition_info); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override; + Status do_partitioning(RuntimeState* state, Block* block) const override; + const std::vector& get_channel_ids() const override; + Status clone(RuntimeState* state, std::unique_ptr& partitioner) override; + +private: + ShuffleHashMethod _hash_method; + TExternalTableSinkHashPartitionInfo _partition_info; + HashValType _logical_partition_count; + std::unique_ptr _partition_function; + mutable std::unique_ptr _writer_assigner; + mutable std::vector _logical_partition_ids; + mutable std::vector _channel_ids; +}; + +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp new file mode 100644 index 00000000000000..ee3df55f0e0c80 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.cpp @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/paimon_fixed_bucket_partition_function.h" + +#include "common/status.h" +#include "exec/partitioner/external/paimon_native_row_hash.h" + +namespace doris { + +PaimonFixedBucketPartitionFunction::PaimonFixedBucketPartitionFunction( + HashValType partition_count, TPaimonFixedBucketInfo fixed_bucket_info) + : PaimonRowHashPartitionFunction(partition_count), + _fixed_bucket_info(std::move(fixed_bucket_info)) {} + +Status PaimonFixedBucketPartitionFunction::init(const std::vector& texprs) { + RETURN_IF_ERROR(PaimonRowHashPartitionFunction::init(texprs)); + if (_fixed_bucket_info.num_buckets <= 0) { + return Status::InvalidArgument("Paimon fixed-bucket count must be positive"); + } + RETURN_IF_ERROR(_validate_field_indexes(_fixed_bucket_info.partition_field_indexes, false)); + return _validate_field_indexes(_fixed_bucket_info.bucket_field_indexes, true); +} + +Status PaimonFixedBucketPartitionFunction::get_partitions( + RuntimeState* /*state*/, Block* block, size_t partition_count, + std::vector& partitions) const { + if (partition_count != _partition_count) { + return Status::InvalidArgument("Paimon writer count {} does not match planned count {}", + partition_count, _partition_count); + } + const size_t rows = block->rows(); + if (rows == 0) { + partitions.clear(); + return Status::OK(); + } + + std::vector fields; + RETURN_IF_ERROR(_evaluate_fields(block, fields)); + std::vector partition_hashes; + std::vector bucket_hashes; + RETURN_IF_ERROR( + _hash_fields(_fixed_bucket_info.partition_field_indexes, fields, partition_hashes)); + RETURN_IF_ERROR(_hash_fields(_fixed_bucket_info.bucket_field_indexes, fields, bucket_hashes)); + partitions.resize(rows); + for (size_t row = 0; row < rows; ++row) { + auto bucket = + paimon_native::default_bucket(bucket_hashes[row], _fixed_bucket_info.num_buckets); + if (!bucket.has_value()) { + return Status::InternalError("Failed to compute Paimon fixed bucket"); + } + auto channel = paimon_native::fixed_bucket_channel(partition_hashes[row], *bucket, + _partition_count); + if (!channel.has_value()) { + return Status::InternalError("Failed to compute Paimon fixed-bucket writer"); + } + partitions[row] = *channel; + } + return Status::OK(); +} + +Status PaimonFixedBucketPartitionFunction::clone( + RuntimeState* state, std::unique_ptr& function) const { + auto cloned = std::make_unique(_partition_count, + _fixed_bucket_info); + RETURN_IF_ERROR(_clone_expr_ctxs(state, cloned->_field_expr_ctxs)); + function = std::move(cloned); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h new file mode 100644 index 00000000000000..a6f5e71c385579 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_fixed_bucket_partition_function.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "exec/partitioner/external/paimon_row_hash_partition_function.h" + +namespace doris { + +// Stateless native implementation of Paimon FixedBucketWriteSelector for the +// explicitly supported primitive routing types. +class PaimonFixedBucketPartitionFunction final : public PaimonRowHashPartitionFunction { +public: + PaimonFixedBucketPartitionFunction(HashValType partition_count, + TPaimonFixedBucketInfo fixed_bucket_info); + + Status init(const std::vector& texprs) override; + Status get_partitions(RuntimeState* state, Block* block, size_t partition_count, + std::vector& partitions) const override; + Status clone(RuntimeState* state, std::unique_ptr& function) const override; + +private: + TPaimonFixedBucketInfo _fixed_bucket_info; +}; + +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_native_row_hash.cpp b/be/src/exec/partitioner/external/paimon_native_row_hash.cpp new file mode 100644 index 00000000000000..a702fdfc219a38 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_native_row_hash.cpp @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/paimon_native_row_hash.h" + +#include +#include +#include +#include + +namespace doris::paimon_native { +namespace { + +constexpr uint32_t MURMUR_C1 = 0xcc9e2d51U; +constexpr uint32_t MURMUR_C2 = 0x1b873593U; +constexpr uint32_t MURMUR_SEED = 42U; + +uint32_t rotate_left(uint32_t value, int distance) { + return (value << distance) | (value >> (32 - distance)); +} + +uint32_t mix_k1(uint32_t value) { + value *= MURMUR_C1; + value = rotate_left(value, 15); + value *= MURMUR_C2; + return value; +} + +uint32_t mix_h1(uint32_t hash, uint32_t value) { + hash ^= value; + hash = rotate_left(hash, 13); + return hash * 5U + 0xe6546b64U; +} + +uint32_t fmix(uint32_t hash, size_t length) { + hash ^= static_cast(length); + hash ^= hash >> 16; + hash *= 0x85ebca6bU; + hash ^= hash >> 13; + hash *= 0xc2b2ae35U; + hash ^= hash >> 16; + return hash; +} + +template +void put_native(std::vector* bytes, size_t offset, T value) { + std::memcpy(bytes->data() + offset, &value, sizeof(value)); +} + +size_t round_to_word(size_t size) { + return (size + 7U) & ~size_t {7U}; +} + +} // namespace + +BinaryRowEncoder::BinaryRowEncoder(size_t arity) + : _arity(arity), + _null_bits_size((arity + 63U + 8U) / 64U * 8U), + _fixed_size(_null_bits_size + arity * 8U), + _cursor(_fixed_size), + _bytes(_fixed_size, 0) {} + +void BinaryRowEncoder::reset() { + _cursor = _fixed_size; + _bytes.assign(_fixed_size, 0); +} + +bool BinaryRowEncoder::_valid_position(size_t position) const { + return position < _arity; +} + +size_t BinaryRowEncoder::_field_offset(size_t position) const { + return _null_bits_size + position * 8U; +} + +void BinaryRowEncoder::_set_null_bit(size_t position) { + const size_t bit_index = position + 8U; + _bytes[bit_index >> 3U] |= static_cast(1U << (bit_index & 7U)); +} + +void BinaryRowEncoder::_ensure_capacity(size_t size) { + if (_bytes.size() < size) { + _bytes.resize(size, 0); + } +} + +void BinaryRowEncoder::_set_offset_and_size(size_t position, uint32_t offset, uint32_t size) { + uint64_t offset_and_size = (static_cast(offset) << 32U) | size; + put_native(&_bytes, _field_offset(position), offset_and_size); +} + +bool BinaryRowEncoder::set_null(size_t position) { + if (!_valid_position(position)) { + return false; + } + _set_null_bit(position); + std::fill_n(_bytes.begin() + _field_offset(position), 8, uint8_t {0}); + return true; +} + +bool BinaryRowEncoder::write_boolean(size_t position, bool value) { + return write_tinyint(position, value ? 1 : 0); +} + +bool BinaryRowEncoder::write_tinyint(size_t position, int8_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_smallint(size_t position, int16_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_int(size_t position, int32_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_bigint(size_t position, int64_t value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_float(size_t position, float value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::write_double(size_t position, double value) { + if (!_valid_position(position)) { + return false; + } + put_native(&_bytes, _field_offset(position), value); + return true; +} + +bool BinaryRowEncoder::_write_bytes(size_t position, std::string_view bytes) { + if (!_valid_position(position) || bytes.size() > std::numeric_limits::max()) { + return false; + } + if (bytes.size() <= 7U) { + uint64_t inline_value = (static_cast(bytes.size()) | 0x80U) << 56U; + for (size_t index = 0; index < bytes.size(); ++index) { + if constexpr (std::endian::native == std::endian::little) { + inline_value |= static_cast(static_cast(bytes[index])) + << (index * 8U); + } else { + inline_value |= static_cast(static_cast(bytes[index])) + << ((6U - index) * 8U); + } + } + put_native(&_bytes, _field_offset(position), inline_value); + return true; + } + + const size_t rounded_size = round_to_word(bytes.size()); + if (rounded_size > std::numeric_limits::max() || + _cursor > std::numeric_limits::max() - rounded_size) { + return false; + } + _ensure_capacity(_cursor + rounded_size); + std::fill_n(_bytes.begin() + _cursor, rounded_size, uint8_t {0}); + std::memcpy(_bytes.data() + _cursor, bytes.data(), bytes.size()); + _set_offset_and_size(position, static_cast(_cursor), + static_cast(bytes.size())); + _cursor += rounded_size; + return true; +} + +bool BinaryRowEncoder::write_string(size_t position, std::string_view utf8) { + return _write_bytes(position, utf8); +} + +bool BinaryRowEncoder::write_binary(size_t position, std::string_view bytes) { + return _write_bytes(position, bytes); +} + +int32_t BinaryRowEncoder::hash() const { + return binary_row_hash(std::string_view(reinterpret_cast(_bytes.data()), _cursor)); +} + +int32_t binary_row_hash(std::string_view bytes) { + uint32_t hash = MURMUR_SEED; + size_t offset = 0; + const size_t aligned_length = bytes.size() - bytes.size() % 4U; + for (; offset < aligned_length; offset += 4U) { + uint32_t word; + std::memcpy(&word, bytes.data() + offset, sizeof(word)); + hash = mix_h1(hash, mix_k1(word)); + } + for (; offset < bytes.size(); ++offset) { + int32_t signed_byte = static_cast(bytes[offset]); + hash = mix_h1(hash, mix_k1(static_cast(signed_byte))); + } + uint32_t mixed = fmix(hash, bytes.size()); + int32_t result; + std::memcpy(&result, &mixed, sizeof(result)); + return result; +} + +std::optional default_bucket(int32_t bucket_key_hash, int32_t num_buckets) { + if (num_buckets <= 0) { + return std::nullopt; + } + int32_t remainder = bucket_key_hash % num_buckets; + return static_cast(remainder < 0 ? -remainder : remainder); +} + +std::optional fixed_bucket_channel(int32_t partition_hash, uint32_t bucket, + uint32_t num_channels) { + if (num_channels == 0) { + return std::nullopt; + } + if (partition_hash == std::numeric_limits::min()) { + partition_hash = std::numeric_limits::max(); + } + uint32_t start_channel = + static_cast(partition_hash < 0 ? -partition_hash : partition_hash) % + num_channels; + return static_cast((static_cast(start_channel) + bucket) % num_channels); +} + +} // namespace doris::paimon_native diff --git a/be/src/exec/partitioner/external/paimon_native_row_hash.h b/be/src/exec/partitioner/external/paimon_native_row_hash.h new file mode 100644 index 00000000000000..49f3a1a924e6f4 --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_native_row_hash.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris::paimon_native { + +// Builds the byte representation used by Paimon BinaryRowWriter for the supported routing +// types. MemorySegment primitive access is native-endian. +class BinaryRowEncoder { +public: + explicit BinaryRowEncoder(size_t arity); + + void reset(); + bool set_null(size_t position); + bool write_boolean(size_t position, bool value); + bool write_tinyint(size_t position, int8_t value); + bool write_smallint(size_t position, int16_t value); + bool write_int(size_t position, int32_t value); + bool write_bigint(size_t position, int64_t value); + bool write_float(size_t position, float value); + bool write_double(size_t position, double value); + bool write_string(size_t position, std::string_view utf8); + bool write_binary(size_t position, std::string_view bytes); + + int32_t hash() const; + const std::vector& bytes() const { return _bytes; } + +private: + bool _valid_position(size_t position) const; + size_t _field_offset(size_t position) const; + void _set_null_bit(size_t position); + void _ensure_capacity(size_t size); + void _set_offset_and_size(size_t position, uint32_t offset, uint32_t size); + bool _write_bytes(size_t position, std::string_view bytes); + + const size_t _arity; + const size_t _null_bits_size; + const size_t _fixed_size; + size_t _cursor; + std::vector _bytes; +}; + +int32_t binary_row_hash(std::string_view bytes); + +// Reproduces Paimon DefaultBucketFunction. nullopt means invalid metadata. +std::optional default_bucket(int32_t bucket_key_hash, int32_t num_buckets); + +// Reproduces ChannelComputer.select(partition, bucket, numChannels). +std::optional fixed_bucket_channel(int32_t partition_hash, uint32_t bucket, + uint32_t num_channels); + +} // namespace doris::paimon_native diff --git a/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp new file mode 100644 index 00000000000000..a941c4f19cfa1f --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.cpp @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/external/paimon_row_hash_partition_function.h" + +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type_nullable.h" +#include "exec/partitioner/external/paimon_native_row_hash.h" + +namespace doris { + +namespace { +bool is_supported_type(PrimitiveType type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + case TYPE_BINARY: + case TYPE_VARBINARY: + return true; + default: + return false; + } +} + +template +bool read_fixed_value(const IColumn& column, size_t row, T* value) { + StringRef data = column.get_data_at(row); + if (data.size != sizeof(T)) { + return false; + } + std::memcpy(value, data.data, sizeof(T)); + return true; +} + +Status encode_field(paimon_native::BinaryRowEncoder* encoder, size_t target_position, + const ColumnWithTypeAndName& field, size_t row) { + const IColumn& column = *field.column; + if (column.is_null_at(row)) { + if (!encoder->set_null(target_position)) { + return Status::InternalError("Failed to encode null Paimon routing field {}", + target_position); + } + return Status::OK(); + } + + bool encoded = false; + switch (remove_nullable(field.type)->get_primitive_type()) { + case TYPE_BOOLEAN: { + uint8_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_boolean(target_position, value != 0); + break; + } + case TYPE_TINYINT: { + int8_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_tinyint(target_position, value); + break; + } + case TYPE_SMALLINT: { + int16_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_smallint(target_position, value); + break; + } + case TYPE_INT: { + int32_t value = 0; + encoded = + read_fixed_value(column, row, &value) && encoder->write_int(target_position, value); + break; + } + case TYPE_BIGINT: { + int64_t value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_bigint(target_position, value); + break; + } + case TYPE_FLOAT: { + float value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_float(target_position, value); + break; + } + case TYPE_DOUBLE: { + double value = 0; + encoded = read_fixed_value(column, row, &value) && + encoder->write_double(target_position, value); + break; + } + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: { + StringRef value = column.get_data_at(row); + encoded = encoder->write_string(target_position, std::string_view(value.data, value.size)); + break; + } + case TYPE_BINARY: + case TYPE_VARBINARY: { + StringRef value = column.get_data_at(row); + encoded = encoder->write_binary(target_position, std::string_view(value.data, value.size)); + break; + } + default: + return Status::InvalidArgument("Unsupported Doris type {} for Paimon native routing", + field.type->get_name()); + } + if (!encoded) { + return Status::InvalidArgument("Doris column {} cannot be encoded for Paimon routing", + field.name); + } + return Status::OK(); +} + +Status encode_fields(paimon_native::BinaryRowEncoder* encoder, + const std::vector& field_indexes, + const std::vector& fields, size_t row) { + encoder->reset(); + for (size_t position = 0; position < field_indexes.size(); ++position) { + RETURN_IF_ERROR(encode_field(encoder, position, fields[field_indexes[position]], row)); + } + return Status::OK(); +} +} // namespace + +PaimonRowHashPartitionFunction::PaimonRowHashPartitionFunction(HashValType partition_count) + : _partition_count(partition_count) {} + +Status PaimonRowHashPartitionFunction::init(const std::vector& texprs) { + if (_partition_count == 0) { + return Status::InvalidArgument("Paimon writer count must be positive"); + } + RETURN_IF_ERROR(VExpr::create_expr_trees(texprs, _field_expr_ctxs)); + for (const auto& context : _field_expr_ctxs) { + PrimitiveType type = remove_nullable(context->root()->data_type())->get_primitive_type(); + if (!is_supported_type(type)) { + return Status::InvalidArgument("Unsupported Paimon native routing type {}", + context->root()->data_type()->get_name()); + } + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_validate_field_indexes(const std::vector& indexes, + bool require_non_empty) const { + if (require_non_empty && indexes.empty()) { + return Status::InvalidArgument("Paimon routing fields are missing"); + } + for (int32_t index : indexes) { + if (index < 0 || index >= _field_expr_ctxs.size()) { + return Status::InvalidArgument("Invalid Paimon routing field index {}", index); + } + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) { + return VExpr::prepare(_field_expr_ctxs, state, row_desc); +} + +Status PaimonRowHashPartitionFunction::open(RuntimeState* state) { + return VExpr::open(_field_expr_ctxs, state); +} + +Status PaimonRowHashPartitionFunction::_evaluate_fields( + Block* block, std::vector& fields) const { + fields.resize(_field_expr_ctxs.size()); + for (size_t index = 0; index < _field_expr_ctxs.size(); ++index) { + RETURN_IF_ERROR(_field_expr_ctxs[index]->execute(block, fields[index])); + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_hash_fields( + const std::vector& indexes, const std::vector& fields, + std::vector& hashes) const { + paimon_native::BinaryRowEncoder encoder(indexes.size()); + hashes.resize(fields.empty() ? 0 : fields.front().column->size()); + for (size_t row = 0; row < hashes.size(); ++row) { + RETURN_IF_ERROR(encode_fields(&encoder, indexes, fields, row)); + hashes[row] = encoder.hash(); + } + return Status::OK(); +} + +Status PaimonRowHashPartitionFunction::_clone_expr_ctxs(RuntimeState* state, + VExprContextSPtrs& destination) const { + destination.resize(_field_expr_ctxs.size()); + for (size_t index = 0; index < _field_expr_ctxs.size(); ++index) { + RETURN_IF_ERROR(_field_expr_ctxs[index]->clone(state, destination[index])); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h new file mode 100644 index 00000000000000..d4e0a25f61aecd --- /dev/null +++ b/be/src/exec/partitioner/external/paimon_row_hash_partition_function.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "exec/partitioner/partitioner.h" + +namespace doris { + +// Shared expression lifecycle and BinaryRow hashing for Paimon routing functions. +class PaimonRowHashPartitionFunction : public PartitionFunction { +public: + explicit PaimonRowHashPartitionFunction(HashValType partition_count); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override { return Status::OK(); } + HashValType partition_count() const override { return _partition_count; } + +protected: + Status _validate_field_indexes(const std::vector& indexes, + bool require_non_empty) const; + Status _evaluate_fields(Block* block, std::vector& fields) const; + Status _hash_fields(const std::vector& indexes, + const std::vector& fields, + std::vector& hashes) const; + Status _clone_expr_ctxs(RuntimeState* state, VExprContextSPtrs& destination) const; + + const HashValType _partition_count; + VExprContextSPtrs _field_expr_ctxs; +}; + +} // namespace doris diff --git a/be/src/exec/partitioner/partitioner.cpp b/be/src/exec/partitioner/partitioner.cpp index a7290be8c2925b..747f9ed33377cb 100644 --- a/be/src/exec/partitioner/partitioner.cpp +++ b/be/src/exec/partitioner/partitioner.cpp @@ -84,6 +84,51 @@ Status Crc32CHashPartitioner::clone(RuntimeState* state, return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); } +HashPartitionFunction::HashPartitionFunction(HashValType partition_count, + ShuffleHashMethod hash_method) + : _partition_count(partition_count), _hash_method(hash_method) {} + +Status HashPartitionFunction::init(const std::vector& texprs) { + if (_hash_method == ShuffleHashMethod::CRC32C) { + _partitioner = std::make_unique(_partition_count); + } else { + _partitioner = std::make_unique>(_partition_count); + } + return _partitioner->init(texprs); +} + +Status HashPartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) { + return _partitioner->prepare(state, row_desc); +} + +Status HashPartitionFunction::open(RuntimeState* state) { + return _partitioner->open(state); +} + +Status HashPartitionFunction::close(RuntimeState* state) { + return _partitioner->close(state); +} + +Status HashPartitionFunction::get_partitions(RuntimeState* state, Block* block, + size_t partition_count, + std::vector& partitions) const { + if (partition_count != _partition_count) { + return Status::InvalidArgument("Hash partition count {} does not match planned count {}", + partition_count, _partition_count); + } + RETURN_IF_ERROR(_partitioner->do_partitioning(state, block)); + partitions = _partitioner->get_channel_ids(); + return Status::OK(); +} + +Status HashPartitionFunction::clone(RuntimeState* state, + std::unique_ptr& function) const { + auto cloned = std::make_unique(_partition_count, _hash_method); + RETURN_IF_ERROR(_partitioner->clone(state, cloned->_partitioner)); + function = std::move(cloned); + return Status::OK(); +} + template class Crc32HashPartitioner; template class Crc32HashPartitioner; template class Crc32HashPartitioner; diff --git a/be/src/exec/partitioner/partitioner.h b/be/src/exec/partitioner/partitioner.h index 98607c3623634f..3f6562f8e3bb9f 100644 --- a/be/src/exec/partitioner/partitioner.h +++ b/be/src/exec/partitioner/partitioner.h @@ -55,6 +55,11 @@ class PartitionerBase { const HashValType _partition_count; }; +enum class ShuffleHashMethod { + CRC32, + CRC32C, +}; + class PartitionFunction { public: using HashValType = PartitionerBase::HashValType; @@ -78,9 +83,25 @@ class PartitionFunction { std::unique_ptr& function) const = 0; }; -enum class ShuffleHashMethod { - CRC32, - CRC32C, +// Adapts the standard Doris expression hash partitioner to the composable +// PartitionFunction interface used by sink routing. +class HashPartitionFunction final : public PartitionFunction { +public: + HashPartitionFunction(HashValType partition_count, ShuffleHashMethod hash_method); + + Status init(const std::vector& texprs) override; + Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state) override; + Status get_partitions(RuntimeState* state, Block* block, size_t partition_count, + std::vector& partitions) const override; + HashValType partition_count() const override { return _partition_count; } + Status clone(RuntimeState* state, std::unique_ptr& function) const override; + +private: + HashValType _partition_count; + ShuffleHashMethod _hash_method; + std::unique_ptr _partitioner; }; template diff --git a/be/src/exec/partitioner/writer_assigner.cpp b/be/src/exec/partitioner/writer_assigner.cpp new file mode 100644 index 00000000000000..3245616b1fe36c --- /dev/null +++ b/be/src/exec/partitioner/writer_assigner.cpp @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/writer_assigner.h" + +#include "exec/connector/skewed_partition_rebalancer.h" + +namespace doris { + +namespace { +Status validate_assignment_input(const std::vector& partition_ids, + const std::vector* mask, size_t rows) { + if (partition_ids.size() < rows) { + return Status::InvalidArgument("Writer assignment has {} partition ids for {} rows", + partition_ids.size(), rows); + } + if (mask != nullptr && mask->size() < rows) { + return Status::InvalidArgument("Writer assignment mask has {} entries for {} rows", + mask->size(), rows); + } + return Status::OK(); +} +} // namespace + +Status IdentityWriterAssigner::assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, + size_t /*block_bytes*/, std::vector& writer_ids) { + RETURN_IF_ERROR(validate_assignment_input(partition_ids, mask, rows)); + if (writer_ids.size() != rows && &writer_ids != &partition_ids) { + writer_ids.resize(rows); + } + for (size_t row = 0; row < rows; ++row) { + if (mask != nullptr && (*mask)[row] == 0) { + continue; + } + if (partition_ids[row] >= _writer_count) { + return Status::InvalidArgument("Logical partition {} exceeds writer count {}", + partition_ids[row], _writer_count); + } + writer_ids[row] = partition_ids[row]; + } + return Status::OK(); +} + +SkewedWriterAssigner::SkewedWriterAssigner(int partition_count, int task_count, + int task_bucket_count, + long min_partition_data_processed_rebalance_threshold, + long min_data_processed_rebalance_threshold) + : _rebalancer(std::make_unique( + partition_count, task_count, task_bucket_count, + min_partition_data_processed_rebalance_threshold, + min_data_processed_rebalance_threshold)), + _writer_count(task_count), + _partition_row_counts(partition_count, 0), + _partition_writer_ids(partition_count, -1), + _partition_writer_indexes(partition_count, 0) {} + +SkewedWriterAssigner::~SkewedWriterAssigner() = default; + +Status SkewedWriterAssigner::assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, + size_t block_bytes, std::vector& writer_ids) { + RETURN_IF_ERROR(validate_assignment_input(partition_ids, mask, rows)); + if (rows == 0) { + return Status::OK(); + } + if (_partition_row_counts.empty()) { + return Status::InvalidArgument("Skewed writer assignment has no logical partitions"); + } + if (writer_ids.size() != rows && &writer_ids != &partition_ids) { + writer_ids.resize(rows); + } + + std::fill(_partition_row_counts.begin(), _partition_row_counts.end(), 0); + std::fill(_partition_writer_ids.begin(), _partition_writer_ids.end(), -1); + _rebalancer->rebalance(); + + const size_t partition_count = _partition_row_counts.size(); + for (size_t row = 0; row < rows; ++row) { + if (mask != nullptr && (*mask)[row] == 0) { + continue; + } + const uint32_t partition_id = partition_ids[row]; + if (partition_id >= partition_count) { + return Status::InvalidArgument("Logical partition {} exceeds partition count {}", + partition_id, partition_count); + } + _partition_row_counts[partition_id] += 1; + int writer_id = _partition_writer_ids[partition_id]; + if (writer_id == -1) { + writer_id = _get_next_writer_id(partition_id); + if (writer_id < 0 || writer_id >= _writer_count) { + return Status::InternalError("Skewed writer assignment returned invalid writer {}", + writer_id); + } + _partition_writer_ids[partition_id] = writer_id; + } + writer_ids[row] = static_cast(writer_id); + } + + for (size_t partition_id = 0; partition_id < partition_count; ++partition_id) { + if (_partition_row_counts[partition_id] > 0) { + _rebalancer->add_partition_row_count(static_cast(partition_id), + _partition_row_counts[partition_id]); + } + } + _rebalancer->add_data_processed(static_cast(block_bytes)); + return Status::OK(); +} + +int SkewedWriterAssigner::_get_next_writer_id(uint32_t partition_id) { + return _rebalancer->get_task_id(partition_id, _partition_writer_indexes[partition_id]++); +} + +int64_t scale_writer_threshold_by_task(int64_t value, int task_num) { + if (task_num <= 0) { + return value; + } + int64_t scaled = value / task_num; + return scaled == 0 ? value : scaled; +} + +} // namespace doris diff --git a/be/src/exec/partitioner/writer_assigner.h b/be/src/exec/partitioner/writer_assigner.h new file mode 100644 index 00000000000000..7a4237cc3645ea --- /dev/null +++ b/be/src/exec/partitioner/writer_assigner.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris { +class SkewedPartitionRebalancer; +} + +namespace doris { + +// Maps logical partitions computed by a PartitionFunction to Doris exchange channels. +class WriterAssigner { +public: + virtual ~WriterAssigner() = default; + + virtual Status assign(const std::vector& partition_ids, + const std::vector* mask, size_t rows, size_t block_bytes, + std::vector& writer_ids) = 0; +}; + +// Preserves stable ownership: one logical partition always maps to one writer id. +class IdentityWriterAssigner final : public WriterAssigner { +public: + explicit IdentityWriterAssigner(uint32_t writer_count) : _writer_count(writer_count) {} + + Status assign(const std::vector& partition_ids, const std::vector* mask, + size_t rows, size_t block_bytes, std::vector& writer_ids) override; + +private: + uint32_t _writer_count; +}; + +// Allows a hot logical partition to use multiple writers while retaining the existing +// ScaleWriter affinity and rebalance behavior. +class SkewedWriterAssigner final : public WriterAssigner { +public: + SkewedWriterAssigner(int partition_count, int task_count, int task_bucket_count, + long min_partition_data_processed_rebalance_threshold, + long min_data_processed_rebalance_threshold); + + ~SkewedWriterAssigner() override; + + Status assign(const std::vector& partition_ids, const std::vector* mask, + size_t rows, size_t block_bytes, std::vector& writer_ids) override; + +private: + int _get_next_writer_id(uint32_t partition_id); + + std::unique_ptr _rebalancer; + int _writer_count; + std::vector _partition_row_counts; + std::vector _partition_writer_ids; + std::vector _partition_writer_indexes; +}; + +// Scale table-sink thresholds by local pipeline task count while preserving the historical +// behavior for very small values. +int64_t scale_writer_threshold_by_task(int64_t value, int task_num); + +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 2fa064c8a68e09..092eaa08f30801 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -88,6 +88,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -1376,6 +1377,14 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS output_exprs); break; } + case TDataSinkType::PAIMON_TABLE_SINK: { + if (!thrift_sink.__isset.paimon_table_sink) { + return Status::InternalError("Missing paimon table sink."); + } + _sink = std::make_shared(next_sink_operator_id(), row_desc, + output_exprs); + break; + } case TDataSinkType::JDBC_TABLE_SINK: { if (!thrift_sink.__isset.jdbc_table_sink) { return Status::InternalError("Missing data jdbc sink."); @@ -2526,6 +2535,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } _append_external_file_commit_data(req, ¶ms); + if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(), + pcm.end()); + } else if (!req.runtime_states.empty()) { + for (auto* rs : req.runtime_states) { + if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), + rs_pcm.begin(), rs_pcm.end()); + } + } + } + req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); diff --git a/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp b/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp deleted file mode 100644 index 6cdebdcf6d982e..00000000000000 --- a/be/src/exec/sink/scale_writer_partitioning_exchanger.hpp +++ /dev/null @@ -1,126 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/block/block.h" -#include "exec/connector/skewed_partition_rebalancer.h" -#include "exec/partitioner/partitioner.h" - -namespace doris { -class ScaleWriterPartitioner final : public PartitionerBase { -public: - ScaleWriterPartitioner(int channel_size, int partition_count, int task_count, - int task_bucket_count, - long min_partition_data_processed_rebalance_threshold, - long min_data_processed_rebalance_threshold) - : PartitionerBase(partition_count), - _channel_size(channel_size), - _partition_rebalancer(partition_count, task_count, task_bucket_count, - min_partition_data_processed_rebalance_threshold, - min_data_processed_rebalance_threshold), - _partition_row_counts(partition_count, 0), - _partition_writer_ids(partition_count, -1), - _partition_writer_indexes(partition_count, 0), - _task_count(task_count), - _task_bucket_count(task_bucket_count), - _min_partition_data_processed_rebalance_threshold( - min_partition_data_processed_rebalance_threshold), - _min_data_processed_rebalance_threshold(min_data_processed_rebalance_threshold) { - _crc_partitioner = - std::make_unique>(_partition_count); - } - - ~ScaleWriterPartitioner() override = default; - - Status init(const std::vector& texprs) override { - return _crc_partitioner->init(texprs); - } - - Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override { - return _crc_partitioner->prepare(state, row_desc); - } - - Status open(RuntimeState* state) override { return _crc_partitioner->open(state); } - - Status close(RuntimeState* state) override { return _crc_partitioner->close(state); } - - Status do_partitioning(RuntimeState* state, Block* block) const override { - _hash_vals.resize(block->rows()); - for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) { - _partition_row_counts[partition_id] = 0; - _partition_writer_ids[partition_id] = -1; - } - - _partition_rebalancer.rebalance(); - - RETURN_IF_ERROR(_crc_partitioner->do_partitioning(state, block)); - const auto& channel_ids = _crc_partitioner->get_channel_ids(); - for (size_t position = 0; position < block->rows(); position++) { - auto partition_id = channel_ids[position]; - _partition_row_counts[partition_id] += 1; - - // Get writer id for this partition by looking at the scaling state - int writer_id = _partition_writer_ids[partition_id]; - if (writer_id == -1) { - writer_id = _get_next_writer_id(partition_id); - _partition_writer_ids[partition_id] = writer_id; - } - _hash_vals[position] = writer_id; - } - - for (int partition_id = 0; partition_id < _partition_row_counts.size(); partition_id++) { - _partition_rebalancer.add_partition_row_count(partition_id, - _partition_row_counts[partition_id]); - } - _partition_rebalancer.add_data_processed(block->bytes()); - - return Status::OK(); - } - - const std::vector& get_channel_ids() const override { return _hash_vals; } - - Status clone(RuntimeState* state, std::unique_ptr& partitioner) override { - partitioner = std::make_unique( - _channel_size, (int)_partition_count, _task_count, _task_bucket_count, - _min_partition_data_processed_rebalance_threshold, - _min_data_processed_rebalance_threshold); - return Status::OK(); - } - -private: - int _get_next_writer_id(HashValType partition_id) const { - return _partition_rebalancer.get_task_id(partition_id, - _partition_writer_indexes[partition_id]++); - } - - int _channel_size; - std::unique_ptr _crc_partitioner; - mutable SkewedPartitionRebalancer _partition_rebalancer; - mutable std::vector _partition_row_counts; - mutable std::vector _partition_writer_ids; - mutable std::vector _partition_writer_indexes; - mutable std::vector _hash_vals; - const int _task_count; - const int _task_bucket_count; - const long _min_partition_data_processed_rebalance_threshold; - const long _min_data_processed_rebalance_threshold; -}; -} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp new file mode 100644 index 00000000000000..a5abfcdc15c41c --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" + +namespace doris { + +Status FfiPaimonWriteBackend::open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::create_writer(std::unique_ptr*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::close() { + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h new file mode 100644 index 00000000000000..be833d53b79bcd --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +/// Placeholder for the future paimon-rust writer implementation. Keeping this +/// backend in the factory makes the integration boundary explicit without +/// introducing a BE commit contract that the Rust writer will not own. +class FfiPaimonWriteBackend final : public IPaimonWriteBackend { +public: + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::FFI; } +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp new file mode 100644 index 00000000000000..d46fb1d99f691f --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "exec/spill/spill_file_manager.h" +#include "format/arrow/arrow_block_convertor.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/defer_op.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" + +namespace doris { + +namespace { +constexpr std::string_view PAIMON_JNI_WRITER_IO_TMP_DIR = "paimon_jni_writer_io_tmp"; + +void throw_java_io_exception(JNIEnv* env, const std::string& message) { + jclass exception_class = env->FindClass("java/io/IOException"); + env->ThrowNew(exception_class, message.c_str()); + env->DeleteLocalRef(exception_class); +} + +jobjectArray get_paimon_spill_directories(JNIEnv* env, jclass, jlong spill_session_handle) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr) { + throw_java_io_exception(env, "Paimon external spill session is null"); + return nullptr; + } + + std::vector paths; + Status st = spill_session->get_paths(&paths); + if (!st.ok()) { + throw_java_io_exception(env, st.to_string()); + return nullptr; + } + jclass string_class = env->FindClass("java/lang/String"); + if (string_class == nullptr) { + return nullptr; + } + jobjectArray result = + env->NewObjectArray(static_cast(paths.size()), string_class, nullptr); + env->DeleteLocalRef(string_class); + if (result == nullptr) { + return nullptr; + } + for (jsize i = 0; i < static_cast(paths.size()); ++i) { + jstring path = env->NewStringUTF(paths[i].c_str()); + if (path == nullptr) { + return nullptr; + } + env->SetObjectArrayElement(result, i, path); + env->DeleteLocalRef(path); + if (env->ExceptionCheck()) { + return nullptr; + } + } + return result; +} + +void reserve_paimon_spill(JNIEnv* env, jclass, jlong spill_session_handle, jstring path, + jlong bytes) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr || path == nullptr) { + throw_java_io_exception(env, "Paimon external spill session or path is null"); + return; + } + const char* path_chars = env->GetStringUTFChars(path, nullptr); + if (path_chars == nullptr) { + return; + } + std::string native_path(path_chars); + env->ReleaseStringUTFChars(path, path_chars); + Status st = spill_session->reserve(native_path, bytes); + if (!st.ok()) { + throw_java_io_exception(env, st.to_string()); + } +} + +void update_paimon_spill_accounting(JNIEnv* env, jclass, jlong spill_session_handle, jstring path, + jlong current_bytes_delta, jlong write_bytes, + jlong read_bytes) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr || path == nullptr) { + return; + } + const char* path_chars = env->GetStringUTFChars(path, nullptr); + if (path_chars == nullptr) { + return; + } + std::string native_path(path_chars); + env->ReleaseStringUTFChars(path, path_chars); + spill_session->update_accounting(native_path, current_bytes_delta, write_bytes, read_bytes); +} + +Status register_paimon_spill_natives(JNIEnv* env, jclass writer_class) { + static char get_spill_directories_name[] = "getPaimonSpillDirectories"; + static char get_spill_directories_signature[] = "(J)[Ljava/lang/String;"; + static char reserve_spill_name[] = "reservePaimonSpill"; + static char reserve_spill_signature[] = "(JLjava/lang/String;J)V"; + static char update_spill_name[] = "updatePaimonSpillAccounting"; + static char update_spill_signature[] = "(JLjava/lang/String;JJJ)V"; + static ::JNINativeMethod methods[] = { + {get_spill_directories_name, get_spill_directories_signature, + reinterpret_cast(&get_paimon_spill_directories)}, + {reserve_spill_name, reserve_spill_signature, + reinterpret_cast(&reserve_paimon_spill)}, + {update_spill_name, update_spill_signature, + reinterpret_cast(&update_paimon_spill_accounting)}, + }; + if (env->RegisterNatives(writer_class, methods, + static_cast(sizeof(methods) / sizeof(methods[0]))) != JNI_OK) { + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, true, "JNI exception registering Paimon spill native methods: ")); + return Status::JniError("Failed to register Paimon spill native methods"); + } + return Status::OK(); +} + +std::atomic& paimon_jni_close_failed() { + static std::atomic failed {false}; + return failed; +} + +struct RetainedPaimonResources { + std::unique_ptr memory_manager; + std::unique_ptr spill_session; +}; + +std::mutex& retained_resources_mutex() { + static auto* mutex = new std::mutex(); + return *mutex; +} + +std::vector& retained_resources() { + static auto* resources = new std::vector(); + return *resources; +} + +void retain_resources_after_failed_close(std::unique_ptr memory_manager, + std::unique_ptr spill_session) { + // An unconfirmed Java close means a background Paimon task may still reference this manager's + // native pages or spill callbacks. Quarantine both resources and stop admitting new writers so + // repeated failures cannot accumulate process-lifetime resources without a bound. + paimon_jni_close_failed().store(true, std::memory_order_release); + if (memory_manager == nullptr && spill_session == nullptr) { + return; + } + std::lock_guard lock(retained_resources_mutex()); + retained_resources().emplace_back(RetainedPaimonResources { + .memory_manager = std::move(memory_manager), + .spill_session = std::move(spill_session), + }); +} + +} // namespace + +// ──────────────────────────────────────────────────────────── +// JNI helpers β€” class loading +// ──────────────────────────────────────────────────────────── + +static constexpr const char* PAIMON_JNI_WRITER_CLASS = "org/apache/doris/paimon/PaimonJniWriter"; +const char* const PAIMON_JNI_WRITER_OPEN_SIGNATURE = + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZZLjava/lang/" + "String;JJJ)V"; + +PaimonJniWriterOpenMode PaimonJniWriterOpenMode::from_write_mode( + TPaimonWriteMode::type write_mode) { + return {static_cast(write_mode == TPaimonWriteMode::OVERWRITE), + static_cast(write_mode == TPaimonWriteMode::CHANGELOG)}; +} + +JniPaimonWriteBackend::JniPaimonWriteBackend() = default; + +JniPaimonWriteBackend::~JniPaimonWriteBackend() { + Status st = close(); + if (!st.ok()) { + LOG(WARNING) << "Failed to close Paimon JNI backend during destruction: " << st.to_string(); + } +} + +Status JniPaimonWriteBackend::close() { + if (_jni_writer_obj == nullptr && _jni_writer_cls == nullptr) { + _memory_manager.reset(); + _arrow_schema.reset(); + _spill_session.reset(); + _opened = false; + return Status::OK(); + } + + JNIEnv* env = nullptr; + Status env_status = Jni::Env::Get(&env); + if (!env_status.ok()) { + bool java_users_may_exist = _jni_writer_obj != nullptr; + // JNI global references cannot be released without an environment. + // Deliberately abandon the handles so the Java writer remains alive. + _jni_writer_obj = nullptr; + _jni_writer_cls = nullptr; + if (java_users_may_exist) { + retain_resources_after_failed_close(std::move(_memory_manager), + std::move(_spill_session)); + } else { + _memory_manager.reset(); + _spill_session.reset(); + } + _arrow_schema.reset(); + _opened = false; + return env_status; + } + + Status close_status = Status::OK(); + if (_jni_writer_obj != nullptr) { + _refresh_memory_profile(); + if (_close_id == nullptr) { + close_status = Status::InternalError("PaimonJniWriter.close method is unavailable"); + } else { + env->CallVoidMethod(_jni_writer_obj, _close_id); + close_status = _check_jni_exception(env, "close PaimonJniWriter"); + } + env->DeleteGlobalRef(_jni_writer_obj); + _jni_writer_obj = nullptr; + } + if (_jni_writer_cls != nullptr) { + env->DeleteGlobalRef(_jni_writer_cls); + _jni_writer_cls = nullptr; + } + + if (close_status.ok()) { + _memory_manager.reset(); + _spill_session.reset(); + } else { + if (_memory_manager != nullptr) { + LOG(WARNING) + << "Retaining Paimon JNI native memory after an unconfirmed Java close: limit=" + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) << ", peak=" + << PrettyPrinter::print_bytes(_memory_manager->native_peak_allocated_bytes()); + } + // Paimon may still have asynchronous tasks using Doris-backed pages or spill callbacks. + // Retain ownership until process exit and fence subsequent writer admission. + retain_resources_after_failed_close(std::move(_memory_manager), std::move(_spill_session)); + } + _arrow_schema.reset(); + _opened = false; + return close_status; +} + +Status JniPaimonWriteBackend::_check_jni_exception(JNIEnv* env, const std::string& method_name) { + if (env->ExceptionCheck()) { + Status st = + Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in " + method_name + ": "); + LOG(WARNING) << st.to_string(); + return st; + } + return Status::OK(); +} + +static Status _get_paimon_arrow_schema(JNIEnv* env, jobject writer, jmethodID get_schema_id, + std::shared_ptr* schema) { + auto schema_bytes = static_cast(env->CallObjectMethod(writer, get_schema_id)); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception in PaimonJniWriter.getArrowSchema: ")); + if (schema_bytes == nullptr) { + return Status::InternalError("PaimonJniWriter.getArrowSchema returned null"); + } + + const jsize size = env->GetArrayLength(schema_bytes); + if (size <= 0) { + env->DeleteLocalRef(schema_bytes); + return Status::InternalError("PaimonJniWriter.getArrowSchema returned empty data"); + } + std::string serialized_schema(static_cast(size), '\0'); + env->GetByteArrayRegion(schema_bytes, 0, size, + reinterpret_cast(serialized_schema.data())); + env->DeleteLocalRef(schema_bytes); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception while reading Paimon Arrow schema: ")); + + auto input = std::make_shared( + arrow::Buffer::FromString(std::move(serialized_schema))); + auto reader_result = arrow::ipc::RecordBatchStreamReader::Open(input); + if (!reader_result.ok()) { + return Status::InternalError("Failed to deserialize Paimon Arrow schema: {}", + reader_result.status().ToString()); + } + *schema = reader_result.ValueOrDie()->schema(); + return Status::OK(); +} +Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) { + if (paimon_jni_close_failed().load(std::memory_order_acquire)) { + return Status::InternalError( + "Paimon JNI writes are disabled on this BE because a previous Java writer close " + "could not be confirmed; restart the BE to reclaim retained native memory safely"); + } + _arrow_schema.reset(); + DORIS_CHECK(sink.__isset.column_names); + DORIS_CHECK(sink.__isset.write_mode); + DORIS_CHECK(sink.__isset.serialized_table); + DORIS_CHECK(!sink.serialized_table.empty()); + DORIS_CHECK(sink.__isset.transaction_id); + DORIS_CHECK(sink.transaction_id > 0); + DORIS_CHECK(sink.__isset.commit_user); + DORIS_CHECK(!sink.commit_user.empty()); + DORIS_CHECK(profile != nullptr); + + RETURN_IF_ERROR(PaimonJniMemoryManager::create(state, &_memory_manager)); + RuntimeProfile* jni_profile = profile->create_child("JniPaimonWriteBackend", true, true); + _native_page_memory_limit = ADD_COUNTER(jni_profile, "NativePageMemoryLimit", TUnit::BYTES); + _native_page_memory_peak = ADD_COUNTER(jni_profile, "NativePageMemoryPeak", TUnit::BYTES); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + if (env->PushLocalFrame(32) != JNI_OK) { + Status st = _check_jni_exception(env, "create PaimonJniWriter open local reference frame"); + return st.ok() ? Status::InternalError("Failed to create JNI local reference frame") : st; + } + Defer pop_local_frame([&]() { env->PopLocalFrame(nullptr); }); + + // Step 1: Load PaimonJniWriter class through ScannerLoader (Paimon jars are + // not on the default application classpath, so FindClass won't work). + Jni::LocalObject local_writer_class; + RETURN_IF_ERROR( + Jni::Util::get_jni_scanner_class(env, PAIMON_JNI_WRITER_CLASS, &local_writer_class)); + auto writer_class = static_cast(local_writer_class.get()); + _jni_writer_cls = static_cast(env->NewGlobalRef(writer_class)); + RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter class reference")); + if (_jni_writer_cls == nullptr) { + return Status::JniError("Failed to create global PaimonJniWriter class reference"); + } + RETURN_IF_ERROR(PaimonJniMemoryManager::register_natives(env, _jni_writer_cls)); + RETURN_IF_ERROR(register_paimon_spill_natives(env, _jni_writer_cls)); + + // Step 2: Cache JNI method IDs for write, prepareCommit, abort, close. + jmethodID open_id = env->GetMethodID(_jni_writer_cls, "open", PAIMON_JNI_WRITER_OPEN_SIGNATURE); + jmethodID get_arrow_schema_id = env->GetMethodID(_jni_writer_cls, "getArrowSchema", "()[B"); + _write_id = env->GetMethodID(_jni_writer_cls, "writeArrow", "(JJ)V"); + _prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B"); + _abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V"); + _close_id = env->GetMethodID(_jni_writer_cls, "close", "()V"); + RETURN_IF_ERROR(_check_jni_exception(env, "resolve PaimonJniWriter methods")); + + // Step 3: Create the Java PaimonJniWriter instance. + jmethodID ctor_id = env->GetMethodID(_jni_writer_cls, "", "()V"); + jobject local_obj = env->NewObject(_jni_writer_cls, ctor_id); + RETURN_IF_ERROR(_check_jni_exception(env, "create PaimonJniWriter")); + _jni_writer_obj = env->NewGlobalRef(local_obj); + RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter object reference")); + if (_jni_writer_obj == nullptr) { + return Status::JniError("Failed to create global PaimonJniWriter object reference"); + } + + // Step 4: Create a lazy query-scoped spill session. Java requests its path only when Paimon + // first uses the IOManager, so a memory-only writer does not depend on spill storage. + auto* spill_file_manager = state->exec_env()->spill_file_mgr(); + if (spill_file_manager != nullptr) { + auto spill_relative_path = + fmt::format("{}-{}", PAIMON_JNI_WRITER_IO_TMP_DIR, spill_file_manager->next_id()); + RETURN_IF_ERROR(spill_file_manager->create_external_spill_session( + spill_relative_path, state->get_query_ctx(), &_spill_session)); + } + + // Step 5: Build Java arguments and call PaimonJniWriter.open(). + const std::map empty_config; + jstring j_serialized_table = env->NewStringUTF(sink.serialized_table.c_str()); + Jni::LocalObject j_hadoop_config; + RETURN_IF_ERROR(Jni::Util::convert_to_java_map( + env, sink.__isset.hadoop_config ? sink.hadoop_config : empty_config, &j_hadoop_config)); + jstring j_commit_user = env->NewStringUTF(sink.commit_user.c_str()); + jstring j_time_zone = env->NewStringUTF(state->timezone().c_str()); + + jclass string_cls = env->FindClass("java/lang/String"); + jobjectArray j_cols = + env->NewObjectArray(static_cast(sink.column_names.size()), string_cls, nullptr); + for (size_t i = 0; i < sink.column_names.size(); ++i) { + jstring column_name = env->NewStringUTF(sink.column_names[i].c_str()); + env->SetObjectArrayElement(j_cols, static_cast(i), column_name); + env->DeleteLocalRef(column_name); + } + RETURN_IF_ERROR(_check_jni_exception(env, "build PaimonJniWriter open arguments")); + + PaimonJniWriterOpenMode open_mode = PaimonJniWriterOpenMode::from_write_mode(sink.write_mode); + env->CallVoidMethod( + _jni_writer_obj, open_id, j_serialized_table, j_hadoop_config.get(), j_cols, + static_cast(sink.transaction_id), j_commit_user, open_mode.overwrite, + open_mode.changelog, j_time_zone, static_cast(_memory_manager->memory_limit()), + reinterpret_cast(_memory_manager.get()), + _spill_session == nullptr ? 0 : reinterpret_cast(_spill_session.get())); + Status st = _check_jni_exception(env, "open PaimonJniWriter"); + + if (st.ok()) { + st = _get_paimon_arrow_schema(env, _jni_writer_obj, get_arrow_schema_id, &_arrow_schema); + } + if (st.ok()) { + _opened = true; + _refresh_memory_profile(); + LOG(INFO) << "Paimon JNI writer memory limit: " + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) + << ", sink_pipeline_task_count=" << std::max(1, state->task_num()); + } + return st; +} + +// Writer creation stays non-const because the backend interface also supports future stateful FFI +// implementations. +Status JniPaimonWriteBackend::create_writer( // NOLINT(readability-make-member-function-const) + std::unique_ptr* writer) { + DORIS_CHECK(_opened); + DORIS_CHECK(_arrow_schema != nullptr); + *writer = std::make_unique(_jni_writer_obj, _write_id, _prepare_commit_id, + _abort_id, _arrow_schema); + return Status::OK(); +} + +JniPaimonWriter::JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, + jmethodID prepare_commit_id, jmethodID abort_id, + std::shared_ptr arrow_schema) + : _jni_writer_obj(jni_writer_obj), + _write_id(write_id), + _prepare_commit_id(prepare_commit_id), + _abort_id(abort_id), + _arrow_schema(std::move(arrow_schema)) {} + +Status JniPaimonWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + if (_arrow_schema == nullptr || _arrow_schema->num_fields() != block.columns()) { + return Status::InvalidArgument( + "Paimon Arrow schema column count does not match Doris Block: schema={}, block={}", + _arrow_schema == nullptr ? 0 : _arrow_schema->num_fields(), block.columns()); + } + + // The schema comes from the pinned Paimon table, so timestamp timezone, nested nullability and + // Variant layout are fixed before the first write. Arrow builders remain on the Doris side and + // are charged to the current query's MemTracker through ArrowMemoryPool. + std::shared_ptr record_batch; + RETURN_IF_ERROR(convert_to_arrow_batch(block, _arrow_schema, &_arrow_pool, &record_batch, + state->timezone_obj())); + + ArrowArray c_array {}; + ArrowSchema c_schema {}; + auto arrow_status = arrow::ExportRecordBatch(*record_batch, &c_array, &c_schema); + if (!arrow_status.ok()) { + return Status::InternalError("Failed to export Paimon Arrow RecordBatch: {}", + arrow_status.ToString()); + } + // Java consumes both C Data release callbacks on a successful import. On every exit, release + // whichever struct still retains its callback; this covers partial imports and JNI failures + // without double release. + Defer release_c_data {[&] { + if (c_array.release != nullptr) { + c_array.release(&c_array); + } + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + }}; + // writeArrow is synchronous and this operator runs on the blocking scheduler. The exported + // RecordBatch therefore stays alive until Paimon has consumed all rows; Java never owns an IPC + // copy, and any synchronous SDK flush or memory wait occupies only a blocking-scheduler worker. + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + env->CallVoidMethod(_jni_writer_obj, _write_id, reinterpret_cast(&c_array), + reinterpret_cast(&c_schema)); + return Jni::Env::GetJniExceptionMsg(env, false, + "JNI exception in JniPaimonWriter::writeArrow: "); +} + +Status JniPaimonWriter::prepare_commit(std::vector& messages) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + + // Call PaimonJniWriter.prepareCommit() which returns byte[][] β€” + // each element is a DPCM-framed serialized CommitMessage chunk produced + // by PaimonCommitCodec.encode(). + jobject j_payloads_obj = env->CallObjectMethod(_jni_writer_obj, _prepare_commit_id); + Status st = Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in prepareCommit: "); + if (!st.ok()) { + return st; + } + + if (j_payloads_obj == nullptr) { + return Status::InternalError("PaimonJniWriter.prepareCommit returned null"); + } + + // Unpack the byte[][] into TPaimonCommitMessage structs for FE transport. + auto* j_payloads = static_cast(j_payloads_obj); + jsize num_payloads = env->GetArrayLength(j_payloads); + + for (jsize i = 0; i < num_payloads; ++i) { + auto j_bytes = static_cast(env->GetObjectArrayElement(j_payloads, i)); + if (j_bytes == nullptr) { + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned a null payload"); + } + jsize len = env->GetArrayLength(j_bytes); + if (len == 0) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned an empty payload"); + } + TPaimonCommitMessage msg; + msg.payload.resize(static_cast(len)); + env->GetByteArrayRegion(j_bytes, 0, len, reinterpret_cast(msg.payload.data())); + Status copy_status = Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception while reading Paimon commit payload: "); + if (!copy_status.ok()) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + return copy_status; + } + msg.__isset.payload = true; + messages.emplace_back(std::move(msg)); + env->DeleteLocalRef(j_bytes); + } + env->DeleteLocalRef(j_payloads); + return Status::OK(); +} + +Status JniPaimonWriter::abort() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + env->CallVoidMethod(_jni_writer_obj, _abort_id); + return Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in abort: "); +} + +void JniPaimonWriteBackend::_refresh_memory_profile() { + if (_memory_manager == nullptr) { + return; + } + COUNTER_SET(_native_page_memory_limit, _memory_manager->memory_limit()); + COUNTER_SET(_native_page_memory_peak, _memory_manager->native_peak_allocated_bytes()); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h new file mode 100644 index 00000000000000..d5a5de3b9ea1bb --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "format/parquet/arrow_memory_pool.h" +#include "runtime/runtime_profile.h" + +namespace arrow { +class Schema; +} + +namespace doris { + +class ExternalSpillSession; +class RuntimeState; + +extern const char* const PAIMON_JNI_WRITER_OPEN_SIGNATURE; + +struct PaimonJniWriterOpenMode { + jboolean overwrite; + jboolean changelog; + + static PaimonJniWriterOpenMode from_write_mode(TPaimonWriteMode::type write_mode); +}; + +/// JNI backend that owns the Java PaimonJniWriter object and its JNI method +/// handles. Creates lightweight JniPaimonWriter adapters that share this +/// backend's JVM connection. +/// +/// Each JniPaimonWriteBackend corresponds to one Java PaimonJniWriter +/// instance; the JniPaimonWriter adapters are thin wrappers that delegate +/// write/prepare_commit/abort calls through the cached JNI method IDs. JNI-only +/// memory ownership and Profile counters stay here and are not part of the +/// common backend contract. +class JniPaimonWriteBackend final : public IPaimonWriteBackend { +public: + JniPaimonWriteBackend(); + ~JniPaimonWriteBackend() override; + + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::JNI; } + +private: + Status _check_jni_exception(JNIEnv* env, const std::string& method_name); + void _refresh_memory_profile(); + + // JNI global references β€” live for the duration of this backend. + jclass _jni_writer_cls = nullptr; + jobject _jni_writer_obj = nullptr; + + // Cached JNI method IDs for the PaimonJniWriter Java methods. + jmethodID _write_id = nullptr; + jmethodID _prepare_commit_id = nullptr; + jmethodID _abort_id = nullptr; + jmethodID _close_id = nullptr; + + std::unique_ptr _memory_manager; + std::shared_ptr _arrow_schema; + std::unique_ptr _spill_session; + RuntimeProfile::Counter* _native_page_memory_limit = nullptr; + RuntimeProfile::Counter* _native_page_memory_peak = nullptr; + bool _opened = false; +}; + +/// Lightweight C++ adapter that delegates to the shared JNI backend. +/// +/// Owns the Arrow memory pool used for Block β†’ Arrow RecordBatch conversion. +/// Each JniPaimonWriter is created by JniPaimonWriteBackend::create_writer() +/// and shares the backend's JNI method IDs and Java writer object reference. +class JniPaimonWriter final : public IPaimonWriter { +public: + JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, jmethodID prepare_commit_id, + jmethodID abort_id, std::shared_ptr arrow_schema); + + Status write(RuntimeState* state, Block& block) override; + Status prepare_commit(std::vector& messages) override; + Status abort() override; + +private: + // Shared JNI state (owned by JniPaimonWriteBackend, not this adapter). + jobject _jni_writer_obj; + jmethodID _write_id; + jmethodID _prepare_commit_id; + jmethodID _abort_id; + + // Arrow resources owned by this writer adapter. + ArrowMemoryPool<> _arrow_pool; + std::shared_ptr _arrow_schema; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp new file mode 100644 index 00000000000000..1ea94294033c97 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/allocator.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" + +namespace doris { + +class PaimonJniMemoryManager::Impl { +public: + Impl(std::shared_ptr resource_context, int64_t memory_limit) + : _resource_context(std::move(resource_context)), _memory_limit(memory_limit) { + DORIS_CHECK(_resource_context != nullptr); + DORIS_CHECK(_memory_limit > 0); + } + + ~Impl() { + // Java may retain direct buffers until its writer is closed. Release + // every outstanding page here as the final native ownership boundary. + try { + release_all_pages(); + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: " << e.what(); + } catch (...) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: unknown exception"; + } + } + + jobject allocate_page(JNIEnv* env, jint bytes) { + if (bytes <= 0) { + throw Exception(Status::InvalidArgument( + "Paimon JNI memory page size must be positive, actual={}", bytes)); + } + + // Reserve the writer-local budget before entering the allocator. This + // prevents concurrent JNI callbacks from transiently allocating past + // the configured cap and only discovering it after query accounting + // or the system allocator has already rejected the request. + { + std::lock_guard lock(_mutex); + if (bytes > _memory_limit - _native_allocated_bytes - _native_reserved_bytes) { + throw Exception(Status::Error( + "Paimon JNI write buffer exceeded its {} native memory limit", + PrettyPrinter::print_bytes(_memory_limit))); + } + _native_reserved_bytes += bytes; + } + bool reservation_committed = false; + Defer rollback_reservation {[&]() { + if (!reservation_committed) { + std::lock_guard lock(_mutex); + _native_reserved_bytes -= bytes; + } + }}; + + // Allocate and account while attached to the query's resource + // context. The callback can run on a JVM-created thread, so merely + // relying on the calling BE thread's context would bypass query + // memory accounting. + void* address = with_resource_context([&]() { + enable_thread_catch_bad_alloc++; + Defer restore_bad_alloc_catch {[&]() { enable_thread_catch_bad_alloc--; }}; + void* allocated = _allocator.alloc(static_cast(bytes)); + try { + std::lock_guard lock(_mutex); + _allocations.emplace_back(allocated, static_cast(bytes)); + _native_reserved_bytes -= bytes; + _native_allocated_bytes += bytes; + _native_peak_allocated_bytes = + std::max(_native_peak_allocated_bytes, _native_allocated_bytes); + reservation_committed = true; + } catch (...) { + _allocator.free(allocated, static_cast(bytes)); + throw; + } + return allocated; + }); + + // NewDirectByteBuffer does not copy memory; Paimon will read/write the + // page directly. If JNI rejects the address, undo the native + // allocation and its accounting entry before returning. + jobject buffer = env->NewDirectByteBuffer(address, bytes); + if (buffer == nullptr || env->ExceptionCheck()) { + remove_and_free_page(address, static_cast(bytes)); + return nullptr; + } + return buffer; + } + + int64_t memory_limit() const { return _memory_limit; } + + int64_t native_peak_allocated_bytes() const { + std::lock_guard lock(_mutex); + return _native_peak_allocated_bytes; + } + +private: + template + auto with_resource_context(Function&& function) + -> decltype(std::forward(function)()) { + // JNI normally re-enters on the attached blocking pipeline thread. Attach + // Java-created threads explicitly too, so every allocation/free is + // charged to the query rather than to an unrelated thread context. + if (!pthread_context_ptr_init && bthread_self() == 0) { + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + if (thread_context()->is_attach_task()) { + SCOPED_SWITCH_RESOURCE_CONTEXT(_resource_context); + return std::forward(function)(); + } + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + + void release_all_pages() { + // Detach ownership from the bookkeeping vector under the lock, then + // free outside the lock. Allocator/free may invoke code that takes + // unrelated locks and must not block page accounting readers. + std::vector> allocations; + { + std::lock_guard lock(_mutex); + allocations.swap(_allocations); + _native_allocated_bytes = 0; + } + if (allocations.empty()) { + return; + } + + with_resource_context([&]() { + for (const auto& [address, bytes] : allocations) { + _allocator.free(address, bytes); + } + }); + } + + void remove_and_free_page(void* address, size_t bytes) { + // Roll back a page whose Java direct-buffer wrapper could not be + // created. The address is removed under the same lock used by the + // normal accounting path, while the potentially expensive free is + // performed after releasing it. + { + std::lock_guard lock(_mutex); + auto it = std::find_if( + _allocations.begin(), _allocations.end(), + [&](const auto& allocation) { return allocation.first == address; }); + if (it != _allocations.end()) { + _allocations.erase(it); + _native_allocated_bytes -= bytes; + } + } + with_resource_context([&]() { _allocator.free(address, bytes); }); + } + + // Query resource context used for all native allocator operations. + std::shared_ptr _resource_context; + // Immutable per-writer cap, calculated by PaimonJniMemoryManager::create. + const int64_t _memory_limit; + // Doris allocator used instead of JVM/Arrow allocation so native pages are + // visible to Doris' memory accounting and allocator hooks. + Allocator _allocator; + // Protects the allocation list and both usage counters. JNI callbacks and + // Java close/finalizer paths may arrive concurrently. + mutable std::mutex _mutex; + // Every entry is (native address, size) and remains here until released. + std::vector> _allocations; + // Bytes reserved by callbacks which have passed the local limit check but + // have not yet completed their allocator call. + int64_t _native_reserved_bytes = 0; + // Committed and high-water native page usage, respectively. + int64_t _native_allocated_bytes = 0; + int64_t _native_peak_allocated_bytes = 0; +}; + +namespace { + +jobject allocate_paimon_memory_page(JNIEnv* env, jclass, jlong manager_handle, jint bytes) { + // This is called from PaimonJniWriter's Java memory pool. The handle is + // the native manager address passed when the writer is opened; ownership + // stays with the C++ writer/backend, so this callback must never delete it. + auto* manager = reinterpret_cast(manager_handle); + if (manager == nullptr) { + jclass exception_class = env->FindClass("java/lang/IllegalStateException"); + env->ThrowNew(exception_class, "Paimon JNI memory manager is null"); + env->DeleteLocalRef(exception_class); + return nullptr; + } + try { + return manager->allocate_page(env, bytes); + } catch (const std::exception& e) { + jclass exception_class = env->FindClass("java/lang/RuntimeException"); + // Avoid dynamic allocation while reporting a failed allocation. + char message[1024]; + std::snprintf(message, sizeof(message), "Paimon JNI native page allocation failed: %.900s", + e.what()); + env->ThrowNew(exception_class, message); + env->DeleteLocalRef(exception_class); + return nullptr; + } +} + +} // namespace + +PaimonJniMemoryManager::PaimonJniMemoryManager(std::unique_ptr impl) + : _impl(std::move(impl)) {} + +PaimonJniMemoryManager::~PaimonJniMemoryManager() = default; + +Status PaimonJniMemoryManager::create(RuntimeState* state, + std::unique_ptr* manager) { + DORIS_CHECK(state != nullptr); + DORIS_CHECK(manager != nullptr); + if (state->query_mem_tracker() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot size its write buffer without a query tracker"); + } + if (state->get_query_ctx() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot allocate native memory without QueryContext"); + } + + // Each task in this sink pipeline owns one Paimon writer. Use the task count produced by the + // BE pipeline builder rather than num_local_sink, which is an FE-provided field currently set + // only for OLAP sinks. This also reflects any local-exchange parallelism chosen by the BE. + const int64_t writer_count = std::max(1, state->task_num()); + const int64_t query_limit = state->query_mem_tracker()->limit(); + const int64_t query_share = query_limit > 0 ? query_limit / writer_count : query_limit; + // Paimon requests pages lazily, can flush/preempt owners inside its MemoryPoolFactory, and may + // retain allocated pages until writer close. Bound and account those actual page allocations. + // Arrow C Data keeps the batch body in Doris-owned buffers, so there is no separate Java Arrow + // body budget to subtract from this writer's Paimon page allowance. + const int64_t configured_memory_limit = config::paimon_jni_writer_memory_pool_limit_bytes; + const int64_t memory_limit = query_share > 0 ? std::min(query_share, configured_memory_limit) + : configured_memory_limit; + if (memory_limit <= 0) { + return Status::Error( + "Paimon JNI writer has insufficient memory budget: query_limit={}, " + "sink_pipeline_task_count={}, write_buffer_limit={}", + PrettyPrinter::print_bytes(query_limit), writer_count, + PrettyPrinter::print_bytes(memory_limit)); + } + + // ResourceContext is retained by Impl for the manager's whole lifetime so JNI callbacks stay + // associated with the query even if Paimon invokes one from a Java-created thread. + auto impl = std::make_unique(state->get_query_ctx()->resource_ctx(), memory_limit); + *manager = std::unique_ptr(new PaimonJniMemoryManager(std::move(impl))); + return Status::OK(); +} + +Status PaimonJniMemoryManager::register_natives(JNIEnv* env, jclass writer_class) { + // Keep the JNI surface minimal: Java asks native code only for a page; + // all ownership, limits, and cleanup stay in PaimonJniMemoryManager. + static char allocate_name[] = "allocatePaimonMemoryPage"; + static char allocate_signature[] = "(JI)Ljava/nio/ByteBuffer;"; + static ::JNINativeMethod methods[] = { + {allocate_name, allocate_signature, + reinterpret_cast(&allocate_paimon_memory_page)}, + }; + if (env->RegisterNatives(writer_class, methods, + static_cast(sizeof(methods) / sizeof(methods[0]))) != JNI_OK) { + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, true, "JNI exception registering Paimon memory native methods: ")); + return Status::JniError("Failed to register Paimon memory native methods"); + } + return Status::OK(); +} + +jobject PaimonJniMemoryManager::allocate_page(JNIEnv* env, jint bytes) { + return _impl->allocate_page(env, bytes); +} + +int64_t PaimonJniMemoryManager::memory_limit() const { + return _impl->memory_limit(); +} + +int64_t PaimonJniMemoryManager::native_peak_allocated_bytes() const { + return _impl->native_peak_allocated_bytes(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h new file mode 100644 index 00000000000000..037d0c1952ec43 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" + +namespace doris { + +class RuntimeState; + +/// Owns the Doris-side native memory used by one Java Paimon writer. +/// +/// Paimon's sort/merge buffers are Java objects, but their page storage is +/// requested through a JNI callback. This manager is the bridge for that +/// callback: it allocates each page with Doris' allocator, exposes the page as +/// a direct ByteBuffer, tracks it until the writer is closed, and releases all +/// pages in its destructor. The native writer/backend therefore keeps this +/// manager alive for at least as long as the Java writer can access its +/// callback handle. +/// +/// The limit is a per-writer budget derived from the query limit and the sink +/// pipeline's task count. The manager accounts only for pages allocated by +/// this callback; Java heap and other Paimon-managed memory remain under their +/// respective runtimes. +class PaimonJniMemoryManager { +public: + ~PaimonJniMemoryManager(); + + /// Construct a manager whose budget is sized from the query context. + /// + /// The query must provide both a memory tracker and QueryContext. The + /// latter supplies the ResourceContext used whenever allocation/freeing + /// crosses into a JNI-created thread. + static Status create(RuntimeState* state, std::unique_ptr* manager); + + /// Register the static JNI callback used by PaimonJniWriter. + static Status register_natives(JNIEnv* env, jclass writer_class); + + /// Allocate one native page and return it as a direct ByteBuffer. + /// + /// On failure this method leaves no accounting entry behind and reports the error through the + /// JNI environment. The returned buffer remains valid until the manager is destroyed (or + /// allocation of that page is rolled back because NewDirectByteBuffer failed). + jobject allocate_page(JNIEnv* env, jint bytes); + + /// Return the immutable per-writer native page budget in bytes. + int64_t memory_limit() const; + + /// Return the high-water mark of native pages allocated by this manager. + int64_t native_peak_allocated_bytes() const; + +private: + class Impl; + + explicit PaimonJniMemoryManager(std::unique_ptr impl); + + std::unique_ptr _impl; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp new file mode 100644 index 00000000000000..3e0dd58a06fdc9 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_table_writer.h" + +#include "common/check.h" +#include "common/logging.h" +#include "core/block/block.h" +#include "core/block/materialize_block.h" +#include "exprs/vexpr_context.h" +#include "runtime/runtime_state.h" + +namespace doris { + +PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs) + : _t_sink(std::move(t_sink)), _output_expr_ctxs(output_exprs) { + DCHECK(_t_sink.__isset.paimon_table_sink); +} + +Status PaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + + // Register profile counters + _written_rows_counter = ADD_COUNTER(profile, "WrittenRows", TUnit::UNIT); + _written_bytes_counter = ADD_COUNTER(profile, "WrittenBytes", TUnit::BYTES); + _send_data_timer = ADD_TIMER(profile, "SendDataTime"); + _project_timer = ADD_CHILD_TIMER(profile, "ProjectTime", "SendDataTime"); + _file_store_write_timer = ADD_CHILD_TIMER(profile, "FileStoreWriteTime", "SendDataTime"); + _open_timer = ADD_TIMER(profile, "OpenTime"); + _close_timer = ADD_TIMER(profile, "CloseTime"); + _prepare_commit_timer = ADD_TIMER(profile, "PrepareCommitTime"); + _commit_payload_count = ADD_COUNTER(profile, "CommitPayloadCount", TUnit::UNIT); + _commit_payload_bytes_counter = ADD_COUNTER(profile, "CommitPayloadBytes", TUnit::BYTES); + + SCOPED_TIMER(_open_timer); + + // Step 1: Create the backend (JNI or FFI) based on the sink configuration. + RETURN_IF_ERROR(PaimonWriteBackendFactory::create(_t_sink.paimon_table_sink, &_backend)); + DCHECK(_backend); + // Step 2: Open the backend β€” for JNI this loads the Java class and calls PaimonJniWriter.open(). + RETURN_IF_ERROR(_backend->open(_t_sink.paimon_table_sink, state, profile)); + // Step 3: Create a lightweight writer adapter that delegates to the opened backend. + RETURN_IF_ERROR(_backend->create_writer(&_writer)); + DCHECK(_writer); + + LOG(INFO) << "PaimonTableWriter opened: backend=" << static_cast(_backend->type()) + << ", writer_scope=local_state"; + return Status::OK(); +} + +Status PaimonTableWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + SCOPED_TIMER(_send_data_timer); + + // Step 1: Apply output expressions to produce the columns selected by FE. + Block output_block; + { + SCOPED_TIMER(_project_timer); + RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(_output_expr_ctxs, block, + &output_block)); + materialize_block_inplace(output_block); + } + + COUNTER_UPDATE(_written_rows_counter, block.rows()); + COUNTER_UPDATE(_written_bytes_counter, block.bytes()); + state->update_num_rows_load_total(block.rows()); + state->update_num_bytes_load_total(block.bytes()); + + // Step 2: Delegate to the backend writer (JNI or FFI). For the JNI path + // this converts Block β†’ Arrow RecordBatch β†’ Arrow C Data β†’ Java PaimonJniWriter. + DCHECK(_writer); + { + SCOPED_TIMER(_file_store_write_timer); + RETURN_IF_ERROR(_writer->write(state, output_block)); + } + _written_rows += block.rows(); + return Status::OK(); +} + +Status PaimonTableWriter::close(Status status) { + SCOPED_TIMER(_close_timer); + + // Prepare messages first, but do not publish them until the backend confirms + // that every SDK user has stopped and its native backing memory is safe to release. + std::vector messages; + if (status.ok()) { + DCHECK(_writer); + { + SCOPED_TIMER(_prepare_commit_timer); + Status prep_st = _writer->prepare_commit(messages); + if (!prep_st.ok()) { + status = prep_st; + } + } + } + + // If prepare_commit failed or the incoming status was already an error, + // abort the writer to clean up uncommitted data files. + if (!status.ok()) { + LOG(WARNING) << "Paimon writer closing with error: " << status.to_string(); + if (_writer) { + Status abort_st = _writer->abort(); + if (!abort_st.ok()) { + LOG(WARNING) << "Paimon writer abort failed: " << abort_st.to_string(); + } + } + } + + // The adapter only owns Arrow conversion resources. Release it before closing + // the backend, whose Java close is the authoritative SDK shutdown boundary. + _writer.reset(); + + if (_backend) { + Status close_st = _backend->close(); + if (!close_st.ok()) { + if (status.ok()) { + status = close_st; + } else { + LOG(WARNING) << "Paimon backend close also failed: " << close_st.to_string(); + } + } + } + + // Only a fully prepared and cleanly stopped writer may contribute payloads + // to the FE transaction. A Java close failure therefore aborts the Doris + // transaction instead of allowing it to commit potentially unsafe output. + if (status.ok()) { + COUNTER_UPDATE(_commit_payload_count, static_cast(messages.size())); + for (const auto& msg : messages) { + DORIS_CHECK(msg.__isset.payload); + COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast(msg.payload.size())); + } + if (!messages.empty()) { + _state->add_paimon_commit_messages(messages); + LOG(INFO) << "Paimon writer closed: " << messages.size() + << " commit messages, total rows=" << _written_rows; + } + } + + _backend.reset(); + return status; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h new file mode 100644 index 00000000000000..1e9f96c181d658 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "exprs/vexpr_fwd.h" +#include "runtime/runtime_profile.h" + +namespace doris { + +class RuntimeState; + +/// Each PaimonTableSinkLocalState owns one PaimonTableWriter, which in turn +/// owns one IPaimonWriteBackend and one IPaimonWriter. Pipeline parallelism +/// therefore determines the number of independent Paimon writer sessions; +/// each writer session delegates partition and bucket routing to the Paimon +/// SDK (Java via JNI, or Rust via FFI in the future). +/// +/// Doris does NOT compute partition values or bucket ids β€” it passes complete +/// Blocks through the selected backend (JNI/FFI) to the Paimon SDK, which +/// internally computes partition values, bucket ids, and routes rows to the +/// correct file writers. +/// +/// Architecture: +/// PaimonTableSinkOperatorX +/// β”‚ sink_impl() β†’ PaimonTableWriter::write() (synchronous, no routing) +/// β–Ό +/// PaimonTableWriter (one per LocalState / pipeline instance) +/// β”‚ owns IPaimonWriteBackend (JNI or FFI) +/// β”‚ └─ create_writer() β†’ IPaimonWriter +/// β”‚ write() +/// β”‚ β†’ JNI backend: Block β†’ Arrow C Data β†’ Java Paimon SDK +/// β”‚ β†’ FFI backend: Block β†’ Rust writer (future) +/// β”‚ β†’ selected SDK owns row normalization, routing, buffering, +/// β”‚ file writing, and compaction +/// β–Ό +/// close() β†’ prepareCommit() β†’ CommitMessage[] +/// +/// Commit flow (BE only prepares messages; FE is the commit coordinator): +/// close() β†’ writer->prepare_commit() +/// β†’ collect TPaimonCommitMessage[] (DPCM-framed serialized messages) +/// β†’ RuntimeState::add_paimon_commit_messages() +/// β†’ RPC to FE Coordinator β†’ PaimonTransaction +class PaimonTableWriter final { +public: + PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs); + + ~PaimonTableWriter() = default; + + Status open(RuntimeState* state, RuntimeProfile* profile); + + Status write(RuntimeState* state, Block& block); + + Status close(Status status); + +private: + TDataSink _t_sink; + const VExprContextSPtrs& _output_expr_ctxs; + RuntimeState* _state = nullptr; + int64_t _written_rows = 0; + + // Backend owns the JNI/FFI connection and creates the writer adapter. + // Both are scoped to this PaimonTableWriter (one per LocalState). + std::unique_ptr _backend; + std::unique_ptr _writer; + + // Profile counters + RuntimeProfile::Counter* _written_rows_counter = nullptr; + RuntimeProfile::Counter* _written_bytes_counter = nullptr; + RuntimeProfile::Counter* _send_data_timer = nullptr; + RuntimeProfile::Counter* _project_timer = nullptr; + RuntimeProfile::Counter* _file_store_write_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _prepare_commit_timer = nullptr; + RuntimeProfile::Counter* _commit_payload_count = nullptr; + RuntimeProfile::Counter* _commit_payload_bytes_counter = nullptr; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h new file mode 100644 index 00000000000000..44e2667c7c103a --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" + +namespace doris { + +class RuntimeState; +class RuntimeProfile; + +enum class PaimonBackendType { + JNI, // Java via JNI (PaimonJniWriter) + FFI, // Rust via FFI (placeholder, not yet implemented) +}; + +/// Writer contract implemented by one SDK writer adapter. Each +/// PaimonTableWriter owns one IPaimonWriter, which delegates to the +/// underlying Paimon SDK (Java JNI or Rust FFI). Partition and bucket +/// routing happens inside the selected SDK backend. +/// +/// Lifecycle: created by IPaimonWriteBackend::create_writer() after the +/// backend is opened; used for the duration of one pipeline instance. +class IPaimonWriter { +public: + virtual ~IPaimonWriter() = default; + + /// Write a projected Block to the Paimon SDK. + /// For the JNI path: Block β†’ Arrow RecordBatch β†’ Arrow C Data β†’ Java. + virtual Status write(RuntimeState* state, Block& block) = 0; + + /// Flush all buffered data, close files, and collect serialized commit + /// messages (DPCM-framed). Called once at EOS. + virtual Status prepare_commit(std::vector& messages) = 0; + + /// Discard written data files on error. Called when write or prepare_commit fails. + virtual Status abort() = 0; +}; + +/// Backend boundary for creating writers via JNI (Java) or FFI (Rust). +/// +/// The backend owns the connection/session to the external runtime: +/// - JNI: owns the JVM class reference, method IDs, and the Java writer object. +/// - FFI: (future) owns the Rust FFI handle. +/// +/// Each backend creates one or more IPaimonWriter adapters that share the +/// same underlying connection. Snapshot commit is deliberately excluded from +/// this boundary: BE only prepares commit messages (byte payloads), while FE +/// PaimonTransaction is the single commit coordinator. +class IPaimonWriteBackend { +public: + virtual ~IPaimonWriteBackend() = default; + + /// Initialize the backend connection. For JNI this loads the writer class, + /// creates the Java object, and calls PaimonJniWriter.open(). + virtual Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) = 0; + + /// Create a lightweight writer adapter that delegates to this backend. + virtual Status create_writer(std::unique_ptr* writer) = 0; + + /// Stop all SDK users and release backend resources. + /// + /// A successful return is the ownership boundary after which native memory + /// backing SDK buffers can be reclaimed safely. Callers must not publish + /// prepared commit messages until this succeeds. + virtual Status close() = 0; + + virtual PaimonBackendType type() const = 0; +}; + +/// Factory that selects and creates the appropriate write backend. +/// +/// Backend selection is based on TPaimonTableSink.backend_type: +/// - Default (unset or JNI): JniPaimonWriteBackend +/// - FFI: FfiPaimonWriteBackend (placeholder for future Rust writer) +class PaimonWriteBackendFactory { +public: + /// Create a backend instance based on the sink configuration. + static Status create(const TPaimonTableSink& sink, + std::unique_ptr* backend); + + /// Determine which backend type to use for the given sink. + static PaimonBackendType select_backend_type(const TPaimonTableSink& sink); +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp new file mode 100644 index 00000000000000..087228abbe5d2b --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +Status PaimonWriteBackendFactory::create(const TPaimonTableSink& sink, + std::unique_ptr* backend) { + switch (select_backend_type(sink)) { + case PaimonBackendType::JNI: + *backend = std::make_unique(); + return Status::OK(); + case PaimonBackendType::FFI: + *backend = std::make_unique(); + return Status::OK(); + } + return Status::InternalError("Unknown Paimon write backend"); +} + +PaimonBackendType PaimonWriteBackendFactory::select_backend_type(const TPaimonTableSink& sink) { + if (sink.__isset.backend_type && sink.backend_type == TPaimonWriteBackendType::FFI) { + return PaimonBackendType::FFI; + } + return PaimonBackendType::JNI; +} + +} // namespace doris diff --git a/be/src/exec/spill/spill_file_manager.cpp b/be/src/exec/spill/spill_file_manager.cpp index eb56fb14a132c1..f5629d3b501f42 100644 --- a/be/src/exec/spill/spill_file_manager.cpp +++ b/be/src/exec/spill/spill_file_manager.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -31,14 +32,105 @@ #include "exec/spill/spill_file.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "runtime/query_context.h" #include "storage/olap_define.h" #include "util/debug_points.h" #include "util/parse_util.h" #include "util/pretty_printer.h" #include "util/time.h" +#include "util/uid_util.h" namespace doris { +ExternalSpillSession::ExternalSpillSession(SpillFileManager* manager, QueryContext* query_context, + std::string relative_path) + : _manager(manager), + _query_context(query_context->weak_from_this()), + _resource_context(query_context->resource_ctx()), + _query_id(print_id(query_context->query_id())), + _relative_path(std::move(relative_path)) { + DCHECK(_manager != nullptr); + DCHECK(!_query_context.expired()); + DCHECK(_resource_context != nullptr); +} + +ExternalSpillSession::~ExternalSpillSession() { + _manager->_release_external_spill_session(this); +} + +Status ExternalSpillSession::get_paths(std::vector* paths) { + if (paths == nullptr) { + return Status::InvalidArgument("External spill paths output must not be null"); + } + std::lock_guard lock(_mutex); + if (_data_dir == nullptr) { + RETURN_IF_ERROR(_manager->_initialize_external_spill_session(this)); + } + *paths = {_path}; + return Status::OK(); +} + +bool ExternalSpillSession::_contains(const std::string& path) const { + return path == _path || + (path.size() > _path.size() && path.starts_with(_path) && path[_path.size()] == '/'); +} + +Status ExternalSpillSession::reserve(const std::string& path, int64_t bytes) { + if (bytes <= 0) { + return Status::InvalidArgument("External spill reservation must be positive: {}", bytes); + } + + std::lock_guard lock(_mutex); + if (_data_dir == nullptr || !_contains(path)) { + return Status::InvalidArgument("External spill path is not managed by Doris: {}", path); + } + if (bytes > std::numeric_limits::max() - _accounted_bytes) { + return Status::InvalidArgument("External spill reservation overflows: bytes={}", bytes); + } + if (_data_dir->reach_capacity_limit(bytes)) { + return Status::Error( + "External spill write exceeds the Doris spill storage limit: path={}, bytes={}", + path, bytes); + } + // Match SpillFileWriter: check capacity before the write, then account the accepted bytes. + _data_dir->update_spill_data_usage(bytes); + _accounted_bytes += bytes; + return Status::OK(); +} + +void ExternalSpillSession::update_accounting(const std::string& path, int64_t current_bytes_delta, + int64_t write_bytes, int64_t read_bytes) { + int64_t released_bytes = 0; + SpillDataDir* data_dir = nullptr; + { + std::lock_guard lock(_mutex); + if (_data_dir == nullptr || !_contains(path)) { + LOG(WARNING) << "Ignoring accounting for unmanaged external spill path: " << path; + return; + } + data_dir = _data_dir; + if (current_bytes_delta < 0) { + const int64_t requested_release = + current_bytes_delta == std::numeric_limits::min() + ? std::numeric_limits::max() + : -current_bytes_delta; + released_bytes = std::min(requested_release, _accounted_bytes); + _accounted_bytes -= released_bytes; + } + } + if (released_bytes > 0) { + data_dir->update_spill_data_usage(-released_bytes); + } + if (write_bytes > 0) { + _resource_context->io_context()->update_spill_write_bytes_to_local_storage(write_bytes); + _manager->update_spill_write_bytes(write_bytes); + } + if (read_bytes > 0) { + _resource_context->io_context()->update_spill_read_bytes_from_local_storage(read_bytes); + _manager->update_spill_read_bytes(read_bytes); + } +} + SpillFileManager::~SpillFileManager() { // QueryContext destruction can still queue failed deletions after stop(), for example while // VDataStreamMgr is being destroyed. Retry them once more before dropping the in-memory state. @@ -172,6 +264,76 @@ Status SpillFileManager::create_spill_file(const std::string& relative_path, return Status::OK(); } +Status SpillFileManager::create_external_spill_session( + const std::string& relative_path, QueryContext* query_context, + std::unique_ptr* spill_session) { + if (query_context == nullptr || spill_session == nullptr) { + return Status::InvalidArgument( + "External spill session requires QueryContext and output session"); + } + + spill_session->reset(new ExternalSpillSession(this, query_context, relative_path)); + return Status::OK(); +} + +Status SpillFileManager::_initialize_external_spill_session(ExternalSpillSession* spill_session) { + auto query_context = spill_session->_query_context.lock(); + if (query_context == nullptr) { + return Status::Cancelled("Query ended before the external spill session was initialized"); + } + auto* data_dir = _get_store_for_spill(); + if (data_dir == nullptr) { + return Status::Error( + "no available disk can be used for spill."); + } + + const auto query_dir = data_dir->get_spill_data_path(spill_session->_query_id); + { + // QueryContext teardown uses the regular pending-deletion path while this lease is live. + std::lock_guard lock(_pending_query_spill_directories_mutex); + ++_external_spill_directory_leases[query_dir]; + } + query_context->record_spill_data_dir(data_dir); + spill_session->_data_dir = data_dir; + spill_session->_path = query_dir + "/" + spill_session->_relative_path; + return Status::OK(); +} + +void SpillFileManager::_release_external_spill_session(ExternalSpillSession* spill_session) { + std::lock_guard session_lock(spill_session->_mutex); + if (spill_session->_data_dir == nullptr) { + return; + } + + if (spill_session->_accounted_bytes > 0) { + // Match SpillFile::gc(): QueryContext owns physical cleanup and its retry path, while the + // writer releases logical usage when its lifetime ends. + spill_session->_data_dir->update_spill_data_usage(-spill_session->_accounted_bytes); + spill_session->_accounted_bytes = 0; + } + + const auto query_dir = spill_session->_data_dir->get_spill_data_path(spill_session->_query_id); + std::lock_guard directory_lock(_pending_query_spill_directories_mutex); + auto it = _external_spill_directory_leases.find(query_dir); + DCHECK(it != _external_spill_directory_leases.end()); + if (it == _external_spill_directory_leases.end()) { + return; + } + DCHECK_GT(it->second, 0); + if (--it->second == 0) { + _external_spill_directory_leases.erase(it); + } +} + +SpillDataDir* SpillFileManager::_get_store_for_spill() { + auto data_dirs = _get_stores_for_spill(TStorageMedium::type::SSD); + if (data_dirs.empty()) { + data_dirs = _get_stores_for_spill(TStorageMedium::type::HDD); + } + // Select the first available data dir (sorted by usage ascending). + return data_dirs.empty() ? nullptr : data_dirs.front(); +} + void SpillFileManager::delete_spill_file(SpillFileSPtr spill_file) { if (!spill_file) { LOG(WARNING) << "[spill][delete] null spill_file"; @@ -196,6 +358,13 @@ void SpillFileManager::delete_query_spill_directory(const std::string& query_id, Status SpillFileManager::_try_delete_query_spill_directory( const PendingQuerySpillDirectory& pending_directory) { + { + std::lock_guard lock(_pending_query_spill_directories_mutex); + if (_external_spill_directory_leases.contains(pending_directory.query_dir)) { + return Status::InternalError("external spill directory is still in use: {}", + pending_directory.query_dir); + } + } DBUG_EXECUTE_IF("fault_inject::spill_file_manager::delete_query_spill_directory", { return Status::Error("injected query spill directory deletion failure"); }); diff --git a/be/src/exec/spill/spill_file_manager.h b/be/src/exec/spill/spill_file_manager.h index 1e3042d6ad3aaa..db6f2dced91435 100644 --- a/be/src/exec/spill/spill_file_manager.h +++ b/be/src/exec/spill/spill_file_manager.h @@ -40,6 +40,8 @@ class AtomicGauge; using UIntGauge = AtomicGauge; class MetricEntity; struct MetricPrototype; +class QueryContext; +class ResourceContext; class SpillFileManager; class SpillDataDir { @@ -112,6 +114,38 @@ class SpillDataDir { IntGauge* spill_disk_has_spill_data = nullptr; IntGauge* spill_disk_has_spill_gc_data = nullptr; }; + +// Adapts one external writer to the same root selection, capacity accounting and query cleanup +// used by Doris spill files. +class ExternalSpillSession { +public: + ~ExternalSpillSession(); + + Status get_paths(std::vector* paths); + + Status reserve(const std::string& path, int64_t bytes); + + void update_accounting(const std::string& path, int64_t current_bytes_delta, + int64_t write_bytes, int64_t read_bytes); + +private: + friend class SpillFileManager; + + ExternalSpillSession(SpillFileManager* manager, QueryContext* query_context, + std::string relative_path); + bool _contains(const std::string& path) const; + + SpillFileManager* _manager; + std::weak_ptr _query_context; + std::shared_ptr _resource_context; + std::string _query_id; + std::string _relative_path; + SpillDataDir* _data_dir = nullptr; + std::string _path; + int64_t _accounted_bytes = 0; + std::mutex _mutex; +}; + class SpillFileManager { public: ~SpillFileManager(); @@ -127,6 +161,12 @@ class SpillFileManager { // e.g. "query_id/sort-node_id-task_id-unique_id" Status create_spill_file(const std::string& relative_path, SpillFileSPtr& spill_file); + // Create a lazy managed session for an external spill implementation. A spill root is selected + // and registered only when the external implementation first requests its path. + Status create_external_spill_session(const std::string& relative_path, + QueryContext* query_context, + std::unique_ptr* spill_session); + /// Get a unique ID for constructing spill file paths. uint64_t next_id() { return id_++; } @@ -144,6 +184,8 @@ class SpillFileManager { void update_spill_read_bytes(int64_t bytes) { _spill_read_bytes_counter->increment(bytes); } private: + friend class ExternalSpillSession; + struct PendingQuerySpillDirectory { int failed_count {0}; std::string query_dir; @@ -154,15 +196,22 @@ class SpillFileManager { void _spill_gc_thread_callback(); Status _try_delete_query_spill_directory(const PendingQuerySpillDirectory& pending_directory); void _retry_pending_query_spill_directories(); + Status _initialize_external_spill_session(ExternalSpillSession* spill_session); + void _release_external_spill_session(ExternalSpillSession* spill_session); std::vector _get_stores_for_spill(TStorageMedium::type storage_medium); + SpillDataDir* _get_store_for_spill(); std::unordered_map> _spill_store_map; CountDownLatch _stop_background_threads_latch; std::shared_ptr _spill_gc_thread; + // Query cleanup uses the regular pending-deletion path. External leases only defer deletion + // while an SDK task can still access the same query directory; filesystem I/O never holds this + // mutex. std::mutex _pending_query_spill_directories_mutex; std::vector _pending_query_spill_directories; + std::unordered_map _external_spill_directory_leases; std::atomic_uint64_t id_ = 0; diff --git a/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp b/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp index e35c9b1983b896..66a61f2da9623d 100644 --- a/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp +++ b/be/src/exprs/function/cast/variant_v2/cast_array_to_variant.cpp @@ -137,7 +137,9 @@ Status build_array_node_plan(const ColumnPtr& source, const DataTypePtr& source_ Status build_array_leaf_plan(const ColumnPtr& source, PrimitiveType primitive, ArrayEncodePlan* plan) { - if (primitive == INVALID_TYPE && source->empty()) { + if (primitive == INVALID_TYPE) { + // DataTypeNothing is represented by the element null map, including non-empty + // expressions such as array(NULL). return Status::OK(); } else if (primitive == TYPE_VARIANT) { const auto* variant = check_and_get_column(source.get()); @@ -196,7 +198,7 @@ void append_array_value(const ArrayEncodePlan& plan, size_t index, VariantBatchB } else if (plan.jsonb_leaf != nullptr) { jsonb_to_variant(plan.jsonb_leaf->get_data_at(index), *row); } else { - DORIS_CHECK(false) << "empty Array leaf unexpectedly contains a value"; + DORIS_CHECK(false) << "Array Variant V2 leaf has no encoder"; } return; } diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp index 9dc935e0b62da5..53dd561413628d 100644 --- a/be/src/format/jni/jni_data_bridge.cpp +++ b/be/src/format/jni/jni_data_bridge.cpp @@ -29,6 +29,7 @@ #include "core/column/column_string.h" #include "core/column/column_struct.h" #include "core/column/column_varbinary.h" +#include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" @@ -148,6 +149,9 @@ Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_co case PrimitiveType::TYPE_VARBINARY: status = _fill_varbinary_column(address, data_column, num_rows); break; + case PrimitiveType::TYPE_VARIANT: + status = _fill_variant_v2_column(address, data_column, num_rows); + break; default: status = Status::InvalidArgument("Unsupported type {} in jni scanner", data_type->get_name()); @@ -179,6 +183,26 @@ Status JniDataBridge::_fill_varbinary_column(TableMetaAddress& address, return Status::OK(); } +Status JniDataBridge::_fill_variant_v2_column(TableMetaAddress& address, + MutableColumnPtr& doris_column, size_t num_rows) { + const auto metadata_count = static_cast(address.next_meta_as_long()); + const auto* metadata_offsets = reinterpret_cast(address.next_meta_as_ptr()); + const auto* metadata_bytes = reinterpret_cast(address.next_meta_as_ptr()); + const auto* metadata_ids = reinterpret_cast(address.next_meta_as_ptr()); + const auto* value_offsets = reinterpret_cast(address.next_meta_as_ptr()); + const auto* value_bytes = reinterpret_cast(address.next_meta_as_ptr()); + + auto& variant_column = assert_cast(*doris_column); + variant_column.insert_encoded_rows({ + .metadata_bytes = {metadata_bytes, metadata_offsets[metadata_count]}, + .metadata_offsets = {metadata_offsets, metadata_count + 1}, + .meta_ids = {metadata_ids, num_rows}, + .value_bytes = {value_bytes, value_offsets[num_rows]}, + .value_offsets = {value_offsets, num_rows + 1}, + }); + return Status::OK(); +} + Status JniDataBridge::_fill_string_column(TableMetaAddress& address, MutableColumnPtr& doris_column, size_t num_rows) { auto& string_col = static_cast(*doris_column); @@ -358,6 +382,8 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) { } case TYPE_VARBINARY: return "varbinary"; + case TYPE_VARIANT: + return "variant"; // bitmap, hll, quantile_state, jsonb are transferred as strings via JNI case TYPE_BITMAP: [[fallthrough]]; @@ -433,6 +459,8 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr& << assert_cast(remove_nullable(data_type).get())->len() << ")"; return buffer.str(); + case TYPE_VARIANT: + return "variant"; case TYPE_DECIMALV2: { buffer << "decimalv2(" << DecimalV2Value::PRECISION << "," << DecimalV2Value::SCALE << ")"; return buffer.str(); diff --git a/be/src/format/jni/jni_data_bridge.h b/be/src/format/jni/jni_data_bridge.h index e037ffec3d4d5a..267d0a1711c06c 100644 --- a/be/src/format/jni/jni_data_bridge.h +++ b/be/src/format/jni/jni_data_bridge.h @@ -154,6 +154,9 @@ class JniDataBridge { static Status _fill_varbinary_column(TableMetaAddress& address, MutableColumnPtr& doris_column, size_t num_rows); + static Status _fill_variant_v2_column(TableMetaAddress& address, MutableColumnPtr& doris_column, + size_t num_rows); + static Status _fill_array_column(TableMetaAddress& address, MutableColumnPtr& doris_column, const DataTypePtr& data_type, size_t num_rows); diff --git a/be/src/format/transformer/iceberg_partition_function.cpp b/be/src/format/transformer/iceberg_partition_function.cpp index 4d134b062f3236..56ee559a11540d 100644 --- a/be/src/format/transformer/iceberg_partition_function.cpp +++ b/be/src/format/transformer/iceberg_partition_function.cpp @@ -19,7 +19,6 @@ #include "common/cast_set.h" #include "common/exception.h" -#include "common/logging.h" #include "common/status.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" @@ -139,10 +138,8 @@ Status IcebergInsertPartitionFunction::open(RuntimeState* state) { field.transform); field.transformer = PartitionColumnTransforms::create(partition_field, source_type); } catch (const doris::Exception& e) { - LOG(WARNING) << "Merge partitioning fallback to RR: " << e.what(); - _fallback_to_random = true; - _partition_fields.clear(); - break; + return Status::NotSupported("Unsupported Iceberg partition transform: {}", + e.what()); } } } @@ -152,9 +149,6 @@ Status IcebergInsertPartitionFunction::open(RuntimeState* state) { Status IcebergInsertPartitionFunction::get_partitions(RuntimeState* /*state*/, Block* block, size_t partition_count, std::vector& partitions) const { - if (_fallback_to_random) { - return Status::InternalError("Merge partitioning fallback to random"); - } if (partition_count == 0) { return Status::InternalError("Partition count is zero"); } @@ -193,7 +187,6 @@ Status IcebergInsertPartitionFunction::clone(RuntimeState* state, new_function->_partition_fields.emplace_back(std::move(field)); } } - new_function->_fallback_to_random = _fallback_to_random; return Status::OK(); } diff --git a/be/src/format/transformer/iceberg_partition_function.h b/be/src/format/transformer/iceberg_partition_function.h index 0ab36c91a0ee91..1925edf152d780 100644 --- a/be/src/format/transformer/iceberg_partition_function.h +++ b/be/src/format/transformer/iceberg_partition_function.h @@ -44,8 +44,6 @@ class IcebergInsertPartitionFunction final : public PartitionFunction { HashValType partition_count() const override { return _partition_count; } Status clone(RuntimeState* state, std::unique_ptr& function) const override; - bool fallback_to_random() const { return _fallback_to_random; } - private: struct InsertPartitionField { std::string transform; @@ -69,7 +67,6 @@ class IcebergInsertPartitionFunction final : public PartitionFunction { std::vector _partition_fields_spec; VExprContextSPtrs _partition_expr_ctxs; std::vector _partition_fields; - bool _fallback_to_random = false; }; class IcebergDeletePartitionFunction final : public PartitionFunction { diff --git a/be/src/format/transformer/merge_partitioner.cpp b/be/src/format/transformer/merge_partitioner.cpp index 89cf830d6bba53..ef67960514d03a 100644 --- a/be/src/format/transformer/merge_partitioner.cpp +++ b/be/src/format/transformer/merge_partitioner.cpp @@ -33,16 +33,6 @@ namespace doris { -namespace { -int64_t scale_threshold_by_task(int64_t value, int task_num) { - if (task_num <= 0) { - return value; - } - int64_t scaled = value / task_num; - return scaled == 0 ? value : scaled; -} -} // namespace - MergePartitioner::MergePartitioner(size_t partition_count, const TMergePartitionInfo& merge_info, bool use_new_shuffle_hash_method) : PartitionerBase(static_cast(partition_count)), @@ -92,11 +82,12 @@ Status MergePartitioner::prepare(RuntimeState* state, const RowDescriptor& row_d Status MergePartitioner::open(RuntimeState* state) { RETURN_IF_ERROR(VExpr::open(_operation_expr_ctxs, state)); if (_insert_partition_function != nullptr) { - RETURN_IF_ERROR(_insert_partition_function->open(state)); - if (auto* insert_function = - dynamic_cast(_insert_partition_function.get()); - insert_function != nullptr && insert_function->fallback_to_random()) { + Status status = _insert_partition_function->open(state); + if (status.is()) { + LOG(WARNING) << "Merge partitioning fallback to RR: " << status; _insert_random = true; + } else { + RETURN_IF_ERROR(status); } } if (_delete_partition_function != nullptr) { @@ -183,7 +174,7 @@ Status MergePartitioner::do_partitioning(RuntimeState* state, Block* block) cons _insert_writer_count = static_cast(_partition_count); } } else if (_enable_insert_rebalance) { - _apply_insert_rebalance(ops, insert_hashes, block->bytes()); + RETURN_IF_ERROR(_apply_insert_rebalance(ops, insert_hashes, block->bytes())); } } @@ -276,14 +267,14 @@ Status MergePartitioner::clone(RuntimeState* state, std::unique_ptr& ops, - std::vector& insert_hashes, - size_t block_bytes) const { +Status MergePartitioner::_apply_insert_rebalance(const std::vector& ops, + std::vector& insert_hashes, + size_t block_bytes) const { if (!_enable_insert_rebalance || _insert_writer_assigner == nullptr) { - return; + return Status::OK(); } if (insert_hashes.empty() || _insert_partition_count == 0) { - return; + return Status::OK(); } std::vector mask(ops.size(), 0); for (size_t i = 0; i < ops.size(); ++i) { @@ -291,7 +282,8 @@ void MergePartitioner::_apply_insert_rebalance(const std::vector& ops, mask[i] = 1; } } - _insert_writer_assigner->assign(insert_hashes, &mask, ops.size(), block_bytes, insert_hashes); + return _insert_writer_assigner->assign(insert_hashes, &mask, ops.size(), block_bytes, + insert_hashes); } void MergePartitioner::_init_insert_scaling(RuntimeState* state) { @@ -324,10 +316,10 @@ void MergePartitioner::_init_insert_scaling(RuntimeState* state) { } int task_num = state == nullptr ? 0 : state->task_num(); - int64_t min_partition_threshold = scale_threshold_by_task( + int64_t min_partition_threshold = scale_writer_threshold_by_task( config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold, task_num); - int64_t min_data_threshold = scale_threshold_by_task( + int64_t min_data_threshold = scale_writer_threshold_by_task( config::table_sink_partition_write_min_data_processed_rebalance_threshold, task_num); _insert_writer_assigner = std::make_unique( diff --git a/be/src/format/transformer/merge_partitioner.h b/be/src/format/transformer/merge_partitioner.h index 14619c8eca2f3e..3cc7420344bfcf 100644 --- a/be/src/format/transformer/merge_partitioner.h +++ b/be/src/format/transformer/merge_partitioner.h @@ -22,7 +22,7 @@ #include #include "exec/partitioner/partitioner.h" -#include "format/transformer/writer_assigner.h" +#include "exec/partitioner/writer_assigner.h" namespace doris { @@ -40,8 +40,8 @@ class MergePartitioner final : public PartitionerBase { Status clone(RuntimeState* state, std::unique_ptr& partitioner) override; private: - void _apply_insert_rebalance(const std::vector& ops, - std::vector& insert_hashes, size_t block_bytes) const; + Status _apply_insert_rebalance(const std::vector& ops, + std::vector& insert_hashes, size_t block_bytes) const; void _init_insert_scaling(RuntimeState* state); uint32_t _next_rr_channel() const; Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src, diff --git a/be/src/format/transformer/writer_assigner.h b/be/src/format/transformer/writer_assigner.h deleted file mode 100644 index 4c22862178b8a2..00000000000000 --- a/be/src/format/transformer/writer_assigner.h +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "exec/connector/skewed_partition_rebalancer.h" - -namespace doris { - -class WriterAssigner { -public: - virtual ~WriterAssigner() = default; - - virtual void assign(const std::vector& partition_ids, - const std::vector* mask, size_t rows, size_t block_bytes, - std::vector& writer_ids) = 0; -}; - -class IdentityWriterAssigner final : public WriterAssigner { -public: - void assign(const std::vector& partition_ids, const std::vector* mask, - size_t rows, size_t /*block_bytes*/, std::vector& writer_ids) override { - if (rows == 0) { - return; - } - if (writer_ids.size() != rows && &writer_ids != &partition_ids) { - writer_ids.resize(rows); - } - if (mask == nullptr) { - for (size_t i = 0; i < rows; ++i) { - writer_ids[i] = partition_ids[i]; - } - return; - } - for (size_t i = 0; i < rows; ++i) { - if ((*mask)[i] == 0) { - continue; - } - writer_ids[i] = partition_ids[i]; - } - } -}; - -class SkewedWriterAssigner final : public WriterAssigner { -public: - SkewedWriterAssigner(int partition_count, int task_count, int task_bucket_count, - long min_partition_data_processed_rebalance_threshold, - long min_data_processed_rebalance_threshold) - : _rebalancer(partition_count, task_count, task_bucket_count, - min_partition_data_processed_rebalance_threshold, - min_data_processed_rebalance_threshold), - _partition_row_counts(partition_count, 0), - _partition_writer_ids(partition_count, -1), - _partition_writer_indexes(partition_count, 0) {} - - void assign(const std::vector& partition_ids, const std::vector* mask, - size_t rows, size_t block_bytes, std::vector& writer_ids) override { - if (rows == 0 || _partition_row_counts.empty()) { - return; - } - if (writer_ids.size() != rows && &writer_ids != &partition_ids) { - writer_ids.resize(rows); - } - - std::fill(_partition_row_counts.begin(), _partition_row_counts.end(), 0); - std::fill(_partition_writer_ids.begin(), _partition_writer_ids.end(), -1); - _rebalancer.rebalance(); - - const size_t partition_count = _partition_row_counts.size(); - for (size_t i = 0; i < rows; ++i) { - if (mask != nullptr && (*mask)[i] == 0) { - continue; - } - const uint32_t partition_id = partition_ids[i]; - if (partition_id >= partition_count) { - continue; - } - _partition_row_counts[partition_id] += 1; - int writer_id = _partition_writer_ids[partition_id]; - if (writer_id == -1) { - writer_id = _get_next_writer_id(partition_id); - _partition_writer_ids[partition_id] = writer_id; - } - writer_ids[i] = static_cast(writer_id); - } - - for (size_t i = 0; i < partition_count; ++i) { - if (_partition_row_counts[i] > 0) { - _rebalancer.add_partition_row_count(static_cast(i), _partition_row_counts[i]); - } - } - _rebalancer.add_data_processed(static_cast(block_bytes)); - } - -private: - int _get_next_writer_id(uint32_t partition_id) { - return _rebalancer.get_task_id(partition_id, _partition_writer_indexes[partition_id]++); - } - - SkewedPartitionRebalancer _rebalancer; - std::vector _partition_row_counts; - std::vector _partition_writer_ids; - std::vector _partition_writer_indexes; -}; - -} // namespace doris diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp b/be/src/format_v2/jni/paimon_jni_reader.cpp index 01f33c5cdf0396..66f0870eb39094 100644 --- a/be/src/format_v2/jni/paimon_jni_reader.cpp +++ b/be/src/format_v2/jni/paimon_jni_reader.cpp @@ -32,6 +32,7 @@ constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop."; constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager"; constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir"; constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; +constexpr std::string_view VARIANT_ACCESS_PATH_PREFIX = "variant_access_path."; const std::string* get_paimon_predicate(const TFileScanRangeParams* scan_params, const TPaimonFileDesc& paimon_params) { @@ -135,6 +136,14 @@ Status PaimonJniReader::build_scanner_params(std::map* (*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second; } } + for (size_t column_idx = 0; column_idx < _projected_columns.size(); ++column_idx) { + const auto& access_paths = _projected_columns[column_idx].variant_access_paths; + for (size_t path_idx = 0; path_idx < access_paths.size(); ++path_idx) { + (*params)[std::string(VARIANT_ACCESS_PATH_PREFIX) + std::to_string(column_idx) + "." + + std::to_string(path_idx)] = + JniDataBridge::encode_schema_values(access_paths[path_idx]); + } + } // TODO: Remove legacy split-level paimon_predicate, paimon_options and hadoop_conf from thrift // after the minimum supported FE always sends their scan-level replacements. return Status::OK(); diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 16f4623bb1bafb..f9da3adfbefe80 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -286,7 +286,7 @@ class ScopedBoolOverride { Status validate_variant_layout(const NativeFieldSchema& group_field, std::optional specification_version, - bool allow_optional_shredded_metadata) { + bool allow_paimon_shredded_layout) { if (specification_version.has_value() && *specification_version != 1) { return Status::NotSupported("Parquet Variant specification version {} is not supported", *specification_version); @@ -332,7 +332,7 @@ Status validate_variant_layout(const NativeFieldSchema& group_field, // unannotated overrides; row materialization still rejects null metadata for a non-null value. const bool valid_metadata_repetition = metadata_repetition == tparquet::FieldRepetitionType::REQUIRED || - (allow_optional_shredded_metadata && typed_value != nullptr && + (allow_paimon_shredded_layout && typed_value != nullptr && metadata_repetition == tparquet::FieldRepetitionType::OPTIONAL); if (!metadata->children.empty() || metadata->physical_type != tparquet::Type::BYTE_ARRAY || !valid_metadata_repetition) { @@ -360,8 +360,17 @@ Status validate_variant_layout(const NativeFieldSchema& group_field, std::function validate_typed_value; std::function validate_wrapper; validate_wrapper = [&](const NativeFieldSchema& wrapper, WrapperContext context) -> Status { - if (!wrapper.parquet_schema.__isset.repetition_type || - wrapper.parquet_schema.repetition_type != tparquet::FieldRepetitionType::REQUIRED) { + const bool valid_wrapper_repetition = + wrapper.parquet_schema.__isset.repetition_type && + (wrapper.parquet_schema.repetition_type == + tparquet::FieldRepetitionType::REQUIRED || + (allow_paimon_shredded_layout && wrapper.parquet_schema.repetition_type == + tparquet::FieldRepetitionType::OPTIONAL)); + // The Parquet Variant specification requires wrapper groups. Paimon's unannotated + // physical carrier makes them optional, so accept that representation only through the + // table-format override. Materialization still rejects an actually null array element; + // an absent object wrapper represents a missing key. + if (!valid_wrapper_repetition) { return Status::Corruption("Parquet Variant shredded wrapper {} must be required", wrapper.name); } @@ -399,13 +408,23 @@ Status validate_variant_layout(const NativeFieldSchema& group_field, "Parquet Variant object wrapper {} requires an optional value child", wrapper.name); } + // Paimon makes this leaf required because a fallback-only array element has no typed + // carrier; keep the exception scoped to that exact unannotated layout. + const bool allow_required_fallback = allow_paimon_shredded_layout && + context == WrapperContext::ARRAY_ELEMENT && + typed == nullptr; + const bool valid_fallback_repetition = + fallback != nullptr && fallback->parquet_schema.__isset.repetition_type && + (fallback->parquet_schema.repetition_type == + tparquet::FieldRepetitionType::OPTIONAL || + (allow_required_fallback && fallback->parquet_schema.repetition_type == + tparquet::FieldRepetitionType::REQUIRED)); if (fallback != nullptr && (!fallback->children.empty() || fallback->physical_type != tparquet::Type::BYTE_ARRAY || - !fallback->parquet_schema.__isset.repetition_type || - fallback->parquet_schema.repetition_type != tparquet::FieldRepetitionType::OPTIONAL)) { + !valid_fallback_repetition)) { return Status::Corruption( - "Parquet Variant wrapper {} value must be an optional BYTE_ARRAY", - wrapper.name); + "Parquet Variant wrapper {} value must be an {} BYTE_ARRAY", wrapper.name, + allow_required_fallback ? "optional or required" : "optional"); } if (typed != nullptr) { if (!typed->parquet_schema.__isset.repetition_type || diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h index 47d58f5f1cec4a..3c2de164246ace 100644 --- a/be/src/format_v2/parquet/native_schema_desc.h +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -92,7 +92,7 @@ struct NativeFieldSchema { Status validate_variant_layout(const NativeFieldSchema& group_field, std::optional specification_version = std::nullopt, - bool allow_optional_shredded_metadata = false); + bool allow_paimon_shredded_layout = false); // V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. class NativeFieldDescriptor { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index bd9b849ba7a4de..1aca3b3a73399f 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -573,6 +573,17 @@ class RuntimeState { _mc_commit_datas.emplace_back(mc_commit_data); } + std::vector paimon_commit_messages() const { + std::lock_guard lock(_paimon_commit_messages_mutex); + return _paimon_commit_messages; + } + + void add_paimon_commit_messages(const std::vector& commit_messages) { + std::lock_guard lock(_paimon_commit_messages_mutex); + _paimon_commit_messages.insert(_paimon_commit_messages.end(), commit_messages.begin(), + commit_messages.end()); + } + // local runtime filter mgr, the runtime filter do not have remote target or // not need local merge should regist here. the instance exec finish, the local // runtime filter mgr can release the memory of local runtime filter @@ -1012,6 +1023,9 @@ class RuntimeState { mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; + mutable std::mutex _paimon_commit_messages_mutex; + std::vector _paimon_commit_messages; + std::vector> _op_id_to_local_state; std::unique_ptr _sink_local_state; diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index 0b54a8cb11dd6e..948436c607d31f 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -606,6 +606,9 @@ class Object { bool uninitialized() const { return _obj == nullptr; } + // Access the JNI handle without changing ownership. + jobject get() const { return _obj; } + void reset(JNIEnv* env) { if (_obj == nullptr) { return; diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp index 2867415f9a2af5..f4ac699683f868 100644 --- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp @@ -72,6 +72,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_variant_v2.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" #include "core/types.h" diff --git a/be/test/core/data_type_serde/data_type_variant_v2_serde_output_test.cpp b/be/test/core/data_type_serde/data_type_variant_v2_serde_output_test.cpp index 9f5e6054a1fba7..c895bb5a87506b 100644 --- a/be/test/core/data_type_serde/data_type_variant_v2_serde_output_test.cpp +++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_output_test.cpp @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include #include +#include #include #include @@ -198,6 +200,50 @@ std::vector> orc_values(const DataTypeVariantV2SerDe& return result; } +std::shared_ptr binary_variant_arrow_type() { + return arrow::struct_({arrow::field("value", arrow::binary(), false), + arrow::field("metadata", arrow::binary(), false)}); +} + +std::unique_ptr binary_variant_arrow_builder() { + return std::make_unique( + binary_variant_arrow_type(), arrow::default_memory_pool(), + std::vector> { + std::make_shared(arrow::default_memory_pool()), + std::make_shared(arrow::default_memory_pool())}); +} + +void expect_binary_variant_bytes(const DataTypeVariantV2SerDe& serde, const IColumn& column, + const ColumnVariantV2& encoded, + const NullMap* null_map = nullptr) { + auto builder = binary_variant_arrow_builder(); + const Status status = serde.write_column_to_arrow(column, null_map, builder.get(), 0, + column.size(), cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status; + + std::shared_ptr output; + ASSERT_TRUE(builder->Finish(&output).ok()); + const auto& array = assert_cast(*output); + const auto& values = assert_cast(*array.field(0)); + const auto& metadata = assert_cast(*array.field(1)); + ASSERT_EQ(array.length(), static_cast(column.size())); + const auto view = encoded.read_view(); + for (size_t row = 0; row < column.size(); ++row) { + const bool expected_null = null_map != nullptr && (*null_map)[row] != 0; + EXPECT_EQ(array.IsNull(row), expected_null); + if (expected_null) { + continue; + } + const VariantRef expected = view.value_at(row); + const auto actual_value = values.GetView(row); + const auto actual_metadata = metadata.GetView(row); + EXPECT_EQ(std::string_view(actual_value.data(), actual_value.size()), + std::string_view(expected.value.data, expected.value.size)); + EXPECT_EQ(std::string_view(actual_metadata.data(), actual_metadata.size()), + std::string_view(expected.metadata.data, expected.metadata.size)); + } +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest macros inflate the matrix. void expect_text_surfaces(const DataTypeVariantV2SerDe& serde, const IColumn& encoded, const ColumnVariantV2& typed, @@ -397,4 +443,34 @@ TEST(DataTypeVariantV2SerdeOutputTest, ConstNullableAndOuterMasksPreserveBoundar EXPECT_TRUE(invalid_dates->is_typed()); } +TEST(DataTypeVariantV2SerdeOutputTest, BinaryStructPreservesEncodedAndTypedBytesAndOuterNulls) { + DataTypeVariantV2SerDe serde; + auto documents = encoded_json({R"({"a":[1,null,"x"]})", R"({"hidden":true})", "null"}); + NullMap mask {0, 1, 0}; + expect_binary_variant_bytes(serde, *documents, *documents, &mask); + + auto typed = typed_strings( + {std::string_view("plain"), std::nullopt, std::string_view(R"({"text":"value"})")}); + ColumnPtr encoded = encoded_copy(*typed); + expect_binary_variant_bytes(serde, *typed, assert_cast(*encoded)); + EXPECT_TRUE(typed->is_typed()); +} + +TEST(DataTypeVariantV2SerdeOutputTest, BinaryStructRejectsUnsupportedPaimonPrimitive) { + DataTypeVariantV2SerDe serde; + VariantBatchBuilder builder(VariantBatchBuilder::ReserveHint {.rows = 1}); + auto row = builder.begin_row(); + row.add_time_ntz_micros(1'500'000); + row.finish(); + auto encoded = ColumnVariantV2::create(); + encoded->insert_encoded_batch(builder.finish_batch()); + auto arrow_builder = binary_variant_arrow_builder(); + const Status status = serde.write_column_to_arrow(*encoded, nullptr, arrow_builder.get(), 0, + encoded->size(), cctz::utc_time_zone()); + EXPECT_EQ(status.code(), ErrorCode::NOT_IMPLEMENTED_ERROR); + EXPECT_NE(status.to_string().find("Paimon does not support Variant primitive id 17"), + std::string::npos); + EXPECT_EQ(arrow_builder->length(), 0); +} + } // namespace doris diff --git a/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp b/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp new file mode 100644 index 00000000000000..ffb7e98307fc24 --- /dev/null +++ b/be/test/exec/partitioner/external_table_sink_hash_partitioner_test.cpp @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/external_table_sink_hash_partitioner.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/object_pool.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "runtime/descriptor_helper.h" +#include "runtime/descriptors.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris { + +class ExternalTableSinkHashPartitionerTest : public testing::Test { +protected: + void SetUp() override { + TDescriptorTableBuilder table_builder; + TTupleDescriptorBuilder tuple_builder; + tuple_builder.add_slot(TSlotDescriptorBuilder() + .type(TYPE_INT) + .nullable(false) + .column_name("key") + .column_pos(1) + .build()); + tuple_builder.build(&table_builder); + TDescriptorTable thrift_table = table_builder.desc_tbl(); + + DescriptorTbl* descriptor_table = nullptr; + ASSERT_TRUE(DescriptorTbl::create(&_pool, thrift_table, &descriptor_table).ok()); + _state.set_desc_tbl(descriptor_table); + _tuple_id = thrift_table.tupleDescriptors[0].id; + _slot_id = thrift_table.slotDescriptors[0].id; + _row_descriptor = std::make_unique( + *descriptor_table, std::vector {_tuple_id}, std::vector {false}); + } + + TExpr slot_ref() const { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_num_children(0); + node.__set_type(create_type_desc(TYPE_INT)); + node.__set_is_nullable(false); + + TSlotRef slot_ref; + slot_ref.__set_slot_id(_slot_id); + slot_ref.__set_tuple_id(_tuple_id); + node.__set_slot_ref(slot_ref); + + TExpr expression; + expression.nodes.emplace_back(std::move(node)); + return expression; + } + + Block block(std::initializer_list values) const { + auto column = ColumnInt32::create(); + for (int32_t value : values) { + column->insert_value(value); + } + Block block; + block.insert( + ColumnWithTypeAndName(std::move(column), std::make_shared(), "key")); + return block; + } + + ObjectPool _pool; + MockRuntimeState _state; + std::unique_ptr _row_descriptor; + TTupleId _tuple_id = -1; + TSlotId _slot_id = -1; +}; + +TEST_F(ExternalTableSinkHashPartitionerTest, DirectHashKeepsOneKeyOnOneWriter) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({7, 3, 7, 9, 3}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(5, channels.size()); + EXPECT_EQ(channels[0], channels[2]); + EXPECT_EQ(channels[1], channels[4]); + EXPECT_EQ(1, input.columns()); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, OldFePayloadMissingWriterAssignmentFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("writer assignment is missing"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, NewerFeHashAlgorithmFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(static_cast(99)); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unsupported external sink hash algorithm 99"), + std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, DirectHashSupportsSkewedWriterAssignment) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::DIRECT_HASH); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::SKEWED); + ExternalTableSinkHashPartitioner partitioner(4, ShuffleHashMethod::CRC32C, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({7, 3, 7, 9, 3}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(5, channels.size()); + EXPECT_EQ(channels[0], channels[2]); + EXPECT_EQ(channels[1], channels[4]); + for (uint32_t channel : channels) { + EXPECT_LT(channel, 4); + } + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketUsesSdkCompatibleChannel) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({1, 7, 1}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(3, channels.size()); + EXPECT_EQ(5, channels[0]); + EXPECT_EQ(channels[0], channels[2]); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketIncludesPartitionHash) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({0}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({1, 1}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(2, channels.size()); + EXPECT_EQ(0, channels[0]); + EXPECT_EQ(channels[0], channels[1]); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketRejectsMissingMetadata) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("routing metadata is missing"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, PaimonFixedBucketRejectsEmptyTransformMetadata) { + TPaimonFixedBucketInfo fixed_bucket_info; + fixed_bucket_info.__set_num_buckets(4); + fixed_bucket_info.__set_partition_field_indexes({}); + fixed_bucket_info.__set_bucket_field_indexes({0}); + + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::PAIMON_FIXED_BUCKET); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_partition_transforms({}); + info.__set_paimon_fixed_bucket_info(fixed_bucket_info); + ExternalTableSinkHashPartitioner partitioner(8, ShuffleHashMethod::CRC32, info); + + Status status = partitioner.init({slot_ref()}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains incompatible metadata"), std::string::npos); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, IcebergTransformHashesTransformedValue) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::SKEWED); + info.__set_partition_transforms({"truncate[10]"}); + ExternalTableSinkHashPartitioner partitioner(64, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + ASSERT_TRUE(partitioner.open(&_state).ok()); + + Block input = block({11, 19, 20, 29}); + ASSERT_TRUE(partitioner.do_partitioning(&_state, &input).ok()); + const auto& channels = partitioner.get_channel_ids(); + ASSERT_EQ(4, channels.size()); + EXPECT_EQ(channels[0], channels[1]); + EXPECT_EQ(channels[2], channels[3]); + EXPECT_EQ(1, input.columns()); + + ASSERT_TRUE(partitioner.close(&_state).ok()); +} + +TEST_F(ExternalTableSinkHashPartitionerTest, UnsupportedTransformFailsClosed) { + TExternalTableSinkHashPartitionInfo info; + info.__set_algorithm(TExternalTableSinkHashAlgorithm::ICEBERG_TRANSFORM); + info.__set_writer_assignment(TExternalTableSinkWriterAssignment::IDENTITY); + info.__set_partition_transforms({"unsupported"}); + ExternalTableSinkHashPartitioner partitioner(4, ShuffleHashMethod::CRC32, info); + ASSERT_TRUE(partitioner.init({slot_ref()}).ok()); + ASSERT_TRUE(partitioner.prepare(&_state, *_row_descriptor).ok()); + EXPECT_FALSE(partitioner.open(&_state).ok()); +} + +} // namespace doris diff --git a/be/test/exec/partitioner/paimon_native_row_hash_test.cpp b/be/test/exec/partitioner/paimon_native_row_hash_test.cpp new file mode 100644 index 00000000000000..9b60eb0db2843d --- /dev/null +++ b/be/test/exec/partitioner/paimon_native_row_hash_test.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/partitioner/external/paimon_native_row_hash.h" + +#include + +#include +#include + +namespace doris::paimon_native { + +TEST(PaimonNativeRowHashTest, MatchesPaimonBinaryRowGoldenValues) { + BinaryRowEncoder empty(0); + EXPECT_EQ(empty.bytes().size(), 8); + EXPECT_EQ(empty.hash(), -1670924195); + + BinaryRowEncoder row(1); + ASSERT_TRUE(row.write_int(0, 1)); + EXPECT_EQ(row.bytes().size(), 16); + EXPECT_EQ(row.hash(), 1465514398); + + row.reset(); + ASSERT_TRUE(row.set_null(0)); + EXPECT_EQ(row.hash(), -1748325344); + + row.reset(); + ASSERT_TRUE(row.write_string(0, "abcdefgh")); + EXPECT_EQ(row.bytes().size(), 24); + EXPECT_EQ(row.hash(), -843760178); + + row.reset(); + ASSERT_TRUE(row.write_string(0, "abc")); + EXPECT_EQ(row.bytes().size(), 16); + EXPECT_EQ(row.hash(), -101922419); + + BinaryRowEncoder mixed(2); + ASSERT_TRUE(mixed.write_int(0, 1)); + ASSERT_TRUE(mixed.write_string(1, "abc")); + EXPECT_EQ(mixed.hash(), 261371745); +} + +TEST(PaimonNativeRowHashTest, MatchesDefaultBucketAndChannelComputer) { + ASSERT_EQ(default_bucket(1465514398, 4), 2); + ASSERT_EQ(default_bucket(-7, 4), 3); + ASSERT_FALSE(default_bucket(1, 0).has_value()); + + // Keep these vectors in sync with PaimonNativeRoutingGoldenTest, which computes the same + // ownership using Paimon's DefaultBucketFunction and ChannelComputer directly. + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 1), 0); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 2), 1); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 3), 1); + EXPECT_EQ(fixed_bucket_channel(-1670924195, 2, 8), 5); + + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 1), 0); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 2), 1); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 3), 2); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 4), 3); + EXPECT_EQ(fixed_bucket_channel(1465514398, 1, 8), 7); + + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 1), 0); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 2), 0); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 3), 2); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 4), 2); + EXPECT_EQ(fixed_bucket_channel(-101922419, 3, 8), 6); + + ASSERT_EQ(fixed_bucket_channel(std::numeric_limits::min(), 1, 8), 0); + ASSERT_FALSE(fixed_bucket_channel(1, 0, 0).has_value()); +} + +} // namespace doris::paimon_native diff --git a/be/test/exec/partitioner/writer_assigner_test.cpp b/be/test/exec/partitioner/writer_assigner_test.cpp new file mode 100644 index 00000000000000..04883d2ed1543f --- /dev/null +++ b/be/test/exec/partitioner/writer_assigner_test.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "exec/partitioner/writer_assigner.h" + +#include + +#include +#include + +namespace doris { + +TEST(WriterAssignerTest, IdentityPreservesLogicalPartition) { + IdentityWriterAssigner assigner(3); + std::vector partition_ids {2, 0, 1, 2}; + std::vector writer_ids; + + ASSERT_TRUE(assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids).ok()); + EXPECT_EQ(partition_ids, writer_ids); +} + +TEST(WriterAssignerTest, IdentityRejectsInvalidLogicalPartition) { + IdentityWriterAssigner assigner(2); + std::vector partition_ids {0, 2}; + std::vector writer_ids; + + Status status = assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exceeds writer count"), std::string::npos); +} + +TEST(WriterAssignerTest, SkewedRejectsInvalidLogicalPartition) { + SkewedWriterAssigner assigner(4, 2, 1, 1, 1); + std::vector partition_ids {0, 4}; + std::vector writer_ids; + + Status status = assigner.assign(partition_ids, nullptr, partition_ids.size(), 64, writer_ids); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("exceeds partition count"), std::string::npos); +} + +} // namespace doris diff --git a/be/test/exec/sink/paimon_jni_memory_manager_test.cpp b/be/test/exec/sink/paimon_jni_memory_manager_test.cpp new file mode 100644 index 00000000000000..4453b21f4b3ddc --- /dev/null +++ b/be/test/exec/sink/paimon_jni_memory_manager_test.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" + +#include "common/config.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/defer_op.h" + +namespace doris { + +TEST(PaimonJniMemoryManagerTest, DivideQueryBudgetBySinkPipelineTaskCount) { + constexpr int64_t QUERY_LIMIT = 256L * 1024 * 1024; + constexpr int64_t CONFIGURED_LIMIT = 512L * 1024 * 1024; + constexpr int SINK_PIPELINE_TASKS = 4; + + const int64_t old_configured_limit = config::paimon_jni_writer_memory_pool_limit_bytes; + Defer restore_config { + [&] { config::paimon_jni_writer_memory_pool_limit_bytes = old_configured_limit; }}; + config::paimon_jni_writer_memory_pool_limit_bytes = CONFIGURED_LIMIT; + + TUniqueId query_id; + query_id.hi = 1; + query_id.lo = 2; + TQueryOptions query_options; + query_options.__set_mem_limit(QUERY_LIMIT); + query_options.__set_query_type(TQueryType::SELECT); + TNetworkAddress fe_address; + fe_address.hostname = "127.0.0.1"; + fe_address.port = 9030; + auto query_ctx = + QueryContext::create(query_id, ExecEnv::GetInstance(), query_options, fe_address, true, + fe_address, QuerySource::INTERNAL_FRONTEND); + ASSERT_NE(query_ctx, nullptr); + + auto state = RuntimeState::create_unique(query_id, 0, query_options, query_ctx->query_globals, + ExecEnv::GetInstance(), query_ctx.get()); + state->set_task_num(SINK_PIPELINE_TASKS); + // Paimon must not depend on this FE-provided OLAP sink field. + state->set_num_local_sink(1); + + std::unique_ptr manager; + ASSERT_TRUE(PaimonJniMemoryManager::create(state.get(), &manager).ok()); + ASSERT_NE(manager, nullptr); + EXPECT_EQ(manager->memory_limit(), QUERY_LIMIT / SINK_PIPELINE_TASKS); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp new file mode 100644 index 00000000000000..387e8932793338 --- /dev/null +++ b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +#include + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +namespace doris { + +TEST(PaimonWriteBackendFactoryTest, SelectBackendType) { + TPaimonTableSink sink; + EXPECT_EQ(PaimonBackendType::JNI, PaimonWriteBackendFactory::select_backend_type(sink)); + + sink.__set_backend_type(TPaimonWriteBackendType::FFI); + EXPECT_EQ(PaimonBackendType::FFI, PaimonWriteBackendFactory::select_backend_type(sink)); +} + +TEST(JniPaimonWriteBackendTest, OpenAbiAndWriteModes) { + EXPECT_STREQ( + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZZLjava/lang/" + "String;JJJ)V", + PAIMON_JNI_WRITER_OPEN_SIGNATURE); + + auto append = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::APPEND); + EXPECT_FALSE(append.overwrite); + EXPECT_FALSE(append.changelog); + + auto overwrite = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::OVERWRITE); + EXPECT_TRUE(overwrite.overwrite); + EXPECT_FALSE(overwrite.changelog); + + auto changelog = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::CHANGELOG); + EXPECT_FALSE(changelog.overwrite); + EXPECT_TRUE(changelog.changelog); +} + +} // namespace doris diff --git a/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp b/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp index 3d85b7e5c4211f..a1e20984965853 100644 --- a/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp +++ b/be/test/exprs/function/cast/cast_variant_v2_from_test.cpp @@ -31,6 +31,7 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_ipv6.h" #include "core/data_type/data_type_jsonb.h" +#include "core/data_type/data_type_nothing.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" @@ -713,6 +714,29 @@ TEST(CastVariantV2FromTest, NestedArrayRoundTripPreservesNullAndEmptyArray) { EXPECT_EQ(assert_cast(values.get_nested_column()).get_data()[0], 1); } +TEST(CastVariantV2FromTest, NullOnlyArrayEncodesNonEmptyElements) { + auto array_type = std::make_shared(std::make_shared()); + MutableColumnPtr source = array_type->create_column(); + Array values {Field::create_field(Null()), Field::create_field(Null())}; + source->insert(Field::create_field(std::move(values))); + + auto variant_type = std::make_shared(); + Block block {{source->get_ptr(), array_type, "source"}, + {variant_type->create_column(), variant_type, "result"}}; + RuntimeState state; + auto context = FunctionContext::create_context(&state, {}, {}); + Status status = + create_cast_to_variant_v2_wrapper(array_type)(context.get(), block, {0}, 1, 1, nullptr); + ASSERT_TRUE(status.ok()) << status; + + VariantRef encoded = + assert_cast(*block.get_by_position(1).column).get_value_ref(0); + ASSERT_EQ(encoded.basic_type(), VariantBasicType::ARRAY); + ASSERT_EQ(encoded.num_elements(), 2); + EXPECT_TRUE(encoded.array_at(0).is_null()); + EXPECT_TRUE(encoded.array_at(1).is_null()); +} + TEST(CastVariantV2FromTest, DecimalScale38CastsAndScale39IsRejectedAtEncodingBoundary) { VariantBatchBuilder builder(VariantBatchBuilder::ReserveHint {.rows = 1}); auto row = builder.begin_row(); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 35fa9f415525e1..aa58be6b492eea 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -262,6 +262,59 @@ TEST(ParquetSchemaTest, AppliesPaimonShreddedVariantOverrideWithOptionalMetadata EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT); } +TEST(ParquetSchemaTest, AppliesPaimonFallbackOnlyArrayWithRequiredValue) { + auto schema = shredded_array_variant_schema(true, false); + schema[1].__isset.logicalType = false; + schema[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema[6].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok()); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + + const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(fields.size(), 1); + EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT); +} + +TEST(ParquetSchemaTest, RejectsRequiredShreddedFallbackOutsidePaimonFallbackOnlyArray) { + const auto expect_override_corruption = [](std::vector schema) { + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok()); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); + const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields); + EXPECT_TRUE(status.is()) << status; + }; + + auto paimon_object = shredded_object_variant_schema(); + paimon_object[1].__isset.logicalType = false; + paimon_object[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + paimon_object[5].__set_num_children(1); + paimon_object[5].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + paimon_object[6].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + paimon_object.pop_back(); + expect_override_corruption(std::move(paimon_object)); + + auto paimon_array_with_typed_value = shredded_array_variant_schema(true, true); + paimon_array_with_typed_value[1].__isset.logicalType = false; + paimon_array_with_typed_value[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + paimon_array_with_typed_value[6].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + paimon_array_with_typed_value[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + expect_override_corruption(std::move(paimon_array_with_typed_value)); + + auto annotated_array = shredded_array_variant_schema(true, false); + annotated_array[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + NativeFieldDescriptor descriptor; + const auto annotated_status = descriptor.parse_from_thrift(annotated_array); + EXPECT_TRUE(annotated_status.is()) << annotated_status; +} + TEST(ParquetSchemaTest, RejectsMalformedUnannotatedVariantOverride) { auto schema = unshredded_variant_schema(); schema[1].__isset.logicalType = false; diff --git a/be/test/format_v2/parquet/variant_column_reader_test.cpp b/be/test/format_v2/parquet/variant_column_reader_test.cpp index 3a19c18d649851..a14b22a77075f7 100644 --- a/be/test/format_v2/parquet/variant_column_reader_test.cpp +++ b/be/test/format_v2/parquet/variant_column_reader_test.cpp @@ -168,6 +168,24 @@ ParquetColumnSchema shredded_array_schema() { return schema; } +ParquetColumnSchema shredded_fallback_only_array_schema() { + auto schema = unshredded_schema(); + auto typed = std::make_unique(); + typed->name = "typed_value"; + typed->kind = ParquetColumnSchemaKind::LIST; + auto element = std::make_unique(); + element->name = "element"; + element->kind = ParquetColumnSchemaKind::STRUCT; + auto value = std::make_unique(); + value->name = "value"; + value->kind = ParquetColumnSchemaKind::PRIMITIVE; + value->type = std::make_shared(); + element->children.push_back(std::move(value)); + typed->children.push_back(std::move(element)); + schema.children.push_back(std::move(typed)); + return schema; +} + ParquetColumnSchema shredded_mixed_array_schema() { auto schema = shredded_array_schema(); auto* element = schema.children.back()->children[0].get(); @@ -1809,6 +1827,47 @@ TEST(VariantColumnReaderTest, MaterializesShreddedArrayElements) { EXPECT_EQ(value.array_at(1).get_int(), 4); } +TEST(VariantColumnReaderTest, MaterializesFallbackOnlyArrayFromRequiredValueLeaf) { + const StringRef metadata(VARIANT_EMPTY_METADATA.data(), VARIANT_EMPTY_METADATA.size()); + const std::array first_value { + static_cast(static_cast(VariantPrimitiveId::INT8) + << VARIANT_VALUE_HEADER_SHIFT), + 3}; + const std::array second_value { + static_cast(static_cast(VariantPrimitiveId::INT8) + << VARIANT_VALUE_HEADER_SHIFT), + 4}; + + auto values = ColumnString::create(); + values->insert_data(first_value.data(), first_value.size()); + values->insert_data(second_value.data(), second_value.size()); + MutableColumns wrapper_fields; + wrapper_fields.push_back(std::move(values)); + auto wrappers = ColumnStruct::create(std::move(wrapper_fields)); + auto elements = ColumnNullable::create(std::move(wrappers), ColumnUInt8::create(2, 0)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(2); + auto array = ColumnArray::create(std::move(elements), std::move(offsets)); + + const std::array ignored {0}; + MutableColumns root_fields; + root_fields.push_back(nullable_strings({metadata}, {0})); + root_fields.push_back(nullable_strings({{ignored.data(), 0}}, {1})); + root_fields.push_back(ColumnNullable::create(std::move(array), ColumnUInt8::create(1, 0))); + auto physical = root_wrapper(std::move(root_fields)); + + auto output = make_nullable(std::make_shared())->create_column(); + const auto status = + materialize_variant_rows(shredded_fallback_only_array_schema(), *physical, output); + ASSERT_TRUE(status.ok()) << status; + const auto& nullable = assert_cast(*output); + const auto& variants = assert_cast(nullable.get_nested_column()); + const VariantRef value = variants.get_value_ref(0); + ASSERT_EQ(value.num_elements(), 2); + EXPECT_EQ(value.array_at(0).get_int(), 3); + EXPECT_EQ(value.array_at(1).get_int(), 4); +} + TEST(VariantColumnReaderTest, RejectsCorruptShreddedWrappersWithoutCrashing) { const std::array int_seven { static_cast(static_cast(VariantPrimitiveId::INT8) diff --git a/be/test/vec/spill/spill_file_test.cpp b/be/test/vec/spill/spill_file_test.cpp index c4d4f140635f8d..1a39a576b238d3 100644 --- a/be/test/vec/spill/spill_file_test.cpp +++ b/be/test/vec/spill/spill_file_test.cpp @@ -92,7 +92,7 @@ class SpillFileTest : public testing::Test { auto st = io::global_local_filesystem()->create_directory(spill_data_dir->path(), false); ASSERT_TRUE(st.ok()) << "create directory failed: " << st.to_string(); auto second_spill_data_dir = std::make_unique( - _second_spill_dir, 1024L * 1024 * 128, TStorageMedium::HDD); + _second_spill_dir, 1024L * 1024 * 128, TStorageMedium::SSD); st = io::global_local_filesystem()->create_directory(second_spill_data_dir->path(), false); ASSERT_TRUE(st.ok()) << "create directory failed: " << st.to_string(); @@ -416,10 +416,15 @@ TEST_F(SpillFileTest, OpenCanRetryAfterFailure) { ASSERT_TRUE(st.ok()); } - const auto part_path = + const auto first_part_path = std::filesystem::path(_spill_dir) / "spill" / "test_query" / "open_retry" / "0"; - const auto backup_path = - std::filesystem::path(_spill_dir) / "spill" / "test_query" / "open_retry" / "0.bak"; + const auto second_part_path = + std::filesystem::path(_second_spill_dir) / "spill" / "test_query" / "open_retry" / "0"; + const auto part_path = + std::filesystem::exists(first_part_path) ? first_part_path : second_part_path; + ASSERT_TRUE(std::filesystem::exists(part_path)); + auto backup_path = part_path; + backup_path += ".bak"; std::filesystem::rename(part_path, backup_path); @@ -934,13 +939,17 @@ TEST_F(SpillFileTest, GCCleansUpFiles) { st = writer->close(); ASSERT_TRUE(st.ok()); - // Remember the spill directory path - spill_file_dir = _data_dir_ptr->get_spill_data_path() + "/test_query/gc_test"; - - // Verify directory exists + // Remember the selected spill directory path. bool exists = false; - st = io::global_local_filesystem()->exists(spill_file_dir, &exists); - ASSERT_TRUE(st.ok()); + for (auto* data_dir : {_data_dir_ptr, _second_data_dir_ptr}) { + auto candidate = data_dir->get_spill_data_path() + "/test_query/gc_test"; + st = io::global_local_filesystem()->exists(candidate, &exists); + ASSERT_TRUE(st.ok()); + if (exists) { + spill_file_dir = std::move(candidate); + break; + } + } ASSERT_TRUE(exists); // spill_file goes out of scope here, destructor calls gc() @@ -1383,10 +1392,17 @@ TEST_F(SpillFileTest, DeleteSpillFileThroughManagerSynchronously) { st = writer->close(); ASSERT_TRUE(st.ok()); - auto spill_file_dir = _data_dir_ptr->get_spill_data_path("test_query/mgr_delete"); + std::string spill_file_dir; bool exists = false; - st = io::global_local_filesystem()->exists(spill_file_dir, &exists); - ASSERT_TRUE(st.ok()); + for (auto* data_dir : {_data_dir_ptr, _second_data_dir_ptr}) { + auto candidate = data_dir->get_spill_data_path("test_query/mgr_delete"); + st = io::global_local_filesystem()->exists(candidate, &exists); + ASSERT_TRUE(st.ok()); + if (exists) { + spill_file_dir = std::move(candidate); + break; + } + } ASSERT_TRUE(exists); ExecEnv::GetInstance()->spill_file_mgr()->delete_spill_file(spill_file); @@ -1409,6 +1425,163 @@ TEST_F(SpillFileTest, ManagerNextId) { ASSERT_EQ(id3, id2 + 1); } +TEST_F(SpillFileTest, ManagerAllocatesExternalSpillSessionOnManagedRoot) { + TUniqueId query_id; + query_id.hi = 21; + query_id.lo = 22; + auto query_id_str = print_id(query_id); + auto query_ctx = MockQueryContext::create(query_id); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + + ASSERT_TRUE(st.ok()) << st.to_string(); + std::vector paths; + st = spill_session->get_paths(&paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(paths.size(), 1); + const std::string first_path = _data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"; + const std::string second_path = + _second_data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"; + ASSERT_TRUE(paths.front() == first_path || paths.front() == second_path); + bool exists = false; + for (const auto& path : paths) { + st = io::global_local_filesystem()->exists(path, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_FALSE(exists); + } + + const std::string& selected_path = paths.front(); + const std::string channel = selected_path + "/paimon-io-test/channel"; + ASSERT_TRUE(spill_session->reserve(channel, 1024).ok()); + auto* selected_data_dir = selected_path == first_path ? _data_dir_ptr : _second_data_dir_ptr; + auto* unselected_data_dir = + selected_data_dir == _data_dir_ptr ? _second_data_dir_ptr : _data_dir_ptr; + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 1024); + ASSERT_EQ(unselected_data_dir->get_spill_data_bytes(), 0); + spill_session->update_accounting(channel, -256, 0, 0); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 768); + _create_residual_file(channel); + + // Query teardown must not remove a directory while an asynchronous external writer can still + // use its native callback. The regular spill GC handles deferred cleanup after lease release. + query_ctx.reset(); + auto query_dir = selected_data_dir->get_spill_data_path(query_id_str); + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_TRUE(exists); + + spill_session.reset(); + // Match SpillFile::gc(): logical usage is released with the writer, while QueryContext owns + // physical deletion and retries. + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); + ASSERT_EQ(unselected_data_dir->get_spill_data_bytes(), 0); + + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_TRUE(exists); + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_FALSE(exists); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); +} + +TEST_F(SpillFileTest, ExternalSpillSessionSkipsFullManagedRoot) { + TUniqueId query_id; + query_id.hi = 23; + query_id.lo = 24; + auto query_id_str = print_id(query_id); + auto query_ctx = MockQueryContext::create(query_id); + + const int64_t unavailable_bytes = _data_dir_ptr->get_spill_data_limit() + 1; + _data_dir_ptr->update_spill_data_usage(unavailable_bytes); + Defer release_full_root([&]() { _data_dir_ptr->update_spill_data_usage(-unavailable_bytes); }); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + ASSERT_TRUE(st.ok()) << st.to_string(); + + std::vector paths; + st = spill_session->get_paths(&paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths.front(), _second_data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"); +} + +TEST_F(SpillFileTest, ExternalSpillDirectoryCleanupRetriesAfterLeaseRelease) { + ExecEnv::GetInstance()->spill_file_mgr()->stop(); + TUniqueId query_id; + query_id.hi = 33; + query_id.lo = 34; + auto query_ctx = MockQueryContext::create(query_id); + + std::unique_ptr spill_session; + ASSERT_TRUE(ExecEnv::GetInstance() + ->spill_file_mgr() + ->create_external_spill_session("paimon", query_ctx.get(), &spill_session) + .ok()); + std::vector paths; + ASSERT_TRUE(spill_session->get_paths(&paths).ok()); + auto* selected_data_dir = + paths.front().starts_with(_data_dir_ptr->get_spill_data_path(print_id(query_id))) + ? _data_dir_ptr + : _second_data_dir_ptr; + ASSERT_TRUE(spill_session->reserve(paths.front() + "/paimon-io/channel", 1024).ok()); + _create_residual_file(paths.front() + "/paimon-io/channel"); + + const bool previous_enable_debug_points = config::enable_debug_points; + constexpr auto debug_point_name = + "fault_inject::spill_file_manager::delete_query_spill_directory"; + Defer restore_debug_point([&] { + DebugPoints::instance()->remove(debug_point_name); + config::enable_debug_points = previous_enable_debug_points; + }); + auto debug_point = std::make_shared(); + debug_point->execute_limit = 1; + config::enable_debug_points = true; + DebugPoints::instance()->add(debug_point_name, debug_point); + + query_ctx.reset(); + spill_session.reset(); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); + + const auto query_dir = selected_data_dir->get_spill_data_path(print_id(query_id)); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_TRUE(exists); + + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_TRUE(exists); + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_FALSE(exists); +} + +TEST_F(SpillFileTest, ExternalSpillSessionIsLazyWhenNoRootAvailable) { + TUniqueId query_id; + query_id.hi = 25; + query_id.lo = 26; + auto query_ctx = MockQueryContext::create(query_id); + + _data_dir_ptr->update_spill_data_usage(_data_dir_ptr->get_spill_data_limit()); + _second_data_dir_ptr->update_spill_data_usage(_second_data_dir_ptr->get_spill_data_limit()); + Defer release_full_roots([&]() { + _data_dir_ptr->update_spill_data_usage(-_data_dir_ptr->get_spill_data_limit()); + _second_data_dir_ptr->update_spill_data_usage( + -_second_data_dir_ptr->get_spill_data_limit()); + }); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_NE(spill_session, nullptr); +} + TEST_F(SpillFileTest, ManagerCreateMultipleFiles) { const int num_files = 5; std::vector files; @@ -1601,7 +1774,8 @@ TEST_F(SpillFileTest, DataDirCapacityTracking) { spill_file); ASSERT_TRUE(st.ok()); - auto initial_bytes = _data_dir_ptr->get_spill_data_bytes(); + auto initial_bytes = + _data_dir_ptr->get_spill_data_bytes() + _second_data_dir_ptr->get_spill_data_bytes(); SpillFileWriterSPtr writer; st = spill_file->create_writer(_runtime_state.get(), _profile.get(), writer); @@ -1617,7 +1791,8 @@ TEST_F(SpillFileTest, DataDirCapacityTracking) { st = writer->close(); ASSERT_TRUE(st.ok()); - auto after_write_bytes = _data_dir_ptr->get_spill_data_bytes(); + auto after_write_bytes = + _data_dir_ptr->get_spill_data_bytes() + _second_data_dir_ptr->get_spill_data_bytes(); ASSERT_GT(after_write_bytes, initial_bytes); } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index 983681d24dcb72..00411e8531285b 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -63,6 +63,7 @@ public enum Type { IPV6(16), STRING(-1), VARBINARY(-1), + VARIANT(-1), ARRAY(-1), MAP(-1), STRUCT(-1); @@ -157,6 +158,10 @@ public boolean isVarbinaryType() { return type == Type.BINARY || type == Type.VARBINARY; } + public boolean isVariantType() { + return type == Type.VARIANT; + } + public boolean isComplexType() { return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT; } @@ -250,6 +255,10 @@ public int metaSize() { case VARCHAR: // [const | nullMap | offsets | data ] return 4; + case VARIANT: + // [const | nullMap | metadata count | metadata offsets | metadata bytes + // | metadata ids | value offsets | value bytes] + return 8; default: // [const | nullMap | data] return 3; @@ -344,6 +353,9 @@ private static ColumnType parseType(String columnName, String hiveType, boolean case "varbinary": type = Type.VARBINARY; break; + case "variant": + type = Type.VARIANT; + break; default: if (lowerCaseType.startsWith("timestamptz")) { type = Type.TIMESTAMPTZ; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java index 8911a19f0bd5bd..36c0be38636a9e 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java @@ -77,6 +77,14 @@ default boolean canGetCharAsBytes() { byte[] getBytes(); + default byte[] getVariantMetadata() { + throw new UnsupportedOperationException("Variant metadata is not available"); + } + + default byte[] getVariantValue() { + throw new UnsupportedOperationException("Variant value is not available"); + } + void unpackArray(List values); void unpackMap(List keys, List values); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 36d2329da89976..010bf7eba8ff67 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -75,6 +75,7 @@ public class VectorColumn { // For nested column type: String / Array/ Map / Struct private VectorColumn[] childColumns; + private VectorColumnVariant variantColumn; // For struct, only support to read all fields in struct now // todo: support pruned struct fields @@ -120,6 +121,8 @@ private VectorColumn(ColumnType columnType, int capacity) { childColumns = new VectorColumn[1]; childColumns[0] = new VectorColumn(new ColumnType("#stringBytes", Type.BYTE), capacity * DEFAULT_STRING_LENGTH); + } else if (columnType.isVariantType()) { + variantColumn = new VectorColumnVariant(); } reserveCapacity(capacity); @@ -273,6 +276,10 @@ public void close() { } childColumns = null; } + if (variantColumn != null) { + variantColumn.close(); + variantColumn = null; + } if (nullMap != 0) { OffHeap.freeMemory(nullMap); @@ -354,6 +361,8 @@ private void reserveCapacity(int newCapacity) { this.offsets = OffHeap.reallocateMemory(offsets, oldOffsetSize, newOffsetSize); } else if (columnType.isVarbinaryType()) { this.data = OffHeap.reallocateMemory(data, oldCapacity * 16L, newCapacity * 16L); + } else if (columnType.isVariantType()) { + variantColumn.reserveRows(newCapacity); } else if (!columnType.isStruct()) { throw new RuntimeException("Unhandled type: " + columnType.getName()); } @@ -370,6 +379,9 @@ public void reset() { c.reset(); } } + if (variantColumn != null) { + variantColumn.reset(); + } appendIndex = 0; if (numNulls > 0) { putNotNulls(0, capacity); @@ -462,6 +474,8 @@ public int appendNull(ColumnType.Type typeValue) { case BINARY: case VARBINARY: return appendVarbinary(new byte[0]); + case VARIANT: + return appendVariantNull(); default: throw new RuntimeException("Unknown type value: " + typeValue); } @@ -1530,6 +1544,17 @@ public int appendVarbinary(byte[] src) { return appendIndex++; } + public int appendVariant(byte[] metadata, byte[] value) { + reserve(appendIndex + 1); + variantColumn.append(metadata, value); + return appendIndex++; + } + + private int appendVariantNull() { + variantColumn.appendNull(); + return appendIndex++; + } + public void appendVarbinary(byte[][] batch, boolean isNullable) { if (!isNullable) { checkNullable(batch, batch.length); @@ -1619,6 +1644,9 @@ public void updateMeta(VectorColumn meta) { for (VectorColumn c : childColumns) { c.updateMeta(meta); } + } else if (columnType.isVariantType()) { + meta.appendLong(nullMap); + variantColumn.updateMeta(meta); } else { meta.appendLong(nullMap); meta.appendLong(data); @@ -1903,6 +1931,9 @@ public void appendValue(ColumnValue o) { case VARBINARY: appendVarbinary(o.getBytes()); break; + case VARIANT: + appendVariant(o.getVariantMetadata(), o.getVariantValue()); + break; case ARRAY: { List values = new ArrayList<>(); o.unpackArray(values); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumnVariant.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumnVariant.java new file mode 100644 index 00000000000000..16fff901e31075 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumnVariant.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +import org.apache.doris.common.jni.utils.OffHeap; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** Off-heap encoded Variant storage used by {@link VectorColumn}. */ +final class VectorColumnVariant { + private static final int MAX_CAPACITY = Integer.MAX_VALUE - 15; + private static final byte[] EMPTY_METADATA = new byte[] {1, 0, 0}; + private static final byte[] NULL_VALUE = new byte[] {0}; + + private final Map metadataIds = new HashMap<>(); + private long metadataOffsets; + private long metadataBytes; + private long rowMetadataIds; + private long valueOffsets; + private long valueBytes; + private int rowCapacity; + private int metadataBytesCapacity; + private int valueBytesCapacity; + private int metadataCount; + private int metadataBytesSize; + private int rowCount; + private int valueBytesSize; + + void reserveRows(int requiredCapacity) { + if (requiredCapacity <= rowCapacity) { + return; + } + int newCapacity = growCapacity(rowCapacity, requiredCapacity); + long oldOffsetsSize = rowCapacity == 0 ? 0 : (rowCapacity + 1L) * Integer.BYTES; + metadataOffsets = OffHeap.reallocateMemory( + metadataOffsets, oldOffsetsSize, (newCapacity + 1L) * Integer.BYTES); + rowMetadataIds = OffHeap.reallocateMemory( + rowMetadataIds, (long) rowCapacity * Integer.BYTES, + (long) newCapacity * Integer.BYTES); + valueOffsets = OffHeap.reallocateMemory( + valueOffsets, oldOffsetsSize, (newCapacity + 1L) * Integer.BYTES); + if (rowCapacity == 0) { + OffHeap.putInt(null, metadataOffsets, 0); + OffHeap.putInt(null, valueOffsets, 0); + } + rowCapacity = newCapacity; + } + + void append(byte[] metadata, byte[] value) { + Objects.requireNonNull(metadata, "Variant metadata cannot be null"); + Objects.requireNonNull(value, "Variant value cannot be null"); + reserveRows(rowCount + 1); + Integer metadataId = metadataIds.get(new ByteArrayKey(metadata)); + if (metadataId == null) { + metadataId = appendMetadata(metadata); + } + OffHeap.putInt(null, rowMetadataIds + (long) rowCount * Integer.BYTES, metadataId); + int requiredValueBytes = checkedSize("value", valueBytesSize, value.length); + reserveValueBytes(requiredValueBytes); + OffHeap.copyMemory( + value, OffHeap.BYTE_ARRAY_OFFSET, null, valueBytes + valueBytesSize, value.length); + valueBytesSize = requiredValueBytes; + rowCount++; + OffHeap.putInt(null, valueOffsets + (long) rowCount * Integer.BYTES, valueBytesSize); + } + + void appendNull() { + append(EMPTY_METADATA, NULL_VALUE); + } + + void updateMeta(VectorColumn meta) { + meta.appendLong(metadataCount); + meta.appendLong(metadataOffsets); + meta.appendLong(metadataBytes); + meta.appendLong(rowMetadataIds); + meta.appendLong(valueOffsets); + meta.appendLong(valueBytes); + } + + void reset() { + metadataIds.clear(); + metadataCount = 0; + metadataBytesSize = 0; + rowCount = 0; + valueBytesSize = 0; + if (rowCapacity > 0) { + OffHeap.putInt(null, metadataOffsets, 0); + OffHeap.putInt(null, valueOffsets, 0); + } + } + + void close() { + free(metadataOffsets); + free(metadataBytes); + free(rowMetadataIds); + free(valueOffsets); + free(valueBytes); + metadataOffsets = 0; + metadataBytes = 0; + rowMetadataIds = 0; + valueOffsets = 0; + valueBytes = 0; + rowCapacity = 0; + metadataBytesCapacity = 0; + valueBytesCapacity = 0; + reset(); + } + + private int appendMetadata(byte[] metadata) { + int requiredMetadataBytes = checkedSize("metadata", metadataBytesSize, metadata.length); + reserveMetadataBytes(requiredMetadataBytes); + OffHeap.copyMemory(metadata, OffHeap.BYTE_ARRAY_OFFSET, + null, metadataBytes + metadataBytesSize, metadata.length); + metadataBytesSize = requiredMetadataBytes; + metadataCount++; + OffHeap.putInt(null, metadataOffsets + (long) metadataCount * Integer.BYTES, + metadataBytesSize); + int metadataId = metadataCount - 1; + metadataIds.put(new ByteArrayKey(Arrays.copyOf(metadata, metadata.length)), metadataId); + return metadataId; + } + + private void reserveMetadataBytes(int requiredCapacity) { + if (requiredCapacity <= metadataBytesCapacity) { + return; + } + int newCapacity = growCapacity(metadataBytesCapacity, requiredCapacity); + metadataBytes = OffHeap.reallocateMemory(metadataBytes, metadataBytesCapacity, newCapacity); + metadataBytesCapacity = newCapacity; + } + + private void reserveValueBytes(int requiredCapacity) { + if (requiredCapacity <= valueBytesCapacity) { + return; + } + int newCapacity = growCapacity(valueBytesCapacity, requiredCapacity); + valueBytes = OffHeap.reallocateMemory(valueBytes, valueBytesCapacity, newCapacity); + valueBytesCapacity = newCapacity; + } + + private static int checkedSize(String component, int currentSize, int appendedSize) { + long requiredSize = (long) currentSize + appendedSize; + if (requiredSize > MAX_CAPACITY) { + throw new RuntimeException( + "Variant " + component + " buffer exceeds the Java JNI size limit"); + } + return (int) requiredSize; + } + + private static int growCapacity(int currentCapacity, int requiredCapacity) { + long doubledCapacity = Math.max(1L, currentCapacity * 2L); + int newCapacity = (int) Math.min( + MAX_CAPACITY, Math.max(doubledCapacity, requiredCapacity)); + if (newCapacity < requiredCapacity) { + throw new RuntimeException("Cannot reserve enough bytes for Variant JNI data"); + } + return newCapacity; + } + + private static void free(long address) { + if (address != 0) { + OffHeap.freeMemory(address); + } + } + + private static final class ByteArrayKey { + private final byte[] bytes; + private final int hashCode; + + private ByteArrayKey(byte[] bytes) { + this.bytes = bytes; + this.hashCode = Arrays.hashCode(bytes); + } + + @Override + public boolean equals(Object other) { + return other instanceof ByteArrayKey + && Arrays.equals(bytes, ((ByteArrayKey) other).bytes); + } + + @Override + public int hashCode() { + return hashCode; + } + } +} diff --git a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnVariantTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnVariantTest.java new file mode 100644 index 00000000000000..fc71996883ea15 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnVariantTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +import org.apache.doris.common.jni.utils.OffHeap; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class VectorColumnVariantTest { + private static final byte[] EMPTY_METADATA = new byte[] {1, 0, 0}; + private static final byte[] ONE_KEY_METADATA = new byte[] {1, 1, 0, 1, 'a'}; + + @BeforeClass + public static void setUpClass() { + OffHeap.setTesting(); + } + + @Test + public void testVariantTypeAndEncodedLayout() { + ColumnType variantType = ColumnType.parseType("v", "variant"); + Assert.assertEquals(ColumnType.Type.VARIANT, variantType.getType()); + Assert.assertEquals(8, variantType.metaSize()); + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {variantType}, new String[] {"v"}, 2); + try { + VectorColumn column = table.getColumn(0); + column.appendVariant(EMPTY_METADATA, new byte[] {0}); + column.appendVariant(EMPTY_METADATA.clone(), new byte[] {4}); + column.appendVariant(ONE_KEY_METADATA, new byte[] {8}); + long meta = table.getMetaAddress(); + Assert.assertEquals(3L, OffHeap.getLong(null, meta)); + Assert.assertEquals(2L, OffHeap.getLong(null, meta + 16)); + Assert.assertArrayEquals(new int[] {0, 3, 8}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 24), 3)); + Assert.assertArrayEquals(new int[] {0, 0, 1}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 40), 3)); + Assert.assertArrayEquals(new int[] {0, 1, 2, 3}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 48), 4)); + } finally { + table.close(); + } + } + + @Test + public void testSqlNullUsesValidVariantNullPlaceholder() { + ColumnType variantType = ColumnType.parseType("v", "variant"); + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {variantType}, new String[] {"v"}, 1); + try { + VectorColumn column = table.getColumn(0); + column.appendVariant(EMPTY_METADATA, new byte[] {4}); + column.appendNull(ColumnType.Type.VARIANT); + long meta = table.getMetaAddress(); + Assert.assertArrayEquals(new boolean[] {false, true}, + OffHeap.getBoolean(null, OffHeap.getLong(null, meta + 8), 2)); + Assert.assertEquals(1L, OffHeap.getLong(null, meta + 16)); + Assert.assertArrayEquals(new byte[] {4, 0}, + OffHeap.getByte(null, OffHeap.getLong(null, meta + 56), 2)); + } finally { + table.close(); + } + } + + @Test + public void testResetRebuildsMetadataDictionary() { + ColumnType variantType = ColumnType.parseType("v", "variant"); + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {variantType}, new String[] {"v"}, 1); + try { + table.getColumn(0).appendVariant(EMPTY_METADATA, new byte[] {0}); + table.reset(); + table.getColumn(0).appendVariant(ONE_KEY_METADATA, new byte[] {4}); + long meta = table.getMetaAddress(); + Assert.assertEquals(1L, OffHeap.getLong(null, meta)); + Assert.assertEquals(1L, OffHeap.getLong(null, meta + 16)); + Assert.assertArrayEquals(new int[] {0, 5}, + OffHeap.getInt(null, OffHeap.getLong(null, meta + 24), 2)); + } finally { + table.close(); + } + } +} diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-scanner/pom.xml index fa7c27e4e98319..8bad697778abf4 100644 --- a/fe/be-java-extensions/paimon-scanner/pom.xml +++ b/fe/be-java-extensions/paimon-scanner/pom.xml @@ -61,6 +61,11 @@ under the License. paimon-format + + org.apache.paimon + paimon-arrow + + 1.12.1 1.17.0 @@ -443,7 +443,7 @@ under the License. InstantiationUtil and BE deserializes it with the SAME paimon jar; a version mismatch silently breaks that FE->BE deserialization at runtime. These three MUST stay equal β€” do NOT override paimon.version per-module. --> - 1.3.1 + 1.4.2 3.4.4 17.0.0 @@ -1598,6 +1598,11 @@ under the License. paimon-format ${paimon.version} + + org.apache.paimon + paimon-arrow + ${paimon.version} + org.apache.paimon paimon-s3 @@ -1947,6 +1952,11 @@ under the License. arrow-memory-core ${arrow.version} + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + org.apache.arrow arrow-jdbc diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 1c776b8306644d..7fd600835ea606 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -46,6 +46,7 @@ enum TDataSinkType { MAXCOMPUTE_TABLE_SINK = 18, ICEBERG_DELETE_SINK = 19, ICEBERG_MERGE_SINK = 20, + PAIMON_TABLE_SINK = 21, } enum TResultSinkType { @@ -630,6 +631,31 @@ struct TMaxComputeTableSink { 18: optional i64 txn_id // FE external transaction ID for runtime block_id allocation } +enum TPaimonWriteBackendType { + JNI = 0, + FFI = 1, +} + +enum TPaimonWriteMode { + APPEND = 0, + OVERWRITE = 1, + CHANGELOG = 2, +} + +struct TPaimonCommitMessage { + 1: optional binary payload // Paimon native CommitMessageSerializer bytes (DPCM-framed) +} + +struct TPaimonTableSink { + 1: optional string serialized_table // required at runtime; serialized Paimon Table object (base64) + 2: optional map hadoop_config + 3: optional list column_names + 4: optional TPaimonWriteBackendType backend_type + 5: optional TPaimonWriteMode write_mode + 6: optional i64 transaction_id + 7: optional string commit_user +} + struct TDataSink { 1: required TDataSinkType type 2: optional TDataStreamSink stream_sink @@ -650,4 +676,5 @@ struct TDataSink { 18: optional TMaxComputeTableSink max_compute_table_sink 19: optional TIcebergDeleteSink iceberg_delete_sink 20: optional TIcebergMergeSink iceberg_merge_sink + 21: optional TPaimonTableSink paimon_table_sink } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 90f4a4b41d0213..d9e7b5c7af0a03 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -338,6 +338,8 @@ struct TReportExecStatusParams { 32: optional list mc_commit_datas 33: optional string first_error_msg + + 34: optional list paimon_commit_messages } struct TFeResult { diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index da172fac735c2b..d53498de76bcfd 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -45,11 +45,12 @@ enum TPartitionType { // used for shuffle data by parititon and tablet OLAP_TABLE_SINK_HASH_PARTITIONED = 6, - // used for shuffle data by hive parititon - HIVE_TABLE_SINK_HASH_PARTITIONED = 7, + // used for shuffle data by external table sink ownership key. BE execution + // versions before 12 reject this type because value 7 had different semantics. + EXTERNAL_TABLE_SINK_HASH_PARTITIONED = 7, - // used for hive unparititoned table - HIVE_TABLE_SINK_UNPARTITIONED = 8, + // adaptive writer distribution for external tables without an ownership key + EXTERNAL_TABLE_SINK_UNPARTITIONED = 8, // used for merge partitioning: insert by partition columns, delete by row_id MERGE_PARTITIONED = 9 @@ -196,6 +197,51 @@ struct TMergePartitionInfo { 6: optional i32 partition_spec_id } +// The routing algorithm implemented by an external table sink hash exchange. +// Connector-specific FE distribution specs map to one of these algorithms. +enum TExternalTableSinkHashAlgorithm { + DIRECT_HASH = 0, + ICEBERG_TRANSFORM = 1, + + // Paimon HASH_FIXED routing using the default bucket function and + // ChannelComputer. The partition expressions carry their Doris types. + PAIMON_FIXED_BUCKET = 2 +} + +// Maps the logical partitions produced by the hash algorithm to Doris exchange writers. +// IDENTITY preserves one-writer ownership; SKEWED retains ScaleWriter behavior for formats +// which allow a hot partition to be written by multiple writers. +enum TExternalTableSinkWriterAssignment { + IDENTITY = 0, + SKEWED = 1 +} + +// Minimal metadata for Paimon's stateless HASH_FIXED route. Field indexes refer +// to TDataPartition.partition_exprs, not to physical Block column positions. +struct TPaimonFixedBucketInfo { + 1: required i32 num_buckets + 2: required list partition_field_indexes + 3: required list bucket_field_indexes +} + +// Connector-independent routing metadata for an external table sink hash +// exchange. Algorithm-specific fields are optional for protocol evolution, but +// each algorithm validates the fields it requires before processing rows. +struct TExternalTableSinkHashPartitionInfo { + 1: required TExternalTableSinkHashAlgorithm algorithm + + // Positional Iceberg transforms. Required by ICEBERG_TRANSFORM and must have + // the same size as partition_exprs. + 2: optional list partition_transforms + + // Kept optional on the wire so a new BE can reject an old FE with a clear + // status instead of silently relying on the enum's default value. + 3: optional TExternalTableSinkWriterAssignment writer_assignment + + // Required only by PAIMON_FIXED_BUCKET. + 4: optional TPaimonFixedBucketInfo paimon_fixed_bucket_info +} + // Specification of how a single logical data stream is partitioned. // This leaves out the parameters that determine the physical partition (for hash // partitions, the number of partitions; for range partitions, the partitions' @@ -205,4 +251,5 @@ struct TDataPartition { 2: optional list partition_exprs 3: optional list partition_infos 4: optional TMergePartitionInfo merge_partition_info + 5: optional TExternalTableSinkHashPartitionInfo external_table_sink_hash_partition_info } diff --git a/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out b/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out new file mode 100644 index 00000000000000..7718c425e02479 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out @@ -0,0 +1,125 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !paimon_alter_initial_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N + +-- !paimon_alter_initial_schema -- +0 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}] [] ["id"] + +-- !paimon_alter_add_column_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N + +-- !paimon_alter_add_column_schema -- +1 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":6,"name":"added_after","type":"STRING","description":"added column","defaultValue":"unknown"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}] + +-- !paimon_alter_add_columns_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_add_first_desc -- +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_rename_column_desc -- +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +display_name text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_modify_column_desc -- +score bigint Yes true \N updated score +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +added_after text Yes true \N +display_name text Yes true \N +obsolete text Yes true \N +amount decimal(12,2) Yes true \N +tiny_col tinyint Yes true \N +small_col int Yes true \N +profile struct Yes true \N + +-- !paimon_alter_modify_column_schema -- +9 [{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":12,"name":"first_col","type":"BIGINT","description":""},{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""}] + +-- !paimon_alter_drop_column_desc -- +score bigint Yes true \N updated score +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +added_after text Yes true \N +display_name text Yes true \N +amount decimal(12,2) Yes true \N +tiny_col tinyint Yes true \N +small_col int Yes true \N +profile struct Yes true \N + +-- !paimon_alter_reorder_columns_desc -- +id int Yes true \N identifier +display_name text Yes true \N +score bigint Yes true \N updated score +required_value bigint Yes true \N +small_col int Yes true \N +tiny_col tinyint Yes true \N +added_after text Yes true \N +amount decimal(12,2) Yes true \N +profile struct Yes true \N +first_col bigint Yes true \N + +-- !paimon_alter_final_schema -- +11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}] [] ["id"] + +-- !paimon_alter_failed_batch_schema -- +11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}] + +-- !paimon_alter_partition_initial_desc -- +id int Yes true \N +pt text Yes true \N +payload int Yes true \N + +-- !paimon_alter_partition_initial_schema -- +0 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"INT","description":""}] ["pt"] ["id","pt"] + +-- !paimon_alter_partition_final_desc -- +id int Yes true \N +pt text Yes true \N +payload bigint Yes true \N +extra text Yes true \N + +-- !paimon_alter_partition_final_schema -- +2 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"BIGINT","description":""},{"id":3,"name":"extra","type":"STRING","description":""}] ["pt"] ["id","pt"] + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_jdbc_catalog.out b/regression-test/data/external_table_p0/paimon/test_paimon_jdbc_catalog.out index 0866e5961ba874..0dc7742b94dacb 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_jdbc_catalog.out @@ -3,3 +3,19 @@ 1 alice 2025-01-01 2 bob 2025-01-02 +-- !paimon_jdbc_concurrent_append -- +left 128 128 8128 +right 128 128 136128 + +-- !paimon_jdbc_same_partition_append -- +128 128 324032 + +-- !paimon_jdbc_concurrent_aggregation -- +1 30 + +-- !paimon_jdbc_concurrent_dynamic -- +left 64 64 +right 64 64 + +-- !paimon_jdbc_merge_during_compact -- +2 merge-during-compact 200 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out index f1118d0bd7069e..ec0b7e7ffa76a1 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out @@ -6,10 +6,14 @@ -- !before_snapshots -- 1 --- !after_rows -- +-- !after_append_rows -- 1 10 base-1 2 20 base-2 +3 30 insert-values +4 40 insert-select --- !after_snapshots -- -1 +-- !after_rows -- +6 60 merge-insert +-- !after_snapshots -- +7 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.out new file mode 100644 index 00000000000000..15b6d4b36d7666 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.out @@ -0,0 +1,84 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !create_custom_location_absent -- + +-- !create_sequence_result -- +1 100 newer p1 +2 20 updated-2 p1 +3 5 initial-3 p2 + +-- !create_sequence_schema -- +["dt"] ["id","dt"] created by Doris with write properties + +-- !create_sequence_file_format -- +orc + +-- !create_partial_result -- +1 alice 15 updated +2 bob 20 initial + +-- !create_first_row_result -- +1 first-1 10 +2 first-2 20 +3 first-3 30 + +-- !create_aggregation_result -- +1 17 90 latest +2 8 70 stable +3 4 50 new + +-- !create_lookup_changelog -- ++I 1 new 11 ++I 2 stable 20 ++I 3 added 30 + +-- !create_lookup_result -- +1 new 11 +2 stable 20 +3 added 30 + +-- !create_dynamic_bucket_result -- +12 0 11 + +-- !create_dynamic_bucket_rows -- +p1 0 v0 +p1 1 v1 +p1 10 v10 +p1 11 v11 +p1 2 v2 +p1 3 v3 +p1 4 v4 +p1 5 v5 +p1 6 v6 +p1 7 v7 +p1 8 v8 +p1 9 v9 + +-- !create_dynamic_bucket_files -- +0 +1 +2 +3 + +-- !create_write_options -- +aggregation bucket 1 +aggregation fields.highest.aggregate-function max +aggregation fields.total.aggregate-function sum +aggregation merge-engine aggregation +dynamic_bucket bucket -1 +dynamic_bucket dynamic-bucket.initial-buckets 1 +dynamic_bucket dynamic-bucket.max-buckets 4 +dynamic_bucket dynamic-bucket.target-row-num 2 +first_row bucket 1 +first_row merge-engine first-row +lookup bucket 1 +lookup changelog-producer lookup +partial_update bucket 2 +partial_update bucket-key id +partial_update merge-engine partial-update +sequence bucket 2 +sequence bucket-key id +sequence file.format orc +sequence sequence.field seq +sequence snapshot.num-retained.max 5 +sequence snapshot.num-retained.min 2 + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_append_only.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_append_only.out new file mode 100644 index 00000000000000..384cd097d91eea --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_append_only.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ao_basic -- +1 alice 95.5 +2 bob 87 +3 charlie 92.3 + +-- !ao_part -- +1 alice 95.5 east +2 bob 87 west +3 charlie 92.3 east +4 diana 88 north +5 erin 86.5 south +6 \N \N east + +-- !ao_auto_partition_data -- +1 alpha 2026-07-01 +2 beta 2026-07-02 +3 gamma 2026-07-01 +4 delta 2026-07-01 +5 epsilon 2026-07-03 +6 default_partition \N + +-- !ao_auto_partition_metadata -- +dt=2026-07-01 3 +dt=2026-07-02 1 +dt=2026-07-03 1 +dt=__DEFAULT_PARTITION__ 1 + +-- !ao_empty -- +1 \N +2 reordered + +-- !ao_default_value -- +1 unknown + +-- !ao_default_after_explicit_null -- +1 unknown + +-- !ao_partition_default_data -- +1 omitted-partition 2026-07-01 + +-- !ao_partition_default_metadata -- +dt=2026-07-01 1 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_bucket_modes.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_bucket_modes.out new file mode 100644 index 00000000000000..55d921313e10fa --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_bucket_modes.out @@ -0,0 +1,63 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bucket_hash_fixed -- +16 0 15 2 + +-- !bucket_rescale_partial -- +p1 1 p1-old-1 +p1 10 p1-after-rescale +p1 2 p1-old-2 +p2 3 p2-old-3 +p2 4 p2-old-4 + +-- !bucket_hash_dynamic -- +p1 1 v1_updated +p1 2 v2 +p1 3 v3 +p1 4 v4_updated +p1 5 v5 +p1 6 v6 +p2 1 p2_v1 +p2 2 p2_v2_updated + +-- !bucket_hash_dynamic_partial -- +p1 1 alice 15 +p1 2 bob 20 +p1 3 \N 30 + +-- !bucket_hash_dynamic_overwrite -- +10 new_10 +11 new_11 +12 new_12 + +-- !bucket_key_dynamic -- +p2 1 id1_moved +p2 2 id2_stable +p2 4 id4_added +p3 3 id3_moved + +-- !bucket_key_dynamic_partial -- +p1 10 old_10 15 +p2 20 stable_20 20 +p3 30 \N 30 + +-- !bucket_key_dynamic_first_row -- +p1 1 first_1 +p2 2 first_2 + +-- !bucket_key_dynamic_aggregation -- +p1 1 17 +p2 2 20 + +-- !bucket_key_dynamic_scale_samples -- +p2 1023 txn2_1023 +p2 2047 txn2_2047 +p3 0 txn2_0 +p4 3071 txn3_3071 +p4 4095 txn3_4095 +p5 2048 txn3_2048 + +-- !bucket_unaware -- +32 0 31 2 + +-- !bucket_postpone -- +0 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_changelog_producer.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_changelog_producer.out new file mode 100644 index 00000000000000..590dd61fdc77ce --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_changelog_producer.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !changelog_input_partial -- +1 alice 15 +2 bob 20 +3 \N 30 + +-- !changelog_lookup -- +1 new 11 +2 stable 20 +3 added 30 + +-- !changelog_lookup_aggregation -- +1 17 +2 20 +3 30 + +-- !changelog_full_compaction -- +p1 1 new +p2 2 stable +p2 3 added + +-- !changelog_full_compaction_dynamic -- +p1 1 new_1 +p1 2 stable_2 +p1 4 added_4 +p2 3 new_3 +p2 5 added_5 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_compaction.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_compaction.out new file mode 100644 index 00000000000000..3cddf7ced90981 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_compaction.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !compaction_pk -- +1 new 11 +2 stable 20 +3 added 30 + +-- !compaction_append -- +1 a +2 b +3 c +4 d diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_complex_types.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_complex_types.out new file mode 100644 index 00000000000000..ed6709f3b2ec5c --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_complex_types.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cx_array -- +1 [1, 2, 3] ["a", "b", "c"] [1.1, 2.2] +2 [] [] [] +3 [10, null, 30] ["x", null, "z"] [null, 2] +4 \N \N \N + +-- !cx_map -- +1 {"math":90, "eng":95} {1:"one", 2:"two"} +2 {} {} +3 {"science":null} {3:null} +4 \N \N + +-- !cx_struct -- +1 {"name":"alice", "age":30} +2 {"name":null, "age":null} +3 \N + +-- !cx_nested -- +1 [1, 2] [3, 4, 5] \N +2 \N \N [] +3 \N \N \N + +-- !cx_recursive -- +1 [1.250000, -2.500000] ["2024-01-01", "2024-12-31"] ["2024-01-01 01:02:03.123456", "2024-12-31 23:59:59.654321"] {1.25:2.50, -3.75:4.00} {"flag":1, "amount":123.456789, "event_date":"2024-02-29", "event_time":"2024-02-29 12:34:56.000001"} {"term":[{"score":90, "label":"good"}, {"score":95, "label":"better"}]} +2 [null, 0.000001] [null, "1970-01-01"] [null, "1970-01-01 00:00:00.000001"] {5.25:null} {"flag":null, "amount":null, "event_date":null, "event_time":null} {"nullable":[{"score":null, "label":null}]} +3 [] [] [] {} {"flag":0, "amount":0.000000, "event_date":"1970-01-01", "event_time":"1970-01-01 00:00:00.000000"} {} +4 [8.800008] ["2025-01-01"] ["2025-01-01 00:00:00.000008"] {8.80:9.90} {"flag":1, "amount":8.800000, "event_date":"2025-01-01", "event_time":"2025-01-01 08:08:08.000008"} {"reverse":[{"score":88, "label":"reordered"}]} +5 \N ["2026-01-01"] \N \N \N {"partial":[{"score":77, "label":"subset"}]} + +-- !cx_binary -- +1 0001FEFF 2 41 1 102030 binary_1 DEADBEEF +2 \N 0 \N 0 \N binary_2 \N +3 E4B8ADE69687 \N \N \N \N \N \N +4 060708 2 03 1 0102 reordered ABCD + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_deletion_vector.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_deletion_vector.out new file mode 100644 index 00000000000000..a13f2ca16fc5f2 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_deletion_vector.out @@ -0,0 +1,38 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dv_before_compact_jni -- +1 merged-1 12 +4 insert-4 40 +5 inserted-5 50 + +-- !dv_before_compact_native -- +1 merged-1 12 +4 insert-4 40 +5 inserted-5 50 + +-- !dv_after_compact_jni -- +1 merged-1 12 +4 insert-4 40 +5 inserted-5 50 + +-- !dv_after_compact_native -- +1 merged-1 12 +4 insert-4 40 +5 inserted-5 50 + +-- !dv_post_compact_write_jni -- +1 post-compact-1 13 +4 insert-4 40 +5 inserted-5 50 +6 post-compact-6 60 + +-- !dv_post_compact_write_native -- +1 post-compact-1 13 +4 insert-4 40 +5 inserted-5 50 +6 post-compact-6 60 + +-- !dv_enabled_after_mor_jni -- +1 mow-new-1 + +-- !dv_enabled_after_mor_native -- +1 mow-new-1 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_edge_cases.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_edge_cases.out new file mode 100644 index 00000000000000..3d0418d4724068 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_edge_cases.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !edge_str -- +1 +2 short_str +3 abcdefghij +4 max10chars + +-- !edge_numeric -- +1 127 32767 2147483647 9223372036854775807 +2 -128 -32768 -2147483648 -9223372036854775808 +3 0 0 0 0 + +-- !edge_bool -- +1 true +2 false +3 \N + +-- !edge_pk_null -- +1 first +2 updated +3 third + +-- !edge_mixed -- +1 a 10 +2 b 20 +3 c 30 +4 a_copy 40 +5 b_copy 50 +6 c_copy 60 + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out new file mode 100644 index 00000000000000..e5763fe6c94693 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out @@ -0,0 +1,77 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !external_round_robin_initial -- +p-bulk 100 2048 +p-bulk 101 2048 +p-bulk 102 2048 +p-bulk 103 2048 +p-bulk 104 2048 +p-bulk 105 2048 +p-bulk 106 2048 +p-bulk 107 2048 +p-bulk 108 2048 +p-bulk 109 2048 +p-bulk 110 2048 +p-bulk 111 2048 +p-bulk 112 2048 +p-bulk 113 2048 +p-bulk 114 2048 +p-bulk 115 2048 +p1 1 3 +p1 2 3 +p2 3 5 +p2 4 4 + +-- !external_round_robin_changed -- +p-bulk 100 2048 +p-bulk 101 2048 +p-bulk 102 2048 +p-bulk 103 2048 +p-bulk 104 2048 +p-bulk 105 2048 +p-bulk 106 2048 +p-bulk 107 2048 +p-bulk 108 2048 +p-bulk 109 2048 +p-bulk 110 2048 +p-bulk 111 2048 +p-bulk 112 2048 +p-bulk 113 2048 +p-bulk 114 2048 +p-bulk 115 2048 +p-new-bulk 200 2048 +p-new-bulk 201 2048 +p-new-bulk 202 2048 +p-new-bulk 203 2048 +p-new-bulk 204 2048 +p-new-bulk 205 2048 +p-new-bulk 206 2048 +p-new-bulk 207 2048 +p-new-bulk 208 2048 +p-new-bulk 209 2048 +p-new-bulk 210 2048 +p-new-bulk 211 2048 +p-new-bulk 212 2048 +p-new-bulk 213 2048 +p-new-bulk 214 2048 +p-new-bulk 215 2048 +p1 1 3 +p1 2 3 +p2 3 5 +p2 4 4 +p3 5 4 +p3 6 3 + +-- !external_weight_robin -- +1 weight-1 +2 weight-2 +3 weight-3 +4 weight-4 +5 weight-5 +6 weight-6 + +-- !external_specific_fs -- +1 specific-1 +2 specific-2 + +-- !external_default_path -- +1 default-path diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_failures.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_failures.out new file mode 100644 index 00000000000000..02d2e530b2672a --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_failures.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !failure_atomic_before -- +1 baseline p0 + +-- !failure_atomic_snapshot_before -- +1 + +-- !failure_atomic_after -- +1 baseline p0 + +-- !failure_atomic_snapshot_after -- +1 + +-- !failure_recovered -- +1 baseline p0 +5 recovered p5 + +-- !failure_recovered_snapshot -- +2 + +-- !failure_overwrite_after -- +1 baseline p0 +5 recovered p5 + +-- !failure_overwrite_snapshot_after -- +2 + +-- !failure_pk_recovered -- +1 valid_after_failure + +-- !failure_pk_snapshot -- +1 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_engine.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_engine.out new file mode 100644 index 00000000000000..9354ca3d8e5e02 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_engine.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partial_update -- +1 alice 15.5 score_updated +2 bob_full 25 full_update +3 charlie \N \N + +-- !first_row -- +1 first_1 10 +2 first_2 20 +3 first_3 30 + +-- !aggregation -- +1 37 95 latest_1 +2 8 80 first_2 +3 7 60 first_3 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_semantics.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_semantics.out new file mode 100644 index 00000000000000..5db9108a54830a --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_semantics.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !merge_semantics_result -- +1 15 11 base-1-source-1 updated required-1-new +3 30 3 base-3 stable required-3 +4 40 44 source-4 inserted required-4 + +-- !merge_semantics_recovered -- +5 recovered diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_partition_delete.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_partition_delete.out new file mode 100644 index 00000000000000..92f03fca8c8e4f --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_partition_delete.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partition_delete_full_partition -- +\N 6 60 default-partition +p2 3 30 p2-a +p2 4 40 p2-b +p3 5 50 p3-a + +-- !partition_delete_partial_partition -- +\N 6 60 default-partition +p2 3 30 p2-a +p3 5 50 p3-a + +-- !partition_delete_expression -- +\N 6 60 default-partition +p3 5 50 p3-a + +-- !partition_delete_not_exists -- +\N 6 60 default-partition + +-- !partition_delete_default_partition -- diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_pk.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_pk.out new file mode 100644 index 00000000000000..5e0eddd7312e0f --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_pk.out @@ -0,0 +1,52 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pk_dedup -- +1 alice 95.5 +2 bob 87 +3 charlie 92.3 +4 diana 91 +5 eve 85 + +-- !pk_interleaved -- +100 key100_v3 12 3000 +200 key200_v2 21 2000 + +-- !pk_bucket -- +0 row0 +1 row1 +10 row10 +11 row11 +12 row12 +13 row13 +14 row14 +15 row15 +16 row16 +17 row17 +18 row18 +19 row19 +2 row2 +3 row3 +4 row4 +5 row5 +6 row6 +7 row7 +8 row8 +9 row9 + +-- !pk_composite -- +1 100 click_updated 99 +1 200 view 2 +2 100 click 3 + +-- !pk_string_bucket -- +alpha 1 alpha_v2 +beta 2 δΈ­ζ–‡_payload +emoji_πŸ˜€ 3 emoji_payload + +-- !pk_writer_scaling_plan -- +PhysicalPaimonTableSink +--PhysicalDistribute[DistributionSpecPaimonTableSinkHashPartitioned] +----PhysicalProject +------PhysicalTVFRelation + +-- !pk_writer_scaling -- +1 1 1 4096 4096 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_level_dml.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_level_dml.out new file mode 100644 index 00000000000000..99fd08cf36640d --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_level_dml.out @@ -0,0 +1,49 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !paimon_update -- +1 Alice_updated 11 active +2 Bob_updated 21 active +3 Charlie 30 active +4 Diana 40 active + +-- !paimon_delete -- +1 Alice_updated 11 active +2 Bob_updated 21 active +4 Diana 40 active + +-- !paimon_merge -- +1 Alice_merged 100 merged +4 Diana 40 active +5 Eve 50 inserted + +-- !paimon_merge_branch_priority -- +1 priority_update 101 first-matched +4 Diana 40 active +5 Eve 50 inserted +6 priority_insert 60 first-not-matched + +-- !paimon_merge_duplicate_unchanged -- +1 priority_update 101 first-matched +4 Diana 40 active +5 Eve 50 inserted +6 priority_insert 60 first-not-matched + +-- !paimon_merge_duplicate_insert_unchanged -- +1 priority_update 101 first-matched +4 Diana 40 active +5 Eve 50 inserted +6 priority_insert 60 first-not-matched + +-- !paimon_merge_many_unmatched -- +1024 + +-- !paimon_ignore_delete_unchanged -- +10 keep + +-- !paimon_partial_update_delete -- +0 + +-- !paimon_aggregation_no_delete_unchanged -- +30 30 + +-- !paimon_aggregation_delete -- +0 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.out new file mode 100644 index 00000000000000..1f71d511f7db26 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !row_tracking_after_spark_changes -- +1 one-merged +2 two-updated +4 four + +-- !row_tracking_after_compact_write -- +1 one-merged +2 two-updated +4 four +5 five-after-compact + +-- !data_evolution_after_spark_merge -- +1 11 100 +2 22 200 +3 30 \N +4 44 444 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_schema_change.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_schema_change.out new file mode 100644 index 00000000000000..5931eef1acee3b --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_schema_change.out @@ -0,0 +1,532 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sc_append_initial -- +1 100 alice 10 1.10 old-a 2026-07-01 +2 200 bob 20 2.20 old-b 2026-07-02 + +-- !sc_add_after_before_insert -- +1 100 alice 10 \N 1.10 old-a 2026-07-01 +2 200 bob 20 \N 2.20 old-b 2026-07-02 + +-- !sc_add_after_after_insert -- +1 100 alice 10 \N 1.10 old-a 2026-07-01 +2 200 bob 20 \N 2.20 old-b 2026-07-02 +3 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 400 dave 40 unknown 4.40 old-d 2026-07-01 + +-- !sc_add_default_omitted -- +100 unknown + +-- !sc_add_first_before_insert -- +1 \N 100 alice 10 \N 1.10 old-a 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d 2026-07-01 + +-- !sc_add_first_after_insert -- +1 \N 100 alice 10 \N 1.10 old-a 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e 2026-07-04 + +-- !sc_add_columns_before_insert -- +1 \N 100 alice 10 \N 1.10 old-a \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e \N \N 2026-07-04 + +-- !sc_add_columns_after_insert -- +1 \N 100 alice 10 \N 1.10 old-a \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 old-f 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 old-partial \N \N 2026-07-05 + +-- !sc_drop_before_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 + +-- !sc_drop_after_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 + +-- !sc_rename_before_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 + +-- !sc_rename_after_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 +8 8000 800 heidi 80 after-rename 8.80 8 800 2026-07-02 + +-- !sc_modify_bigint_before_insert -- +1 alice 10 1.10 100 2026-07-01 +100 default-value 100 100.00 10000 2026-07-10 +2 bob 20 2.20 200 2026-07-02 +3 carol 30 3.30 300 2026-07-03 +4 dave 40 4.40 400 2026-07-01 +5 erin 50 5.50 500 2026-07-04 +6 frank 60 6.60 600 2026-07-05 +60 partial-columns 600 60.60 6000 2026-07-05 +7 grace 70 7.70 700 2026-07-06 +8 heidi 80 8.80 800 2026-07-02 + +-- !sc_modify_bigint_after_insert -- +1 alice 10 1.10 100 2026-07-01 +100 default-value 100 100.00 10000 2026-07-10 +2 bob 20 2.20 200 2026-07-02 +3 carol 30 3.30 300 2026-07-03 +4 dave 40 4.40 400 2026-07-01 +5 erin 50 5.50 500 2026-07-04 +6 frank 60 6.60 600 2026-07-05 +60 partial-columns 600 60.60 6000 2026-07-05 +7 grace 70 7.70 700 2026-07-06 +8 heidi 80 8.80 800 2026-07-02 +9 ivan 3000000000 9.90 900 2026-07-07 + +-- !sc_modify_decimal_before_insert -- +1 alice 10 1.10 2026-07-01 +100 default-value 100 100.00 2026-07-10 +2 bob 20 2.20 2026-07-02 +3 carol 30 3.30 2026-07-03 +4 dave 40 4.40 2026-07-01 +5 erin 50 5.50 2026-07-04 +6 frank 60 6.60 2026-07-05 +60 partial-columns 600 60.60 2026-07-05 +7 grace 70 7.70 2026-07-06 +8 heidi 80 8.80 2026-07-02 +9 ivan 3000000000 9.90 2026-07-07 + +-- !sc_modify_decimal_after_insert -- +1 alice 10 1.10 2026-07-01 +10 judy 100 1234567890.12 2026-07-08 +100 default-value 100 100.00 2026-07-10 +2 bob 20 2.20 2026-07-02 +3 carol 30 3.30 2026-07-03 +4 dave 40 4.40 2026-07-01 +5 erin 50 5.50 2026-07-04 +6 frank 60 6.60 2026-07-05 +60 partial-columns 600 60.60 2026-07-05 +7 grace 70 7.70 2026-07-06 +8 heidi 80 8.80 2026-07-02 +9 ivan 3000000000 9.90 2026-07-07 + +-- !sc_modify_nullable_before_insert -- +1 alice 100 2026-07-01 +10 judy 1000 2026-07-08 +100 default-value 10000 2026-07-10 +2 bob 200 2026-07-02 +3 carol 300 2026-07-03 +4 dave 400 2026-07-01 +5 erin 500 2026-07-04 +6 frank 600 2026-07-05 +60 partial-columns 6000 2026-07-05 +7 grace 700 2026-07-06 +8 heidi 800 2026-07-02 +9 ivan 900 2026-07-07 + +-- !sc_modify_nullable_after_insert -- +1 alice 100 2026-07-01 +10 judy 1000 2026-07-08 +100 default-value 10000 2026-07-10 +11 kate \N 2026-07-09 +2 bob 200 2026-07-02 +3 carol 300 2026-07-03 +4 dave 400 2026-07-01 +5 erin 500 2026-07-04 +6 frank 600 2026-07-05 +60 partial-columns 6000 2026-07-05 +7 grace 700 2026-07-06 +8 heidi 800 2026-07-02 +9 ivan 900 2026-07-07 + +-- !sc_modify_metadata_desc -- +added_after text Yes true \N changed comment +first_col bigint Yes true \N +id int Yes true \N +required_value bigint Yes true \N +full_name text Yes true \N +score bigint Yes true \N +amount decimal(12,2) Yes true \N +dt text Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small integer + +-- !sc_modify_metadata_before_insert -- +1 alice \N 10 2026-07-01 +10 judy after-decimal 100 2026-07-08 +100 default-value unknown 100 2026-07-10 +11 kate after-nullable 110 2026-07-09 +2 bob \N 20 2026-07-02 +3 carol added-3 30 2026-07-03 +4 dave unknown 40 2026-07-01 +5 erin added-first 50 2026-07-04 +6 frank added-columns 60 2026-07-05 +60 partial-columns explicit-default-column 600 2026-07-05 +7 grace after-drop 70 2026-07-06 +8 heidi after-rename 80 2026-07-02 +9 ivan after-bigint 3000000000 2026-07-07 + +-- !sc_modify_default_omitted -- +120 changed-default + +-- !sc_modify_metadata_after_insert -- +1 alice \N 10 2026-07-01 +10 judy after-decimal 100 2026-07-08 +100 default-value unknown 100 2026-07-10 +11 kate after-nullable 110 2026-07-09 +12 leo after-metadata 120 2026-07-10 +120 modified-default changed-default 1200 2026-07-10 +2 bob \N 20 2026-07-02 +3 carol added-3 30 2026-07-03 +4 dave unknown 40 2026-07-01 +5 erin added-first 50 2026-07-04 +6 frank added-columns 60 2026-07-05 +60 partial-columns explicit-default-column 600 2026-07-05 +7 grace after-drop 70 2026-07-06 +8 heidi after-rename 80 2026-07-02 +9 ivan after-bigint 3000000000 2026-07-07 + +-- !sc_modify_remove_metadata_desc -- +first_col bigint Yes true \N +id int Yes true \N +required_value bigint Yes true \N +full_name text Yes true \N +score bigint Yes true \N +added_after text Yes true \N +amount decimal(12,2) Yes true \N +dt text Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small integer + +-- !sc_modify_remove_metadata_before_insert -- +1 alice 10 \N 2026-07-01 +10 judy 100 after-decimal 2026-07-08 +100 default-value 100 unknown 2026-07-10 +11 kate 110 after-nullable 2026-07-09 +12 leo 120 after-metadata 2026-07-10 +120 modified-default 1200 changed-default 2026-07-10 +2 bob 20 \N 2026-07-02 +3 carol 30 added-3 2026-07-03 +4 dave 40 unknown 2026-07-01 +5 erin 50 added-first 2026-07-04 +6 frank 60 added-columns 2026-07-05 +60 partial-columns 600 explicit-default-column 2026-07-05 +7 grace 70 after-drop 2026-07-06 +8 heidi 80 after-rename 2026-07-02 +9 ivan 3000000000 after-bigint 2026-07-07 + +-- !sc_remove_default_omitted -- +130 \N + +-- !sc_modify_remove_metadata_after_insert -- +1 alice 10 \N 2026-07-01 +10 judy 100 after-decimal 2026-07-08 +100 default-value 100 unknown 2026-07-10 +11 kate 110 after-nullable 2026-07-09 +12 leo 120 after-metadata 2026-07-10 +120 modified-default 1200 changed-default 2026-07-10 +13 mallory 130 after-remove-metadata 2026-07-11 +130 removed-default 1300 \N 2026-07-11 +2 bob 20 \N 2026-07-02 +3 carol 30 added-3 2026-07-03 +4 dave 40 unknown 2026-07-01 +5 erin 50 added-first 2026-07-04 +6 frank 60 added-columns 2026-07-05 +60 partial-columns 600 explicit-default-column 2026-07-05 +7 grace 70 after-drop 2026-07-06 +8 heidi 80 after-rename 2026-07-02 +9 ivan 3000000000 after-bigint 2026-07-07 + +-- !sc_reorder_before_insert -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_reorder_after_insert -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_after_failed_alters -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +15 olivia 150 15.15 \N after-failed-alters 15000 15 1500 2026-07-13 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_after_partition_evolution_failures -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +15 olivia 150 15.15 \N after-failed-alters 15000 15 1500 2026-07-13 +16 peggy 160 16.16 1600 after-partition-evolution-failures 16000 16 1600 2026-07-14 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_append_partitions -- +dt=2026-07-01 2 +dt=2026-07-02 2 +dt=2026-07-03 1 +dt=2026-07-04 1 +dt=2026-07-05 2 +dt=2026-07-06 1 +dt=2026-07-07 1 +dt=2026-07-08 1 +dt=2026-07-09 1 +dt=2026-07-10 3 +dt=2026-07-11 2 +dt=2026-07-12 1 +dt=2026-07-13 1 +dt=2026-07-14 1 + +-- !sc_types_initial -- +1 100 30000 2000000000 1.5 123456.78 + +-- !sc_types_tiny_to_small_before_insert -- +1 100 30000 2000000000 1.5 123456.78 + +-- !sc_types_tiny_to_small_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 + +-- !sc_types_small_to_int_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 + +-- !sc_types_small_to_int_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 + +-- !sc_types_int_to_bigint_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 + +-- !sc_types_int_to_bigint_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 + +-- !sc_types_float_to_double_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 + +-- !sc_types_float_to_double_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 + +-- !sc_types_decimal_widen_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 + +-- !sc_types_decimal_widen_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 +6 204 40003 3000000002 2e+40 1234567890.12 + +-- !sc_types_after_failed_narrow -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 +6 204 40003 3000000002 2e+40 1234567890.12 +7 205 40004 3000000003 3e+40 1234567890.13 + +-- !sc_explicit_types_initial -- +1 100 +2 200 + +-- !sc_explicit_bigint_to_int_before_insert -- +1 100 +2 200 + +-- !sc_explicit_bigint_to_int_after_insert -- +1 100 +2 200 +3 300 + +-- !sc_explicit_int_to_string_before_insert -- +1 100 +2 200 +3 300 + +-- !sc_explicit_int_to_string_after_insert -- +1 100 +2 200 +3 300 +4 after-explicit-cast + +-- !sc_pk_initial -- +1 2026-08-01 10 pk-a +2 2026-08-01 20 pk-b + +-- !sc_pk_add_before_insert -- +1 2026-08-01 10 \N pk-a +2 2026-08-01 20 \N pk-b + +-- !sc_pk_add_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 20 \N pk-b +3 2026-08-02 30 new-after-add pk-c + +-- !sc_pk_rename_before_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 20 \N pk-b +3 2026-08-02 30 new-after-add pk-c + +-- !sc_pk_rename_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d + +-- !sc_pk_type_before_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d + +-- !sc_pk_type_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d +5 2026-08-03 3000000000 new-after-type pk-e + +-- !sc_pk_drop_before_insert -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type + +-- !sc_pk_partial_default -- +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + +-- !sc_pk_drop_after_insert -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type +6 2026-08-03 60 new-after-drop +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + +-- !sc_pk_after_key_failures -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type +6 2026-08-03 60 new-after-drop +7 2026-08-04 70 after-key-failures +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_group.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_group.out new file mode 100644 index 00000000000000..1c72d08a0cc28a --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_group.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sequence_group_stale_profile -- +1 alice shanghai 10 12 90 11 base + +-- !sequence_group_new_profile -- +1 alice-new shenzhen 12 112 99 11 new-note + +-- !sequence_group_null_sequence -- +1 alice-new shenzhen 12 115 99 13 new-note + +-- !remove_on_delete_empty -- +0 + +-- !remove_on_delete_partial -- +1 old-a new-b + +-- !remove_on_delete_complete -- +1 new-a new-b + +-- !group_remove_on_delete_unchanged -- +1 old-a 100 old-b 100 + +-- !group_remove_on_delete_sequence -- +1 high-a 101 old-b 100 + +-- !property_change_still_writable -- +1 still-writable 10 + +-- !property_change_sequence_group -- +1 high 20 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.out new file mode 100644 index 00000000000000..5b1cf8b4496de1 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sequence_ascending -- +1 10 21 new-second-field +2 1 0 nonnull-wins + +-- !sequence_equal -- +3 second-equal + +-- !sequence_descending -- +1 5 smaller-wins + +-- !rowkind_changelog -- +1 new-1 +3 new-3 + +-- !rowkind_recovered -- +4 recovered diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out new file mode 100644 index 00000000000000..37eaca4cb5bcc9 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out @@ -0,0 +1,150 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !txn_commit -- +1 alice +2 bob +3 charlie +4 diana + +-- !txn_batch -- +1 1 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +2 2 +20 20 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 + +-- !txn_overwrite -- +10 new1 +20 new2 + +-- !txn_empty_overwrite -- +0 + +-- !txn_empty_overwrite_limit_zero -- +0 + +-- !txn_static_partition -- +10 east_new east +2 west_old west + +-- !txn_static_partition_case -- +10 east_new east +2 west_old west + +-- !txn_static_partial -- +10 new_a 1 A +3 keep 2 C + +-- !txn_static_partial_empty -- +3 keep 2 C + +-- !txn_static_null -- +10 null_new \N +2 literal_null null +3 blank_old +4 east_old east + +-- !txn_static_empty -- +10 null_new \N +2 literal_null null +3 blank_old + +-- !txn_static_blank -- +10 null_new \N +2 literal_null null +30 blank_new + +-- !txn_static_typed_boundaries -- +10 null_new \N 2026-07-03 +3 blank 2026-07-01 +4 literal_null null 2026-07-01 +50 special_new a/b=c%20 2026-07-04 +7 keep keep 2026-07-01 + +-- !txn_dynamic_multi -- +10 p1_new p1 +20 p2_new_a p2 +21 p2_new_b p2 +4 p3_keep p3 +5 p4_keep p4 + +-- !txn_dynamic_partition -- +10 east_new east +2 west_old west +30 south_new south + +-- !txn_unsupported_partition_syntax -- +10 east_new east +2 west_old west +30 south_new south + +-- !txn_parallel_writers -- +256 0 255 32640 + +-- !txn_multi_block -- +20480 0 20479 209704960 18618 8 + +-- !txn_multi_block_samples -- +0 0 payload_0 \N p0 +16383 87 payload_16383 49149 p7 +16384 88 payload_16384 49152 p0 +20479 12 payload_20479 61437 p7 +4095 21 payload_4095 12285 p7 +4096 22 payload_4096 12288 p0 +8191 43 payload_8191 24573 p7 +8192 44 payload_8192 24576 p0 + +-- !txn_multi_block_snapshots -- +2 + +-- !txn_spill -- +2048 0 2047 2096128 + +-- !txn_failed_write_before -- +1 committed_before_failure + +-- !txn_failed_snapshot_before -- +1 + +-- !txn_failed_write_after -- +1 committed_before_failure + +-- !txn_failed_snapshot_after -- +1 + +-- !txn_multi -- +1 a 10 +10 j 100 +11 k 110 +12 l 120 +13 m 130 +14 n 140 +15 o 150 +16 p 160 +17 q 170 +18 r 180 +19 s 190 +2 b 20 +20 t 200 +3 c 30 +4 d 40 +5 e 50 +6 f 60 +7 g 70 +8 h 80 +9 i 90 + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_types.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_types.out new file mode 100644 index 00000000000000..a12d29c2c5ce01 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_types.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !types_basic -- +false -2147483648 -9223372036854775808 -3.4E38 -1.7e+308 -1.50 long_string_1234567890 max_varchar 2099-12-31 2099-12-31T23:59:59 +false 2147483647 9223372036854775807 3.4E38 1.7e+308 0.00 1970-01-01 1970-01-01T00:00 +true 0 0 0.0 0 12345678.90 hello fixed_len 2024-06-15 2024-06-15T12:00:00.123456 +true 1 100 1.5 2.71828 99.99 hello short 2024-01-15 2024-01-15T10:30 + +-- !types_null -- +1 100 data 1.5 true +2 \N \N \N \N +3 \N partial 2 false + +-- !types_decimal -- +1 1.5 12345678.90 123456789012.123456 1234567890123456789012345678.1234567890 +2 -1.5 -0.01 -1.000001 1E-10 +3 0.0 0.00 0.000000 0E-10 + +-- !types_dt -- +1970-01-01 1970-01-01T00:00 +2024-06-15 2024-06-15T12:00 +2099-12-31 2099-12-31T23:59:59.999999 + +-- !desc_types_timezone -- +event_time datetime(6) Yes true \N WITH_TIMEZONE + +-- !types_timezone_utc -- +1 2024-01-15T02:30:00.123456 +2 2024-01-15T02:30:00.654321 + +-- !types_timezone_shanghai -- +1 2024-01-15T10:30:00.123456 +2 2024-01-15T10:30:00.654321 + +-- !types_ntz -- +1 2024-03-10T02:30:00.123456 +2 2024-01-15T10:30:00.654321 + diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant.out new file mode 100644 index 00000000000000..c5a8e2653798ec --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant.out @@ -0,0 +1,26 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_heterogeneous -- +20 1 row-one +21 row-two 2 + +-- !variant_object -- +doris Hangzhou 1 true {} [] δΈ­ζ–‡πŸ˜€ 2 + +-- !variant_nulls -- +4 null false true +5 \N true true + +-- !variant_scalars -- +10 true false +11 -128 32767 +12 -2147483648 9223372036854775807 +13 1.25 -2.5 +14 123456.789 -0.000001 +15 plain-string δΈ­ζ–‡πŸ˜€ +16 2024-02-29 2024-02-29 12:34:56.123456 + +-- !variant_long_string -- +45056 large-string + +-- !variant_row_count -- +15 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_dml.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_dml.out new file mode 100644 index 00000000000000..c5b69cc6d55f39 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_dml.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_dml_rows -- +1 {"n":1,"source":"direct"} default-note p1 false +10 {"mode":"reordered"} reordered p3 false +11 {"mode":"default"} default-note p3 false +12 \N default-note p3 true +2 ["direct",2] default-note p1 false +20 {"mode":"cte"} cte-note p4 false +21 {"mode":"union-a"} union p4 false +22 22 union p4 false +3 null default-note p2 false +30 {"generated":0} generated p5 false +37 {"generated":7} generated p5 false +4 \N default-note p2 true +50 {"partition":"static"} static-note static false +51 {"partition":"dynamic-a"} dynamic dynamic-a false +52 {"partition":"dynamic-b"} dynamic dynamic-b false + +-- !variant_partition_overwrite -- +10 {"state":"new-east"} east +2 {"state":"old-west"} west + +-- !variant_full_overwrite -- +20 {"state":"full-a"} all +21 {"state":"full-b"} all + +-- !variant_empty_overwrite -- +0 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out new file mode 100644 index 00000000000000..f1b82f09fbad1e --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_after_errors -- +20 false true {"recovered":true} +21 false \N not-json +22 false \N {"typed":"string"} +23 false \N 7 +24 false \N values-string +25 false \N true +26 false \N [1,2] +27 false \N 8 +28 false \N select-string +29 false \N false +30 false \N [3,4] diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_nested.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_nested.out new file mode 100644 index 00000000000000..e779da3b1426b2 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_nested.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_nested_values -- +array-object null \N 7 2 null \N struct-object first second + +-- !variant_nested_containers -- +2 false 0 false 0 false +3 true \N true \N true + +-- !variant_deep_value -- +1 depth-1 deep-ok + +-- !variant_deep_null -- +true + +-- !variant_deep_count -- +2 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_shredding.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_shredding.out new file mode 100644 index 00000000000000..9bfdb63cb0e664 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_shredding.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_explicit_shredding -- +1 27 Beijing true alice 10 \N kept false +2 28 \N \N \N \N \N \N false +3 29 \N true \N \N \N \N false +4 \N \N \N \N \N \N \N false +5 \N \N \N \N \N \N \N false +6 \N \N \N \N \N \N \N false +7 \N \N \N \N \N \N \N true +8 \N \N \N bob 30 nested-kept root-kept false + +-- !variant_mixed_layout -- +100 100 old \N \N +101 \N \N residual \N +200 200 new \N kept +201 201 \N \N \N + +-- !variant_inferred_shredding -- +300 30 alice \N \N first +301 31 bob \N \N second +400 \N \N Hangzhou true third +401 \N \N Shanghai false fourth diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.out new file mode 100644 index 00000000000000..55e403ae05ecfe --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !variant_pk -- +1 v3 100 3 +2 stable 2 1 + +-- !variant_dynamic_bucket -- +32 496 0 + +-- !variant_schema_evolution -- +1 before \N \N +2 after-add added \N +3 after-normal-column continued default-note + +-- !non_variant -- +1 {"plain":"string"} diff --git a/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy new file mode 100644 index 00000000000000..bccafaad4ae7c7 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("paimon_schema_change_ddl", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "paimon_schema_change_ddl" + String dbName = "paimon_schema_change_ddl_db" + String tableName = "paimon_alter_table" + String partitionTableName = "paimon_alter_partition_table" + + def schemaId = { String table -> + def rows = sql """ + SELECT MAX(schema_id) + FROM `${table}\$schemas` + """ + assertEquals(1, rows.size()) + return (rows[0][0] as Number).longValue() + } + + def columnNames = { String table -> + return sql("DESC `${table}`").collect { row -> row[0].toString() } + } + + def assertColumnOrder = { String table, List expected -> + assertEquals(expected, columnNames(table)) + } + + def assertColumnAbsent = { String table, String column -> + assertFalse(columnNames(table).any { name -> name.equalsIgnoreCase(column) }) + } + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + sql """SET show_column_comment_in_describe = true""" + + try { + // This suite intentionally contains no INSERT. ALTER correctness is + // verified from Doris metadata and Paimon's schema history table. + // Use strict type evolution so narrowing conversions are covered as + // deterministic failures; Paimon permits explicit casts by default. + sql """ + CREATE TABLE `${tableName}` ( + id INT NOT NULL COMMENT 'identifier', + required_value BIGINT NOT NULL, + score INT NULL DEFAULT '1' COMMENT 'initial score', + `MixedCase` STRING NULL, + obsolete STRING NULL, + amount DECIMAL(8, 2) NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'disable-explicit-type-casting' = 'true' + ) + """ + + assertColumnOrder( + tableName, + ["id", "required_value", "score", "MixedCase", "obsolete", "amount"]) + qt_paimon_alter_initial_desc """DESC `${tableName}`""" + qt_paimon_alter_initial_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // ADD COLUMN: default, comment and AFTER position are committed as one + // Paimon schema version. + long beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN added_after STRING NULL DEFAULT 'unknown' + COMMENT 'added column' AFTER score + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + assertColumnOrder( + tableName, + [ + "id", "required_value", "score", "added_after", + "MixedCase", "obsolete", "amount" + ]) + qt_paimon_alter_add_column_desc """DESC `${tableName}`""" + qt_paimon_alter_add_column_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // ADD COLUMNS is one Doris clause and one atomic Paimon schema commit. + // TINYINT and SMALLINT also cover the narrow integer type mapping. + beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` ADD COLUMN ( + tiny_col TINYINT NULL, + small_col SMALLINT NULL COMMENT 'small column', + profile STRUCT NULL + ) + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + qt_paimon_alter_add_columns_desc """DESC `${tableName}`""" + + // FIRST position. + sql """ALTER TABLE `${tableName}` ADD COLUMN first_col BIGINT NULL FIRST""" + assertColumnOrder( + tableName, + [ + "first_col", "id", "required_value", "score", "added_after", + "MixedCase", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + qt_paimon_alter_add_first_desc """DESC `${tableName}`""" + + // Doris resolves column names case-insensitively but sends the canonical + // remote field name to Paimon. + sql """ALTER TABLE `${tableName}` RENAME COLUMN mixedcase display_name""" + assertColumnOrder( + tableName, + [ + "first_col", "id", "required_value", "score", "added_after", + "display_name", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + qt_paimon_alter_rename_column_desc """DESC `${tableName}`""" + + // MODIFY COLUMN: widening type, nullability, default, comment and + // position changes are committed together. + beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` + MODIFY COLUMN score BIGINT NULL DEFAULT '10' + COMMENT 'updated score' FIRST + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + assertColumnOrder( + tableName, + [ + "score", "first_col", "id", "required_value", "added_after", + "display_name", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + + // Additional supported widening conversions and NOT NULL -> NULL. + sql """ALTER TABLE `${tableName}` MODIFY COLUMN small_col INT NULL""" + sql """ALTER TABLE `${tableName}` MODIFY COLUMN amount DECIMAL(12, 2) NULL""" + sql """ALTER TABLE `${tableName}` MODIFY COLUMN required_value BIGINT NULL""" + + // Omitting DEFAULT and COMMENT in a full MODIFY definition removes + // their existing values. + sql """ALTER TABLE `${tableName}` MODIFY COLUMN added_after STRING NULL""" + qt_paimon_alter_modify_column_desc """DESC `${tableName}`""" + qt_paimon_alter_modify_column_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + sql """ALTER TABLE `${tableName}` DROP COLUMN obsolete""" + assertColumnAbsent(tableName, "obsolete") + qt_paimon_alter_drop_column_desc """DESC `${tableName}`""" + + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score, required_value, small_col, + tiny_col, added_after, amount, profile, first_col + ) + """ + assertColumnOrder( + tableName, + [ + "id", "display_name", "score", "required_value", "small_col", + "tiny_col", "added_after", "amount", "profile", "first_col" + ]) + qt_paimon_alter_reorder_columns_desc """DESC `${tableName}`""" + qt_paimon_alter_final_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // A failed ADD COLUMNS must not publish the valid prefix of the batch. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` ADD COLUMN ( + batch_ok INT NULL, + batch_bad INT NOT NULL DEFAULT '1' + ) + """ + exception "cannot specify NOT NULL" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + assertColumnAbsent(tableName, "batch_ok") + assertColumnAbsent(tableName, "batch_bad") + qt_paimon_alter_failed_batch_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // Multiple Doris ALTER clauses cannot be committed atomically by an + // external catalog, so they are rejected before the first mutation. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN multi_a INT NULL, + ADD COLUMN multi_b INT NULL + """ + exception "External table does not support multiple ALTER clauses" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + assertColumnAbsent(tableName, "multi_a") + assertColumnAbsent(tableName, "multi_b") + + // Paimon SDK schema validation. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN required_col INT NOT NULL DEFAULT '1' + """ + exception "cannot specify NOT NULL" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` MODIFY COLUMN score INT NULL""" + exception "cannot be converted" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` + MODIFY COLUMN added_after STRING NOT NULL DEFAULT 'unknown' + """ + exception "Cannot update column type from nullable to non nullable" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` DROP COLUMN id""" + exception "Cannot drop partition key or primary key" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + // Doris/Paimon adapter validation which cannot be delegated to the SDK. + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN ID INT NULL""" + exception "conflicts with an existing Paimon column" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` RENAME COLUMN display_name ID""" + exception "conflicts with an existing Paimon column" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN agg_col INT SUM NULL""" + exception "does not support aggregation method" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN auto_col BIGINT AUTO_INCREMENT""" + exception "does not support AUTO_INCREMENT" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN generated_col INT AS (score + 1)""" + exception "cannot be a generated column in a Paimon table" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN bad_position INT NULL AFTER missing_col""" + exception "does not exist in Paimon table" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score + ) + """ + exception "must contain every Paimon column exactly once" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score, required_value, small_col, + tiny_col, added_after, amount, profile, id + ) + """ + exception "Duplicate column in reorder columns" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + // Partition and primary-key constraints are delegated to Paimon. + sql """ + CREATE TABLE `${partitionTableName}` ( + id INT NOT NULL, + pt STRING NOT NULL, + payload INT NULL + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'primary-key' = 'id,pt' + ) + """ + qt_paimon_alter_partition_initial_desc """DESC `${partitionTableName}`""" + qt_paimon_alter_partition_initial_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${partitionTableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + long partitionSchemaId = schemaId(partitionTableName) + test { + sql """ALTER TABLE `${partitionTableName}` DROP COLUMN pt""" + exception "Cannot drop partition key or primary key" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + test { + sql """ALTER TABLE `${partitionTableName}` RENAME COLUMN pt partition_col""" + exception "Cannot rename partition column" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + test { + sql """ALTER TABLE `${partitionTableName}` MODIFY COLUMN pt INT NOT NULL""" + exception "Cannot update partition column" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + // Non-key columns of a partitioned table can still evolve. + sql """ALTER TABLE `${partitionTableName}` MODIFY COLUMN payload BIGINT NULL""" + sql """ALTER TABLE `${partitionTableName}` ADD COLUMN extra STRING NULL""" + assertColumnOrder(partitionTableName, ["id", "pt", "payload", "extra"]) + qt_paimon_alter_partition_final_desc """DESC `${partitionTableName}`""" + qt_paimon_alter_partition_final_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${partitionTableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + } finally { + sql """DROP TABLE IF EXISTS `${partitionTableName}`""" + sql """DROP TABLE IF EXISTS `${tableName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy index cbb3174ea5faeb..c9a27b344d6907 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + suite("test_paimon_jdbc_catalog", "p0,external") { String enabled = context.config.otherConfigs.get("enablePaimonTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -65,6 +68,11 @@ suite("test_paimon_jdbc_catalog", "p0,external") { String sparkSeedCatalogName = "${catalogName}_seed" // Reuse the fixture-wide Docker command so local and CI permission models behave identically. String dockerCommand = context.config.otherConfigs.get("externalDockerCommand") ?: "docker" + String sparkMaster = context.config.otherConfigs.get("paimon_jdbc_spark_master") + String sparkExternalEnvIp = context.config.otherConfigs.get("paimon_jdbc_spark_external_env_ip") + if (sparkExternalEnvIp == null || sparkExternalEnvIp.isEmpty()) { + sparkExternalEnvIp = externalEnvIp + } assertTrue(jdbcDriversDir != null && !jdbcDriversDir.isEmpty(), "jdbc_drivers_dir must be configured") @@ -91,6 +99,27 @@ suite("test_paimon_jdbc_catalog", "p0,external") { } } + def runConcurrent = { String leftName, Closure leftAction, + String rightName, Closure rightAction -> + CountDownLatch ready = new CountDownLatch(2) + CountDownLatch start = new CountDownLatch(1) + def left = thread(leftName) { + ready.countDown() + start.await() + leftAction() + } + def right = thread(rightName) { + ready.countDown() + start.await() + rightAction() + } + assertTrue(ready.await(30, TimeUnit.SECONDS), + "Both Paimon writers must reach the dispatch barrier") + start.countDown() + left.get() + right.get() + } + executeCommand("mkdir -p ${localDriverDir}", false, 60) if (!new File(localDriverPath).exists()) { executeCommand("/usr/bin/curl --max-time 600 ${driverDownloadUrl} --output ${localDriverPath}", true, 660) @@ -144,7 +173,10 @@ suite("test_paimon_jdbc_catalog", "p0,external") { } executeCommand("${dockerCommand} cp ${localDriverPath} ${sparkContainerName}:${sparkDriverPath}", true, 60) - String sparkMinioEndpoint = "http://${externalEnvIp}:${minioPort}" + String sparkMinioEndpoint = context.config.otherConfigs.get("paimon_jdbc_spark_minio_endpoint") + if (sparkMinioEndpoint == null || sparkMinioEndpoint.isEmpty()) { + sparkMinioEndpoint = "http://${sparkExternalEnvIp}:${minioPort}" + } if (sparkContainerName.contains("spark-iceberg")) { String sparkMinioContainerName = sparkContainerName.replaceFirst("spark-iceberg", "minio") String resolvedSparkMinioContainer = executeCommand( @@ -158,10 +190,14 @@ suite("test_paimon_jdbc_catalog", "p0,external") { } } logger.info("spark seed minio endpoint: ${sparkMinioEndpoint}") + if (sparkMaster == null || sparkMaster.isEmpty()) { + sparkMaster = "spark://${sparkContainerName}:7077" + } + logger.info("spark seed master: ${sparkMaster}") def sparkPaimonJdbc = { String sqlText -> String escapedSql = sqlText.replaceAll('"', '\\\\"') - String command = """${dockerCommand} exec ${sparkContainerName} spark-sql --master spark://${sparkContainerName}:7077 \ + String command = """${dockerCommand} exec ${sparkContainerName} spark-sql --master ${sparkMaster} \ --jars ${sparkDriverPath} \ --driver-class-path ${sparkDriverPath} \ --conf spark.driver.extraClassPath=${sparkDriverPath} \ @@ -170,11 +206,11 @@ suite("test_paimon_jdbc_catalog", "p0,external") { --conf spark.sql.catalog.${sparkSeedCatalogName}=org.apache.paimon.spark.SparkCatalog \ --conf spark.sql.catalog.${sparkSeedCatalogName}.warehouse=s3://${warehouseBucket}/paimon_jdbc_catalog/ \ --conf spark.sql.catalog.${sparkSeedCatalogName}.metastore=jdbc \ ---conf spark.sql.catalog.${sparkSeedCatalogName}.uri=jdbc:postgresql://${externalEnvIp}:${jdbcPort}/postgres \ +--conf spark.sql.catalog.${sparkSeedCatalogName}.uri=jdbc:postgresql://${sparkExternalEnvIp}:${jdbcPort}/postgres \ --conf spark.sql.catalog.${sparkSeedCatalogName}.catalog-key=${catalogName} \ --conf spark.sql.catalog.${sparkSeedCatalogName}.jdbc.user=postgres \ --conf spark.sql.catalog.${sparkSeedCatalogName}.jdbc.password=123456 \ ---conf spark.sql.catalog.${sparkSeedCatalogName}.lock.enabled=false \ +--conf spark.sql.catalog.${sparkSeedCatalogName}.lock.enabled=true \ --conf spark.sql.catalog.${sparkSeedCatalogName}.s3.endpoint=${sparkMinioEndpoint} \ --conf spark.sql.catalog.${sparkSeedCatalogName}.s3.access-key=${minioAk} \ --conf spark.sql.catalog.${sparkSeedCatalogName}.s3.secret-key=${minioSk} \ @@ -203,6 +239,7 @@ suite("test_paimon_jdbc_catalog", "p0,external") { try { sql """switch internal""" sql """DROP CATALOG IF EXISTS ${catalogName}""" + // Paimon requires a catalog lock for safe concurrent snapshot commits on object storage. sql """ CREATE CATALOG ${catalogName} PROPERTIES ( 'type' = 'paimon', @@ -214,6 +251,7 @@ suite("test_paimon_jdbc_catalog", "p0,external") { 'paimon.jdbc.driver_class' = 'org.postgresql.Driver', 'paimon.jdbc.user' = 'postgres', 'paimon.jdbc.password' = '123456', + 'paimon.lock.enabled' = 'true', 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', 's3.access_key' = '${minioAk}', 's3.secret_key' = '${minioSk}', @@ -227,7 +265,14 @@ suite("test_paimon_jdbc_catalog", "p0,external") { assertTrue(catalogs.toString().contains(catalogName)) sql """DROP DATABASE IF EXISTS ${dbName} FORCE""" - sql """CREATE DATABASE ${dbName}""" + test { + sql """CREATE DATABASE ${dbName} PROPERTIES ( + 'location' = 's3://${warehouseBucket}/rejected_database_location/' + )""" + exception "database property 'location' for paimon catalog type: jdbc" + } + // Paimon JDBC persists database properties in paimon_database_properties. + sql """CREATE DATABASE ${dbName} PROPERTIES ('owner' = 'doris')""" def databases = sql """SHOW DATABASES""" assertTrue(databases.toString().contains(dbName)) @@ -347,9 +392,281 @@ suite("test_paimon_jdbc_catalog", "p0,external") { ["_ROW_ID", "_SEQUENCE_NUMBER"], 1 ) + + // Append writers cover both independent partitions and snapshot-isolated writes + // to the same partition. Every successful transaction must publish one snapshot. + sql """DROP TABLE IF EXISTS paimon_jdbc_concurrent_append""" + sql """ + CREATE TABLE ${dbName}.paimon_jdbc_concurrent_append ( + id BIGINT, + writer_id INT, + payload STRING, + pt STRING + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'bucket' = '-1', + 'write-only' = 'true' + ) + """ + + long appendSnapshots = (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_append\$snapshots + """)[0][0] as long + runConcurrent("paimon-jdbc-append-left", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_append + SELECT number, 1, concat('left-', number), 'left' + FROM numbers('number' = '128') + """ + }, "paimon-jdbc-append-right", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_append + SELECT number + 1000, 2, concat('right-', number), 'right' + FROM numbers('number' = '128') + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_append""" + order_qt_paimon_jdbc_concurrent_append """ + SELECT pt, COUNT(*), COUNT(DISTINCT id), SUM(id) + FROM paimon_jdbc_concurrent_append + GROUP BY pt + ORDER BY pt + """ + assertEquals(appendSnapshots + 2L, (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_append\$snapshots + """)[0][0] as long) + + runConcurrent("paimon-jdbc-same-partition-left", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_append + SELECT number + 2000, 3, concat('same-left-', number), 'same' + FROM numbers('number' = '64') + """ + }, "paimon-jdbc-same-partition-right", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_append + SELECT number + 3000, 4, concat('same-right-', number), 'same' + FROM numbers('number' = '64') + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_append""" + qt_paimon_jdbc_same_partition_append """ + SELECT COUNT(*), COUNT(DISTINCT id), SUM(id) + FROM paimon_jdbc_concurrent_append + WHERE pt = 'same' + """ + assertEquals(appendSnapshots + 4L, (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_append\$snapshots + """)[0][0] as long) + + // Fixed-bucket deduplication may expose either value, but it must preserve + // primary-key uniqueness and publish both successful transactions. + sql """DROP TABLE IF EXISTS paimon_jdbc_concurrent_pk""" + sql """ + CREATE TABLE ${dbName}.paimon_jdbc_concurrent_pk ( + id INT, + payload STRING + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'deduplicate' + ) + """ + + long pkSnapshots = (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_pk\$snapshots + """)[0][0] as long + runConcurrent("paimon-jdbc-pk-left", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_pk + VALUES (1, 'left') + """ + }, "paimon-jdbc-pk-right", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_pk + VALUES (1, 'right') + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_pk""" + def pkRows = sql """SELECT id, payload FROM paimon_jdbc_concurrent_pk""" + assertEquals(1, pkRows.size()) + assertEquals(1, pkRows[0][0] as int) + assertTrue(["left", "right"].contains(pkRows[0][1].toString())) + assertEquals(pkSnapshots + 2L, (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_pk\$snapshots + """)[0][0] as long) + + // Aggregation is a lost-update oracle because both deltas must remain visible. + sql """DROP TABLE IF EXISTS paimon_jdbc_concurrent_aggregation""" + sql """ + CREATE TABLE ${dbName}.paimon_jdbc_concurrent_aggregation ( + id INT, + total BIGINT + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum' + ) + """ + + runConcurrent("paimon-jdbc-aggregation-left", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_aggregation + VALUES (1, 10) + """ + }, "paimon-jdbc-aggregation-right", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_aggregation + VALUES (1, 20) + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_aggregation""" + order_qt_paimon_jdbc_concurrent_aggregation """ + SELECT id, total FROM paimon_jdbc_concurrent_aggregation + ORDER BY id + """ + + // Dynamic bucket only permits multiple jobs when they own disjoint partitions. + sql """DROP TABLE IF EXISTS paimon_jdbc_concurrent_dynamic""" + sql """ + CREATE TABLE ${dbName}.paimon_jdbc_concurrent_dynamic ( + id INT, + pt STRING, + payload STRING + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'primary-key' = 'id,pt', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '32' + ) + """ + + runConcurrent("paimon-jdbc-dynamic-left", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_dynamic + SELECT number, 'left', concat('left-', number) + FROM numbers('number' = '64') + """ + }, "paimon-jdbc-dynamic-right", { + sql """ + INSERT INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_dynamic + SELECT number + 1000, 'right', concat('right-', number) + FROM numbers('number' = '64') + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_dynamic""" + order_qt_paimon_jdbc_concurrent_dynamic """ + SELECT pt, COUNT(*), COUNT(DISTINCT id) + FROM paimon_jdbc_concurrent_dynamic + GROUP BY pt + ORDER BY pt + """ + + // P12: Row-level writers use the same catalog lock and snapshot commit + // protocol as INSERT. Concurrent MERGEs on one key may expose either + // last value, but both transactions must commit without duplicating it. + sql """DROP TABLE IF EXISTS paimon_jdbc_concurrent_merge""" + sql """ + CREATE TABLE ${dbName}.paimon_jdbc_concurrent_merge ( + id INT, + payload STRING, + score INT + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'deduplicate', + 'num-sorted-run.compaction-trigger' = '100' + ) + """ + sql """INSERT INTO paimon_jdbc_concurrent_merge VALUES + (1, 'base-1', 0), (2, 'base-2', 0) + """ + long mergeSnapshots = (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_merge\$snapshots + """)[0][0] as long + + runConcurrent("paimon-jdbc-merge-left", { + sql """ + MERGE INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_merge t + USING (SELECT 1 AS id, 'left' AS payload, 10 AS score) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET + payload = s.payload, score = s.score + """ + }, "paimon-jdbc-merge-right", { + sql """ + MERGE INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_merge t + USING (SELECT 1 AS id, 'right' AS payload, 20 AS score) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET + payload = s.payload, score = s.score + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_merge""" + def sameKeyMergeRows = sql """ + SELECT id, payload, score + FROM paimon_jdbc_concurrent_merge + WHERE id = 1 + """ + assertEquals(1, sameKeyMergeRows.size()) + assertTrue([ + [1, "left", 10], + [1, "right", 20] + ].contains(sameKeyMergeRows[0])) + assertEquals(mergeSnapshots + 2L, (sql """ + SELECT COUNT(*) FROM paimon_jdbc_concurrent_merge\$snapshots + """)[0][0] as long) + + // Interleave a Doris MERGE with a Spark full compaction. Whichever + // operation obtains the catalog lock first, the MERGE result must remain + // visible and compaction must not resurrect the pre-update value. + runConcurrent("paimon-jdbc-merge-compact", { + sql """ + MERGE INTO ${catalogName}.${dbName}.paimon_jdbc_concurrent_merge t + USING (SELECT 2 AS id, 'merge-during-compact' AS payload, + 200 AS score) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET + payload = s.payload, score = s.score + """ + }, "paimon-jdbc-spark-compact", { + sparkPaimonJdbc """ + CALL ${sparkSeedCatalogName}.sys.compact( + table => '${dbName}.paimon_jdbc_concurrent_merge', + compact_strategy => 'full') + """ + }) + sql """REFRESH TABLE paimon_jdbc_concurrent_merge""" + order_qt_paimon_jdbc_merge_during_compact """ + SELECT id, payload, score + FROM paimon_jdbc_concurrent_merge + WHERE id = 2 + ORDER BY id + """ + + sparkPaimonJdbc """ + SELECT assert_true( + COUNT(*) = 2 AND + SUM(CASE WHEN id = 2 + AND payload = 'merge-during-compact' + AND score = 200 + THEN 1 ELSE 0 END) = 1) + FROM ${sparkSeedCatalogName}.${dbName}.paimon_jdbc_concurrent_merge + """ } finally { try { sql """SWITCH ${catalogName}""" + sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_concurrent_merge""" + sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_concurrent_dynamic""" + sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_concurrent_aggregation""" + sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_concurrent_pk""" + sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_concurrent_append""" sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_row_tracking_tbl""" sql """DROP TABLE IF EXISTS ${dbName}.paimon_jdbc_tbl""" sql """DROP DATABASE IF EXISTS ${dbName} FORCE""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy index 84fd84ce342699..12edd7158da06c 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy @@ -64,47 +64,28 @@ suite("test_paimon_write_boundary", qt_before_rows """select id, score, note from write_boundary order by id""" qt_before_snapshots """select count(*) from write_boundary\$snapshots""" - // WB01-WB06 preserve the documented data-write boundary at analysis time. The source table - // and its snapshot list must stay unchanged after every rejected write shape. - // - // The INSERT-family rejections are worded by the connector-SPI path, not by the legacy fe-core - // one: a paimon catalog is a PluginDrivenExternalCatalog, so UnboundTableSinkCreator builds an - // UnboundConnectorTableSink instead of throwing "Load data to PaimonExternalCatalog is not - // supported", and the rejection lands on the connector's declared write capabilities (the paimon - // connector declares none). The boundary asserted here is identical -- every write shape is still - // rejected at analysis time and the table is untouched -- only the message differs. - test { - sql """insert into write_boundary values (3, 30, 'insert-values')""" - exception "does not support INSERT operations" - } - test { - sql """insert into write_boundary select 3, 30, 'insert-select'""" - exception "does not support INSERT operations" - } - test { - // INSERT OVERWRITE is gated earlier, by InsertOverwriteTableCommand's allowInsertOverwrite. - sql """insert overwrite table write_boundary values (3, 30, 'overwrite')""" - exception "insert into overwrite only support" - } - test { - sql """update write_boundary set score = score + 1 where id = 1""" - exception "target table in update command should be an olapTable" - } - test { - sql """delete from write_boundary where id = 1""" - exception "delete command could be only used on olap table" - } - test { - sql """ - merge into write_boundary target - using (select 1 as id, 99 as score, 'merge' as note) source - on target.id = source.id - when matched then update set score = source.score, note = source.note - when not matched then insert (id, score, note) - values (source.id, source.score, source.note) - """ - exception "merge into command only support MOW unique key olapTable" - } + // Exercise both append/overwrite writes and row-level changelog writes through the + // external-table boundary suite. + sql """insert into write_boundary values (3, 30, 'insert-values')""" + sql """insert into write_boundary select 4, 40, 'insert-select'""" + sql """refresh table write_boundary""" + qt_after_append_rows """select id, score, note from write_boundary order by id""" + + sql """insert overwrite table write_boundary values (5, 50, 'overwrite')""" + sql """update write_boundary set score = score + 1 where id = 5""" + sql """ + merge into write_boundary target + using ( + select 5 as id, 99 as score, 'merge-update' as note + union all + select 6 as id, 60 as score, 'merge-insert' as note + ) source + on target.id = source.id + when matched then update set score = source.score, note = source.note + when not matched then insert (id, score, note) + values (source.id, source.score, source.note) + """ + sql """delete from write_boundary where id = 5""" sql """refresh table write_boundary""" qt_after_rows """select id, score, note from write_boundary order by id""" diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.groovy new file mode 100644 index 00000000000000..dcfbec3891c98d --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_create_ddl_write_properties.groovy @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_create_ddl_write_properties", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_create_props_catalog" + String dbName = "test_pw_create_props_db" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + spark_paimon """ + REFRESH TABLE paimon.${dbName}.${tableName} + """ + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """ + SELECT * FROM `${tableName}` ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT MAX(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + assertEquals(1, rows.size()) + assertTrue(rows[0][0] != null) + return rows[0][0].toString() + } + + // Doris maps location to Paimon's path option. The filesystem catalog + // deliberately rejects custom table paths, and that SDK validation + // must be preserved instead of silently ignoring the property. + test { + sql """ + CREATE TABLE `t_create_custom_location` ( + id INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'location' = + 's3://warehouse/wh/${dbName}.db/t_create_custom_location_data' + ) + """ + exception "does not support specifying the table path" + } + qt_create_custom_location_absent """ + SHOW TABLES LIKE 't_create_custom_location' + """ + + // Doris CREATE must preserve the primary/partition keys, table comment + // and storage/write options. Sequence ordering is verified with a + // lower-sequence update followed by a higher one. + sql """ + CREATE TABLE `t_create_sequence` ( + id INT NOT NULL, + seq BIGINT NOT NULL, + payload STRING NULL, + dt STRING NOT NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'primary-key' = 'id,dt', + 'bucket' = '2', + 'bucket-key' = 'id', + 'sequence.field' = 'seq', + 'file.format' = 'orc', + 'snapshot.num-retained.min' = '2', + 'snapshot.num-retained.max' = '5', + 'comment' = 'created by Doris with write properties' + ) + """ + sql """ + INSERT INTO `t_create_sequence` VALUES + (1, 100, 'newer', 'p1'), + (2, 10, 'initial-2', 'p1'), + (3, 5, 'initial-3', 'p2') + """ + sql """ + INSERT INTO `t_create_sequence` VALUES + (1, 50, 'older-must-not-win', 'p1'), + (2, 20, 'updated-2', 'p1') + """ + order_qt_create_sequence_result """ + SELECT id, seq, payload, dt + FROM `t_create_sequence` + ORDER BY dt, id + """ + qt_create_sequence_schema """ + SELECT partition_keys, primary_keys, comment + FROM `t_create_sequence\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + order_qt_create_sequence_file_format """ + SELECT DISTINCT file_format + FROM `t_create_sequence\$files` + ORDER BY file_format + """ + assertTableEquals("t_create_sequence", "ORDER BY dt, id") + + // Fixed bucket properties must be consumed by the writer, while + // partial-update accepts arbitrary value-column subsets. + sql """ + CREATE TABLE `t_create_partial` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL, + note STRING NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'partial-update' + ) + """ + sql """ + INSERT INTO `t_create_partial` VALUES + (1, 'alice', 10, 'initial'), + (2, 'bob', 20, 'initial') + """ + sql """INSERT INTO `t_create_partial` (id, score) VALUES (1, 15)""" + sql """INSERT INTO `t_create_partial` (note, id) VALUES ('updated', 1)""" + order_qt_create_partial_result """ + SELECT id, name, score, note + FROM `t_create_partial` + ORDER BY id + """ + assertTableEquals("t_create_partial", "ORDER BY id") + + // First-row and aggregation semantics prove that CREATE forwarded the + // merge-engine and per-field aggregation properties. + sql """ + CREATE TABLE `t_create_first_row` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'first-row' + ) + """ + sql """ + INSERT INTO `t_create_first_row` VALUES + (1, 'first-1', 10), + (2, 'first-2', 20) + """ + sql """ + INSERT INTO `t_create_first_row` VALUES + (1, 'second-1', 11), + (3, 'first-3', 30) + """ + order_qt_create_first_row_result """ + SELECT id, name, score + FROM `t_create_first_row` + ORDER BY id + """ + assertTableEquals("t_create_first_row", "ORDER BY id") + + sql """ + CREATE TABLE `t_create_aggregation` ( + id INT NOT NULL, + total BIGINT NULL, + highest INT NULL, + label STRING NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'fields.highest.aggregate-function' = 'max' + ) + """ + sql """ + INSERT INTO `t_create_aggregation` VALUES + (1, 10, 80, 'first'), + (2, 5, 70, 'stable') + """ + sql """ + INSERT INTO `t_create_aggregation` VALUES + (1, 7, 90, 'latest'), + (2, 3, 60, NULL), + (3, 4, 50, 'new') + """ + order_qt_create_aggregation_result """ + SELECT id, total, highest, label + FROM `t_create_aggregation` + ORDER BY id + """ + assertTableEquals("t_create_aggregation", "ORDER BY id") + + // Lookup changelog generation is checked independently from the final + // table contents, so merely storing the option is not sufficient. + sql """ + CREATE TABLE `t_create_lookup` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'changelog-producer' = 'lookup' + ) + """ + sql """ + INSERT INTO `t_create_lookup` VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + String lookupBefore = latestSnapshotId("t_create_lookup") + sql """ + INSERT INTO `t_create_lookup` VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + String lookupAfter = latestSnapshotId("t_create_lookup") + def lookupChanges = spark_paimon """ + SELECT rowkind, id, name, score + FROM paimon_incremental_query( + 'paimon.${dbName}.`t_create_lookup\$audit_log`', + '${lookupBefore}', + '${lookupAfter}' + ) + ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END + """ + assertEquals([ + ["-U", 1, "old", 10], + ["+U", 1, "new", 11], + ["+I", 3, "added", 30] + ], lookupChanges) + order_qt_create_lookup_changelog """ + SELECT rowkind, id, name, score + FROM `t_create_lookup\$audit_log` + ORDER BY id, + CASE rowkind + WHEN '+I' THEN 0 + WHEN '-U' THEN 1 + WHEN '+U' THEN 2 + ELSE 3 + END + """ + order_qt_create_lookup_result """ + SELECT id, name, score + FROM `t_create_lookup` + ORDER BY id + """ + assertTableEquals("t_create_lookup", "ORDER BY id") + + // Dynamic bucket options must affect physical routing after a Doris + // INSERT, not only appear in metadata. + sql """ + CREATE TABLE `t_create_dynamic_bucket` ( + pt STRING NOT NULL, + id INT NOT NULL, + value STRING NULL + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.initial-buckets' = '1', + 'dynamic-bucket.max-buckets' = '4' + ) + """ + sql """ + INSERT INTO `t_create_dynamic_bucket` + SELECT 'p1', CAST(number AS INT), concat('v', CAST(number AS STRING)) + FROM numbers("number" = "12") + """ + qt_create_dynamic_bucket_result """ + SELECT COUNT(*), MIN(id), MAX(id) + FROM `t_create_dynamic_bucket` + """ + order_qt_create_dynamic_bucket_rows """ + SELECT pt, id, value + FROM `t_create_dynamic_bucket` + ORDER BY pt, id + """ + order_qt_create_dynamic_bucket_files """ + SELECT DISTINCT bucket + FROM `t_create_dynamic_bucket\$files` + ORDER BY bucket + """ + def dynamicBuckets = spark_paimon """ + SELECT DISTINCT bucket + FROM paimon.${dbName}.`t_create_dynamic_bucket\$files` + ORDER BY bucket + """ + assertFalse(dynamicBuckets.isEmpty()) + assertTrue(dynamicBuckets.every { row -> + int bucket = row[0].toString().toInteger() + return bucket >= 0 && bucket < 4 + }) + assertTrue(dynamicBuckets.size() > 1) + assertTableEquals("t_create_dynamic_bucket", "ORDER BY pt, id") + + // Keep one deterministic view of all important CREATE options. This + // catches property loss or accidental key rewriting in the Doris DDL. + order_qt_create_write_options """ + SELECT 'aggregation' AS table_name, `key`, value + FROM `t_create_aggregation\$options` + WHERE `key` IN ( + 'bucket', 'fields.highest.aggregate-function', + 'fields.total.aggregate-function', 'merge-engine' + ) + UNION ALL + SELECT 'dynamic_bucket', `key`, value + FROM `t_create_dynamic_bucket\$options` + WHERE `key` IN ( + 'bucket', 'dynamic-bucket.initial-buckets', + 'dynamic-bucket.max-buckets', 'dynamic-bucket.target-row-num' + ) + UNION ALL + SELECT 'first_row', `key`, value + FROM `t_create_first_row\$options` + WHERE `key` IN ('bucket', 'merge-engine') + UNION ALL + SELECT 'lookup', `key`, value + FROM `t_create_lookup\$options` + WHERE `key` IN ('bucket', 'changelog-producer') + UNION ALL + SELECT 'partial_update', `key`, value + FROM `t_create_partial\$options` + WHERE `key` IN ('bucket', 'bucket-key', 'merge-engine') + UNION ALL + SELECT 'sequence', `key`, value + FROM `t_create_sequence\$options` + WHERE `key` IN ( + 'bucket', 'bucket-key', 'file.format', + 'sequence.field', 'snapshot.num-retained.max', + 'snapshot.num-retained.min' + ) + ORDER BY table_name, `key` + """ + } finally { + [ + "t_create_dynamic_bucket", + "t_create_lookup", + "t_create_aggregation", + "t_create_first_row", + "t_create_partial", + "t_create_sequence" + ].each { tableName -> + try { + sql """DROP TABLE IF EXISTS `${tableName}`""" + } catch (Exception e) { + logger.info("Failed to drop ${tableName}: ${e.getMessage()}") + } + } + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy new file mode 100644 index 00000000000000..e7a8ccb18f8a47 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_append_only", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_ao_catalog" + String dbName = "test_pw_ao_db" + + // Tables are created via Spark because Doris does not yet support + // Paimon DDL (CREATE TABLE ... engine=paimon). + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_append; + CREATE TABLE paimon.${dbName}.t_append ( + id INT, name STRING, score DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_part; + CREATE TABLE paimon.${dbName}.t_append_part ( + id INT, name STRING, score DOUBLE, region STRING + ) USING paimon + PARTITIONED BY (region) + ; + + DROP TABLE IF EXISTS paimon.${dbName}.t_auto_partition; + CREATE TABLE paimon.${dbName}.t_auto_partition ( + id INT, name STRING, dt STRING + ) USING paimon + PARTITIONED BY (dt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_empty; + CREATE TABLE paimon.${dbName}.t_append_empty ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_default; + CREATE TABLE paimon.${dbName}.t_append_default ( + id INT, name STRING NOT NULL DEFAULT 'unknown' + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_required; + CREATE TABLE paimon.${dbName}.t_append_required ( + id INT, name STRING NOT NULL + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_partition_default; + CREATE TABLE paimon.${dbName}.t_partition_default ( + id INT, name STRING, dt STRING NOT NULL DEFAULT '2026-07-01' + ) USING paimon + PARTITIONED BY (dt); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-001: Append-only table β€” basic INSERT + sql """INSERT INTO t_append VALUES (1, 'alice', 95.5)""" + sql """INSERT INTO t_append VALUES (2, 'bob', 87.0), (3, 'charlie', 92.3)""" + order_qt_ao_basic """SELECT * FROM t_append ORDER BY id""" + + sql """INSERT INTO t_append VALUES (4, 'diana', 88.0), (5, 'eve', 91.0)""" + // Full-column and partial-column writes with columns in non-schema order + sql """INSERT INTO t_append (score, name, id) VALUES (93.0, 'frank', 6)""" + sql """INSERT INTO t_append (name, id) VALUES ('grace', 7)""" + assertTableEquals("t_append", "ORDER BY id") + + // FT-002: Partitioned append-only + sql """INSERT INTO t_append_part VALUES (1, 'alice', 95.5, 'east'), (2, 'bob', 87.0, 'west')""" + sql """INSERT INTO t_append_part VALUES (3, 'charlie', 92.3, 'east'), (4, 'diana', 88.0, 'north')""" + // Keep the partition column away from its schema position in both full and partial writes + sql """INSERT INTO t_append_part (region, score, name, id) + VALUES ('south', 86.5, 'erin', 5)""" + sql """INSERT INTO t_append_part (region, id) VALUES ('east', 6)""" + order_qt_ao_part """SELECT * FROM t_append_part ORDER BY id""" + assertTableEquals("t_append_part", "ORDER BY id") + + // Paimon partitions are implicit: writing a previously unseen partition-key + // value creates the physical partition without an ADD PARTITION operation. + sql """INSERT INTO t_auto_partition VALUES + (1, 'alpha', '2026-07-01'), + (2, 'beta', '2026-07-02'), + (3, 'gamma', '2026-07-01') + """ + sql """INSERT INTO t_auto_partition VALUES + (4, 'delta', '2026-07-01'), + (5, 'epsilon', '2026-07-03'), + (6, 'default_partition', NULL) + """ + order_qt_ao_auto_partition_data """ + SELECT id, name, dt FROM t_auto_partition ORDER BY id + """ + assertTableEquals("t_auto_partition", "ORDER BY id") + + def sparkPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_auto_partition\$partitions` + ORDER BY `partition` + """ + def dorisPartitions = sql """ + SELECT `partition`, record_count + FROM t_auto_partition\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + order_qt_ao_auto_partition_metadata """ + SELECT `partition`, record_count + FROM t_auto_partition\$partitions + ORDER BY `partition` + """ + + // FT-014: Empty INSERT β€” should succeed with 0 rows + sql """INSERT INTO t_append_empty SELECT 1, 'test' WHERE 1 = 0""" + sql """INSERT INTO t_append_empty (id) VALUES (1)""" + sql """INSERT INTO t_append_empty (name, id) VALUES ('reordered', 2)""" + order_qt_ao_empty """SELECT id, name FROM t_append_empty ORDER BY id""" + assertTableEquals("t_append_empty", "ORDER BY id") + + // FT-043: Only omitted fields are filled from the real Paimon schema. + sql """INSERT INTO t_append_default (id) VALUES (1)""" + order_qt_ao_default_value """SELECT id, name FROM t_append_default ORDER BY id""" + assertTableEquals("t_append_default", "ORDER BY id") + + // Explicit NULL remains an input value. Paimon checks the real NOT NULL + // schema before applying its writer-side default wrapper. + test { + sql """INSERT INTO t_append_default (name, id) VALUES (NULL, 2)""" + exception "Cannot write null to non-null column(name)" + } + order_qt_ao_default_after_explicit_null """ + SELECT id, name FROM t_append_default ORDER BY id + """ + + // Doris does not duplicate Paimon's nullability validation. An omitted + // NOT NULL field without a default remains NULL and is rejected by the + // writer against the real Paimon schema. + test { + sql """INSERT INTO t_append_required (id) VALUES (1)""" + exception "Cannot write null to non-null column(name)" + } + + // A defaulted partition field uses the schema default as its logical and + // physical partition value instead of the configured null-partition name. + sql """INSERT INTO t_partition_default (name, id) VALUES ('omitted-partition', 1)""" + order_qt_ao_partition_default_data """ + SELECT id, name, dt FROM t_partition_default ORDER BY id + """ + assertTableEquals("t_partition_default", "ORDER BY id") + def sparkDefaultPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_partition_default\$partitions` + ORDER BY `partition` + """ + def dorisDefaultPartitions = sql """ + SELECT `partition`, record_count + FROM t_partition_default\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkDefaultPartitions, dorisDefaultPartitions) + order_qt_ao_partition_default_metadata """ + SELECT `partition`, record_count + FROM t_partition_default\$partitions + ORDER BY `partition` + """ + test { + sql """INSERT INTO t_partition_default (id, name, dt) + VALUES (2, 'explicit-null-partition', NULL)""" + exception "Cannot write null to non-null column(dt)" + } + + // FT-044: Duplicate target columns are rejected case-insensitively. + test { + sql """INSERT INTO t_append (id, ID) VALUES (8, 9)""" + exception "Duplicate column" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_bucket_modes.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_bucket_modes.groovy new file mode 100644 index 00000000000000..f8ad1af65865fb --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_bucket_modes.groovy @@ -0,0 +1,567 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_bucket_modes", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_bucket_catalog" + String dbName = "test_pw_bucket_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_fixed; + CREATE TABLE paimon.${dbName}.t_hash_fixed ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '4', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_rescale; + CREATE TABLE paimon.${dbName}.t_rescale ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic; + CREATE TABLE paimon.${dbName}.t_hash_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.initial-buckets' = '1', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic_partial; + CREATE TABLE paimon.${dbName}.t_hash_dynamic_partial ( + pt STRING, id INT, name STRING, score INT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic_overwrite; + CREATE TABLE paimon.${dbName}.t_hash_dynamic_overwrite ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic; + CREATE TABLE paimon.${dbName}.t_key_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_partial; + CREATE TABLE paimon.${dbName}.t_key_dynamic_partial ( + pt STRING, id INT, name STRING, score INT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_first_row; + CREATE TABLE paimon.${dbName}.t_key_dynamic_first_row ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'first-row' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_aggregation; + CREATE TABLE paimon.${dbName}.t_key_dynamic_aggregation ( + pt STRING, id INT, total BIGINT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_scale; + CREATE TABLE paimon.${dbName}.t_key_dynamic_scale ( + pt STRING, id BIGINT, payload STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '128', + 'dynamic-bucket.max-buckets' = '16' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_bucket_unaware; + CREATE TABLE paimon.${dbName}.t_bucket_unaware ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'bucket' = '-1' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_postpone; + CREATE TABLE paimon.${dbName}.t_postpone ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-2', + 'postpone.default-bucket-num' = '2' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def bucketIds = { String tableName -> + def rows = spark_paimon """ + SELECT DISTINCT bucket + FROM paimon.${dbName}.`${tableName}\$files` + ORDER BY bucket + """ + return rows.collect { row -> row[0].toString().toInteger() } + } + + def assertBucketsInRange = { String tableName, int minBucket, int maxBucket -> + def buckets = bucketIds(tableName) + assertFalse(buckets.isEmpty()) + assertTrue(buckets.every { bucket -> + bucket >= minBucket && bucket <= maxBucket + }) + return buckets + } + + // Dynamic bucket modes must gather into one fragment instance and one JNI writer. + def hashDynamicPlan = sql """ + EXPLAIN SHAPE PLAN + INSERT INTO t_hash_dynamic + SELECT 'plan_only', CAST(number AS INT), 'unused' + FROM numbers("number" = "8") + """ + assertTrue(hashDynamicPlan.flatten().join("\n").contains("DistributionSpecGather")) + + def keyDynamicPlan = sql """ + EXPLAIN SHAPE PLAN + INSERT INTO t_key_dynamic + SELECT 'plan_only', CAST(number AS INT), 'unused' + FROM numbers("number" = "8") + """ + assertTrue(keyDynamicPlan.flatten().join("\n").contains("DistributionSpecGather")) + + // HASH_FIXED: SDK computes the fixed bucket from bucket-key=id. + sql """ + INSERT INTO t_hash_fixed + SELECT concat('p', CAST(number % 2 AS STRING)), + CAST(number AS INT), + concat('fixed_', CAST(number AS STRING)) + FROM numbers("number" = "16") + """ + qt_bucket_hash_fixed """ + SELECT COUNT(*), MIN(id), MAX(id), COUNT(DISTINCT pt) + FROM t_hash_fixed + """ + assertTableEquals("t_hash_fixed", "ORDER BY pt, id") + def fixedBuckets = assertBucketsInRange("t_hash_fixed", 0, 3) + assertTrue(fixedBuckets.size() > 1) + + // P04: ALTER only changes the configured bucket count. Existing + // partitions must be rewritten before a new writer can use bucket=4. + sql """INSERT INTO t_rescale VALUES + ('p1', 1, 'p1-old-1'), + ('p1', 2, 'p1-old-2'), + ('p2', 3, 'p2-old-3'), + ('p2', 4, 'p2-old-4') + """ + spark_paimon """ + ALTER TABLE paimon.${dbName}.t_rescale + SET TBLPROPERTIES ('bucket' = '4') + """ + sql """refresh table t_rescale""" + long rescaleSnapshot = (sql """ + SELECT max(snapshot_id) FROM t_rescale\$snapshots + """)[0][0] as long + long rescaleFiles = (sql """ + SELECT count(*) FROM t_rescale\$files + """)[0][0] as long + boolean rejectedBeforeRescale = false + try { + sql """INSERT INTO t_rescale VALUES ('p1', 10, 'must-fail-before-rescale')""" + } catch (Exception ignored) { + rejectedBeforeRescale = true + } + assertTrue(rejectedBeforeRescale) + assertEquals(rescaleSnapshot, (sql """ + SELECT max(snapshot_id) FROM t_rescale\$snapshots + """)[0][0] as long) + assertEquals(rescaleFiles, (sql """ + SELECT count(*) FROM t_rescale\$files + """)[0][0] as long) + + // Rescale only p1. The rewritten partition accepts new writes, while p2 + // remains readable with its old layout and still rejects bucket=4. + sql """ + INSERT OVERWRITE TABLE t_rescale PARTITION (pt = 'p1') + SELECT id, name FROM t_rescale WHERE pt = 'p1' + """ + sql """INSERT INTO t_rescale VALUES ('p1', 10, 'p1-after-rescale')""" + boolean unreformedPartitionRejected = false + try { + sql """INSERT INTO t_rescale VALUES ('p2', 20, 'p2-must-still-fail')""" + } catch (Exception ignored) { + unreformedPartitionRejected = true + } + assertTrue(unreformedPartitionRejected) + order_qt_bucket_rescale_partial """ + SELECT * FROM t_rescale ORDER BY pt, id + """ + + sql """ + INSERT OVERWRITE TABLE t_rescale PARTITION (pt = 'p2') + SELECT id, name FROM t_rescale WHERE pt = 'p2' + """ + sql """INSERT INTO t_rescale VALUES ('p2', 20, 'p2-after-rescale')""" + assertTableEquals("t_rescale", "ORDER BY pt, id") + assertBucketsInRange("t_rescale", 0, 3) + + // HASH_DYNAMIC: new keys expand buckets independently per partition. + sql """INSERT INTO t_hash_dynamic VALUES + ('p1', 1, 'v1'), + ('p1', 2, 'v2'), + ('p1', 3, 'v3'), + ('p1', 4, 'v4'), + ('p1', 5, 'v5'), + ('p1', 6, 'v6'), + ('p2', 1, 'p2_v1'), + ('p2', 2, 'p2_v2') + """ + assertTableEquals("t_hash_dynamic", "ORDER BY pt, id") + def dynamicBucketsBeforeUpdate = + assertBucketsInRange("t_hash_dynamic", 0, 3) + assertTrue(dynamicBucketsBeforeUpdate.size() > 1) + + // A new Doris transaction must load the existing hash index. Updating only + // existing keys must not allocate another bucket. + sql """INSERT INTO t_hash_dynamic VALUES + ('p1', 1, 'v1_updated'), + ('p1', 4, 'v4_updated'), + ('p2', 2, 'p2_v2_updated') + """ + order_qt_bucket_hash_dynamic """ + SELECT pt, id, name FROM t_hash_dynamic ORDER BY pt, id + """ + assertTableEquals("t_hash_dynamic", "ORDER BY pt, id") + assertEquals(dynamicBucketsBeforeUpdate, bucketIds("t_hash_dynamic")) + + // Dynamic bucket and partial-update share the same normalized table row. + sql """INSERT INTO t_hash_dynamic_partial VALUES + ('p1', 1, 'alice', 10), + ('p1', 2, 'bob', 20) + """ + sql """INSERT INTO t_hash_dynamic_partial (pt, id, score) VALUES + ('p1', 1, 15), + ('p1', 3, 30) + """ + order_qt_bucket_hash_dynamic_partial """ + SELECT pt, id, name, score FROM t_hash_dynamic_partial ORDER BY pt, id + """ + assertTableEquals("t_hash_dynamic_partial", "ORDER BY pt, id") + assertBucketsInRange("t_hash_dynamic_partial", 0, Integer.MAX_VALUE) + + // HASH_DYNAMIC overwrite uses the SDK's overwrite assigner and replaces + // both data files and the dynamic hash index. + sql """INSERT INTO t_hash_dynamic_overwrite VALUES + (1, 'old_1'), (2, 'old_2'), (3, 'old_3'), (4, 'old_4') + """ + sql """INSERT OVERWRITE TABLE t_hash_dynamic_overwrite VALUES + (10, 'new_10'), (11, 'new_11'), (12, 'new_12') + """ + order_qt_bucket_hash_dynamic_overwrite """ + SELECT id, name FROM t_hash_dynamic_overwrite ORDER BY id + """ + assertTableEquals("t_hash_dynamic_overwrite", "ORDER BY id") + def overwriteRows = sql """ + SELECT id, name FROM t_hash_dynamic_overwrite ORDER BY id + """ + assertEquals([ + [10, "new_10"], + [11, "new_11"], + [12, "new_12"] + ], overwriteRows) + assertBucketsInRange("t_hash_dynamic_overwrite", 0, 3) + + // KEY_DYNAMIC: the second statement bootstraps the existing global index. + // Deduplicate moves an existing primary key to its new partition. + sql """INSERT INTO t_key_dynamic VALUES + ('p1', 1, 'id1_old'), + ('p2', 2, 'id2_stable'), + ('p1', 3, 'id3_old') + """ + sql """INSERT INTO t_key_dynamic VALUES + ('p2', 1, 'id1_moved'), + ('p3', 3, 'id3_moved'), + ('p2', 4, 'id4_added') + """ + order_qt_bucket_key_dynamic """ + SELECT pt, id, name FROM t_key_dynamic ORDER BY id + """ + assertTableEquals("t_key_dynamic", "ORDER BY id") + def keyDynamicRows = sql """ + SELECT pt, id, name FROM t_key_dynamic ORDER BY id + """ + assertEquals([ + ["p2", 1, "id1_moved"], + ["p2", 2, "id2_stable"], + ["p3", 3, "id3_moved"], + ["p2", 4, "id4_added"] + ], keyDynamicRows) + assertBucketsInRange("t_key_dynamic", 0, 3) + + // For cross-partition partial-update, the global index keeps the old + // partition and applies the new non-null fields there. + sql """INSERT INTO t_key_dynamic_partial VALUES + ('p1', 10, 'old_10', 10), + ('p2', 20, 'stable_20', 20) + """ + sql """INSERT INTO t_key_dynamic_partial (pt, id, score) VALUES + ('p9', 10, 15), + ('p3', 30, 30) + """ + order_qt_bucket_key_dynamic_partial """ + SELECT pt, id, name, score FROM t_key_dynamic_partial ORDER BY id + """ + assertTableEquals("t_key_dynamic_partial", "ORDER BY id") + def keyDynamicPartialRows = sql """ + SELECT pt, id, name, score FROM t_key_dynamic_partial ORDER BY id + """ + assertEquals([ + ["p1", 10, "old_10", 15], + ["p2", 20, "stable_20", 20], + ["p3", 30, null, 30] + ], keyDynamicPartialRows) + + // FIRST_ROW ignores a later value even if it arrives in another partition. + sql """INSERT INTO t_key_dynamic_first_row VALUES + ('p1', 1, 'first_1') + """ + sql """INSERT INTO t_key_dynamic_first_row VALUES + ('p2', 1, 'ignored_1'), + ('p2', 2, 'first_2') + """ + order_qt_bucket_key_dynamic_first_row """ + SELECT pt, id, name FROM t_key_dynamic_first_row ORDER BY id + """ + assertTableEquals("t_key_dynamic_first_row", "ORDER BY id") + def keyDynamicFirstRowRows = sql """ + SELECT pt, id, name FROM t_key_dynamic_first_row ORDER BY id + """ + assertEquals([ + ["p1", 1, "first_1"], + ["p2", 2, "first_2"] + ], keyDynamicFirstRowRows) + + // Aggregation also stays in the original partition and combines values. + sql """INSERT INTO t_key_dynamic_aggregation VALUES + ('p1', 1, 10) + """ + sql """INSERT INTO t_key_dynamic_aggregation VALUES + ('p9', 1, 7), + ('p2', 2, 20) + """ + order_qt_bucket_key_dynamic_aggregation """ + SELECT pt, id, total FROM t_key_dynamic_aggregation ORDER BY id + """ + assertTableEquals("t_key_dynamic_aggregation", "ORDER BY id") + def keyDynamicAggregationRows = sql """ + SELECT pt, id, total FROM t_key_dynamic_aggregation ORDER BY id + """ + assertEquals([ + ["p1", 1, 17L], + ["p2", 2, 20L] + ], keyDynamicAggregationRows) + + // Bootstrap a larger KEY_DYNAMIC global index across multiple partitions + // and transactions. REFRESH CATALOG forces the next statement to reopen + // table metadata and construct a new JNI writer before restoring the index. + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST(number % 16 AS STRING)), + number, + concat('txn1_', CAST(number AS STRING)) + FROM numbers("number" = "4096") + """ + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST((number + 3) % 16 AS STRING)), + number, + concat('txn2_', CAST(number AS STRING)) + FROM numbers("number" = "2048") + """ + sql """REFRESH CATALOG ${catalogName}""" + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST((number + 5) % 16 AS STRING)), + number + 2048, + concat('txn3_', CAST(number + 2048 AS STRING)) + FROM numbers("number" = "2048") + """ + def keyDynamicScaleSummary = sql """ + SELECT COUNT(*), COUNT(DISTINCT id), MIN(id), MAX(id), SUM(id), + COUNT(DISTINCT pt), + SUM(IF(payload LIKE 'txn2_%', 1, 0)), + SUM(IF(payload LIKE 'txn3_%', 1, 0)) + FROM t_key_dynamic_scale + """ + assertEquals([[4096L, 4096L, 0L, 4095L, 8386560L, 16L, 2048L, 2048L]], + keyDynamicScaleSummary) + assertEquals(3L, + (sql """SELECT COUNT(*) FROM t_key_dynamic_scale\$snapshots""")[0][0] as long) + assertBucketsInRange("t_key_dynamic_scale", 0, 15) + def sparkScaleSummary = spark_paimon """ + SELECT COUNT(*), COUNT(DISTINCT id), MIN(id), MAX(id), SUM(id), + COUNT(DISTINCT pt), + SUM(CASE WHEN payload LIKE 'txn2_%' THEN 1 ELSE 0 END), + SUM(CASE WHEN payload LIKE 'txn3_%' THEN 1 ELSE 0 END) + FROM paimon.${dbName}.t_key_dynamic_scale + """ + assertSparkDorisResultEquals(sparkScaleSummary, keyDynamicScaleSummary) + order_qt_bucket_key_dynamic_scale_samples """ + SELECT pt, id, payload + FROM t_key_dynamic_scale + WHERE id IN (0, 1023, 2047, 2048, 3071, 4095) + ORDER BY id + """ + + // BUCKET_UNAWARE: append-only writers remain parallel while all files use bucket 0. + sql """SET parallel_pipeline_task_num = 4""" + sql """ + INSERT INTO t_bucket_unaware + SELECT concat('p', CAST(number % 2 AS STRING)), + CAST(number AS INT), + concat('unaware_', CAST(number AS STRING)) + FROM numbers("number" = "32") + """ + sql """SET parallel_pipeline_task_num = 0""" + qt_bucket_unaware """ + SELECT COUNT(*), MIN(id), MAX(id), COUNT(DISTINCT pt) + FROM t_bucket_unaware + """ + assertTableEquals("t_bucket_unaware", "ORDER BY pt, id") + assertEquals([0], bucketIds("t_bucket_unaware")) + + // POSTPONE_MODE commits files to bucket -2. Paimon deliberately + // excludes those files from readers and the files system table until + // an external compaction job assigns final buckets. + sql """INSERT INTO t_postpone VALUES + ('p1', 1, 'old_1'), + ('p1', 2, 'stable_2'), + ('p2', 3, 'stable_3') + """ + assertEquals([], bucketIds("t_postpone")) + assertTableEquals("t_postpone", "ORDER BY pt, id") + def postponeSnapshots = spark_paimon """ + SELECT COUNT(*) + FROM paimon.${dbName}.`t_postpone\$snapshots` + """ + assertEquals(1, postponeSnapshots[0][0].toString().toInteger()) + + sql """INSERT INTO t_postpone VALUES + ('p1', 1, 'new_1'), + ('p2', 4, 'added_4') + """ + assertEquals([], bucketIds("t_postpone")) + assertTableEquals("t_postpone", "ORDER BY pt, id") + postponeSnapshots = spark_paimon """ + SELECT COUNT(*) + FROM paimon.${dbName}.`t_postpone\$snapshots` + """ + assertEquals(2, postponeSnapshots[0][0].toString().toInteger()) + qt_bucket_postpone """SELECT COUNT(*) FROM t_postpone""" + } finally { + sql """SET parallel_pipeline_task_num = 0""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_changelog_producer.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_changelog_producer.groovy new file mode 100644 index 00000000000000..6744820e9dbe2c --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_changelog_producer.groovy @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_changelog_producer", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_changelog_catalog" + String dbName = "test_pw_changelog_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_input_partial; + CREATE TABLE paimon.${dbName}.t_input_partial ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'changelog-producer' = 'input' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_lookup; + CREATE TABLE paimon.${dbName}.t_lookup ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'changelog-producer' = 'lookup' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_lookup_aggregation; + CREATE TABLE paimon.${dbName}.t_lookup_aggregation ( + id INT, total BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'changelog-producer' = 'lookup' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_full_compaction; + CREATE TABLE paimon.${dbName}.t_full_compaction ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '1', + 'changelog-producer' = 'full-compaction', + 'changelog-producer.row-deduplicate' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_full_compaction_dynamic; + CREATE TABLE paimon.${dbName}.t_full_compaction_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4', + 'changelog-producer' = 'full-compaction', + 'changelog-producer.row-deduplicate' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + assertEquals(1, rows.size()) + assertTrue(rows[0][0] != null) + return rows[0][0].toString() + } + + def incrementalAuditLog = { tableName, columns, beforeSnapshot, afterSnapshot, orderBy -> + def rows = spark_paimon """ + SELECT ${columns} + FROM paimon_incremental_query( + 'paimon.${dbName}.`${tableName}\$audit_log`', + '${beforeSnapshot}', + '${afterSnapshot}' + ) + ${orderBy} + """ + return rows + } + + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Input producer preserves the incoming row kind and partial-update payload. + sql """INSERT INTO t_input_partial VALUES + (1, 'alice', 10), + (2, 'bob', 20) + """ + String inputBefore = latestSnapshotId("t_input_partial") + sql """INSERT INTO t_input_partial (id, score) VALUES + (1, 15), + (3, 30) + """ + String inputAfter = latestSnapshotId("t_input_partial") + def inputChanges = incrementalAuditLog( + "t_input_partial", "rowkind, id, name, score", inputBefore, inputAfter, + "ORDER BY id") + assertEquals([ + ["+I", 1, null, 15], + ["+I", 3, null, 30] + ], inputChanges) + order_qt_changelog_input_partial """ + SELECT id, name, score FROM t_input_partial ORDER BY id + """ + assertTableEquals("t_input_partial", "ORDER BY id") + + // Lookup producer resolves previous values and emits complete before/after rows. + sql """INSERT INTO t_lookup VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + String lookupBefore = latestSnapshotId("t_lookup") + sql """INSERT INTO t_lookup VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + String lookupAfter = latestSnapshotId("t_lookup") + def lookupChanges = incrementalAuditLog( + "t_lookup", "rowkind, id, name, score", lookupBefore, lookupAfter, + """ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", 1, "old", 10], + ["+U", 1, "new", 11], + ["+I", 3, "added", 30] + ], lookupChanges) + order_qt_changelog_lookup """ + SELECT id, name, score FROM t_lookup ORDER BY id + """ + assertTableEquals("t_lookup", "ORDER BY id") + + // Lookup producer reports the values before and after aggregation. + sql """INSERT INTO t_lookup_aggregation VALUES + (1, 10), + (2, 20) + """ + String aggregationBefore = latestSnapshotId("t_lookup_aggregation") + sql """INSERT INTO t_lookup_aggregation VALUES + (1, 7), + (3, 30) + """ + String aggregationAfter = latestSnapshotId("t_lookup_aggregation") + def aggregationChanges = incrementalAuditLog( + "t_lookup_aggregation", "rowkind, id, total", + aggregationBefore, aggregationAfter, + """ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", 1, 10L], + ["+U", 1, 17L], + ["+I", 3, 30L] + ], aggregationChanges) + order_qt_changelog_lookup_aggregation """ + SELECT id, total FROM t_lookup_aggregation ORDER BY id + """ + assertTableEquals("t_lookup_aggregation", "ORDER BY id") + + // Full-compaction producer must compact every partition/bucket touched by the batch. + sql """INSERT INTO t_full_compaction VALUES + ('p1', 1, 'old'), + ('p2', 2, 'stable') + """ + String fullCompactionBefore = latestSnapshotId("t_full_compaction") + sql """INSERT INTO t_full_compaction VALUES + ('p1', 1, 'new'), + ('p2', 3, 'added') + """ + String fullCompactionAfter = latestSnapshotId("t_full_compaction") + def fullCompactionChanges = incrementalAuditLog( + "t_full_compaction", "rowkind, pt, id, name", + fullCompactionBefore, fullCompactionAfter, + """ORDER BY pt, id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", "p1", 1, "old"], + ["+U", "p1", 1, "new"], + ["+I", "p2", 3, "added"] + ], fullCompactionChanges) + order_qt_changelog_full_compaction """ + SELECT pt, id, name FROM t_full_compaction ORDER BY pt, id + """ + assertTableEquals("t_full_compaction", "ORDER BY pt, id") + + def fullCompactionSnapshot = spark_paimon """ + SELECT commit_kind, changelog_record_count + FROM paimon.${dbName}.`t_full_compaction\$snapshots` + WHERE snapshot_id = ${fullCompactionAfter} + """ + assertEquals([["COMPACT", 3L]], fullCompactionSnapshot) + + // HASH_DYNAMIC uses writer.write(row, assignedBucket). Combining it with + // full-compaction exercises the explicit-bucket writeAndReturn path and + // compacts every dynamically assigned partition/bucket touched by Doris. + sql """INSERT INTO t_full_compaction_dynamic VALUES + ('p1', 1, 'old_1'), + ('p1', 2, 'stable_2'), + ('p2', 3, 'old_3') + """ + String dynamicCompactionBefore = latestSnapshotId("t_full_compaction_dynamic") + sql """INSERT INTO t_full_compaction_dynamic VALUES + ('p1', 1, 'new_1'), + ('p1', 4, 'added_4'), + ('p2', 3, 'new_3'), + ('p2', 5, 'added_5') + """ + String dynamicCompactionAfter = latestSnapshotId("t_full_compaction_dynamic") + def dynamicCompactionChanges = incrementalAuditLog( + "t_full_compaction_dynamic", "rowkind, pt, id, name", + dynamicCompactionBefore, dynamicCompactionAfter, + """ORDER BY pt, id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", "p1", 1, "old_1"], + ["+U", "p1", 1, "new_1"], + ["+I", "p1", 4, "added_4"], + ["-U", "p2", 3, "old_3"], + ["+U", "p2", 3, "new_3"], + ["+I", "p2", 5, "added_5"] + ], dynamicCompactionChanges) + order_qt_changelog_full_compaction_dynamic """ + SELECT pt, id, name + FROM t_full_compaction_dynamic + ORDER BY pt, id + """ + assertTableEquals("t_full_compaction_dynamic", "ORDER BY pt, id") + + def dynamicCompactionSnapshot = spark_paimon """ + SELECT commit_kind, changelog_record_count + FROM paimon.${dbName}.`t_full_compaction_dynamic\$snapshots` + WHERE snapshot_id = ${dynamicCompactionAfter} + """ + assertEquals([["COMPACT", 6L]], dynamicCompactionSnapshot) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_compaction.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_compaction.groovy new file mode 100644 index 00000000000000..b94fd559838ebf --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_compaction.groovy @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_compaction", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_compaction_catalog" + String dbName = "test_pw_compaction_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_auto_compaction; + CREATE TABLE paimon.${dbName}.t_pk_auto_compaction ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id', + 'num-sorted-run.compaction-trigger' = '2', + 'target-file-size' = '1 gb' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_auto_compaction; + CREATE TABLE paimon.${dbName}.t_append_auto_compaction ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'bucket' = '1', + 'bucket-key' = 'id', + 'compaction.min.file-num' = '2', + 'target-file-size' = '1 gb' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def fetchFiles = { String tableName -> + def rows = spark_paimon """ + SELECT level, record_count, file_source + FROM paimon.${dbName}.`${tableName}\$files` + ORDER BY file_path + """ + return rows + } + + def fetchSnapshots = { String tableName -> + def rows = spark_paimon """ + SELECT snapshot_id, commit_kind + FROM paimon.${dbName}.`${tableName}\$snapshots` + ORDER BY snapshot_id + """ + return rows + } + + // A second primary-key write restores the existing L0 file. Two sorted + // runs trigger merge-tree compaction, which must merge the updated key + // and commit the compact increment produced by the JNI writer. + sql """INSERT INTO t_pk_auto_compaction VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + def pkFilesBefore = fetchFiles("t_pk_auto_compaction") + assertEquals(1, pkFilesBefore.size()) + assertEquals(0, pkFilesBefore[0][0].toString().toInteger()) + assertEquals(2L, pkFilesBefore[0][1].toString().toLong()) + assertEquals("APPEND", pkFilesBefore[0][2].toString()) + + sql """INSERT INTO t_pk_auto_compaction VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + order_qt_compaction_pk """ + SELECT id, name, score FROM t_pk_auto_compaction ORDER BY id + """ + def pkRows = sql """SELECT id, name, score FROM t_pk_auto_compaction ORDER BY id""" + assertEquals([ + [1, "new", 11], + [2, "stable", 20], + [3, "added", 30] + ], pkRows) + assertTableEquals("t_pk_auto_compaction", "ORDER BY id") + + def pkFilesAfter = fetchFiles("t_pk_auto_compaction") + assertEquals(1, pkFilesAfter.size()) + assertTrue(pkFilesAfter[0][0].toString().toInteger() > 0) + assertEquals(3L, pkFilesAfter[0][1].toString().toLong()) + assertEquals("COMPACT", pkFilesAfter[0][2].toString()) + + def pkSnapshots = fetchSnapshots("t_pk_auto_compaction") + assertEquals(3, pkSnapshots.size()) + assertEquals(["APPEND", "APPEND", "COMPACT"], + pkSnapshots.collect { row -> row[1].toString() }) + + // Fixed-bucket append-only tables restore existing files for the bucket. + // The second write reaches compaction.min.file-num and rewrites both + // small APPEND files into one COMPACT file without losing duplicates. + sql """INSERT INTO t_append_auto_compaction VALUES + (1, 'a'), + (2, 'b') + """ + def appendFilesBefore = fetchFiles("t_append_auto_compaction") + assertEquals(1, appendFilesBefore.size()) + assertEquals(2L, appendFilesBefore[0][1].toString().toLong()) + assertEquals("APPEND", appendFilesBefore[0][2].toString()) + + sql """INSERT INTO t_append_auto_compaction VALUES + (3, 'c'), + (4, 'd') + """ + order_qt_compaction_append """ + SELECT id, name FROM t_append_auto_compaction ORDER BY id + """ + def appendRows = sql """SELECT id, name FROM t_append_auto_compaction ORDER BY id""" + assertEquals([ + [1, "a"], + [2, "b"], + [3, "c"], + [4, "d"] + ], appendRows) + assertTableEquals("t_append_auto_compaction", "ORDER BY id") + + def appendFilesAfter = fetchFiles("t_append_auto_compaction") + assertEquals(1, appendFilesAfter.size()) + assertEquals(4L, appendFilesAfter[0][1].toString().toLong()) + assertEquals("COMPACT", appendFilesAfter[0][2].toString()) + + def appendSnapshots = fetchSnapshots("t_append_auto_compaction") + assertEquals(3, appendSnapshots.size()) + assertEquals(["APPEND", "APPEND", "COMPACT"], + appendSnapshots.collect { row -> row[1].toString() }) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_complex_types.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_complex_types.groovy new file mode 100644 index 00000000000000..f7be304181898d --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_complex_types.groovy @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_complex_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_cx_catalog" + String dbName = "test_pw_cx_db" + + spark_paimon_multi """ + SET spark.sql.binaryOutputStyle=HEX; + SET spark.sql.timestampType=TIMESTAMP_NTZ; + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_array; + CREATE TABLE paimon.${dbName}.t_array ( + id INT, + c_array_int ARRAY, + c_array_string ARRAY, + c_array_double ARRAY + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_map; + CREATE TABLE paimon.${dbName}.t_map ( + id INT, + c_map_str_int MAP, + c_map_int_str MAP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_struct; + CREATE TABLE paimon.${dbName}.t_struct ( + id INT, + c_struct STRUCT + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_nested; + CREATE TABLE paimon.${dbName}.t_nested ( + id INT, + c_map_arr MAP> + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_recursive; + CREATE TABLE paimon.${dbName}.t_recursive ( + id INT, + c_array_decimal ARRAY, + c_array_date ARRAY, + c_array_timestamp ARRAY, + c_map_decimal MAP, + c_struct_mixed STRUCT, + c_deep MAP>> + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_binary; + CREATE TABLE paimon.${dbName}.t_binary ( + id INT, + c_binary BINARY, + c_array_binary ARRAY, + c_map_binary MAP, + c_struct_binary STRUCT + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'enable.mapping.varbinary' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-028: ARRAY types β€” normal array, empty array, NULL array + sql """INSERT INTO t_array VALUES + (1, [1, 2, 3], ['a', 'b', 'c'], [1.1, 2.2]), + (2, [], [], []), + (3, [10, NULL, 30], ['x', NULL, 'z'], [NULL, 2.0]), + (4, NULL, NULL, NULL) + """ + order_qt_cx_array """SELECT id, c_array_int, c_array_string, c_array_double FROM t_array ORDER BY id""" + assertTableEquals("t_array", "ORDER BY id") + + // FT-029: MAP types β€” normal map, empty map, NULL value + sql """INSERT INTO t_map VALUES + (1, map('math', 90, 'eng', 95), map(1, 'one', 2, 'two')), + (2, map(), map()), + (3, map('science', NULL), map(3, NULL)), + (4, NULL, NULL) + """ + order_qt_cx_map """SELECT id, c_map_str_int, c_map_int_str FROM t_map ORDER BY id""" + assertTableEquals("t_map", "ORDER BY id") + + // FT-030: STRUCT types + sql """INSERT INTO t_struct VALUES + (1, named_struct('name', 'alice', 'age', 30)), + (2, named_struct('name', NULL, 'age', NULL)), + (3, NULL) + """ + order_qt_cx_struct """SELECT id, c_struct FROM t_struct ORDER BY id""" + assertTableEquals("t_struct", "ORDER BY id") + + // Nested: MAP> + sql """INSERT INTO t_nested VALUES + (1, map('group1', [1, 2], 'group2', [3, 4, 5])), + (2, map('empty', [])), + (3, NULL) + """ + // MAP entry order is not part of the SQL result contract. Project known keys so the + // golden result validates the nested arrays without depending on map rendering order. + order_qt_cx_nested """SELECT id, + element_at(c_map_arr, 'group1'), + element_at(c_map_arr, 'group2'), + element_at(c_map_arr, 'empty') + FROM t_nested ORDER BY id""" + assertTableEquals("t_nested", "ORDER BY id") + + // Recursive conversion covers the non-trivial Arrow child vectors which + // cannot use the primitive column fast path in PaimonArrowConverter. + sql """INSERT INTO t_recursive VALUES + ( + 1, + array(CAST(1.250000 AS DECIMAL(18, 6)), CAST(-2.500000 AS DECIMAL(18, 6))), + array(DATE '2024-01-01', DATE '2024-12-31'), + array(TIMESTAMP '2024-01-01 01:02:03.123456', + TIMESTAMP '2024-12-31 23:59:59.654321'), + map(CAST(1.25 AS DECIMAL(8, 2)), CAST(2.50 AS DECIMAL(8, 2)), + CAST(-3.75 AS DECIMAL(8, 2)), CAST(4.00 AS DECIMAL(8, 2))), + named_struct( + 'flag', true, + 'amount', CAST(123.456789 AS DECIMAL(18, 6)), + 'event_date', DATE '2024-02-29', + 'event_time', TIMESTAMP '2024-02-29 12:34:56.000001'), + map('term', array( + named_struct('score', 90, 'label', 'good'), + named_struct('score', 95, 'label', 'better') + )) + ), + ( + 2, + array(CAST(NULL AS DECIMAL(18, 6)), CAST(0.000001 AS DECIMAL(18, 6))), + array(CAST(NULL AS DATE), DATE '1970-01-01'), + array(CAST(NULL AS DATETIME(6)), TIMESTAMP '1970-01-01 00:00:00.000001'), + map(CAST(5.25 AS DECIMAL(8, 2)), CAST(NULL AS DECIMAL(8, 2))), + named_struct( + 'flag', CAST(NULL AS BOOLEAN), + 'amount', CAST(NULL AS DECIMAL(18, 6)), + 'event_date', CAST(NULL AS DATE), + 'event_time', CAST(NULL AS DATETIME(6))), + map('nullable', array( + named_struct('score', CAST(NULL AS INT), 'label', CAST(NULL AS STRING)) + )) + ), + (3, [], [], [], map(), named_struct( + 'flag', false, + 'amount', CAST(0 AS DECIMAL(18, 6)), + 'event_date', DATE '1970-01-01', + 'event_time', TIMESTAMP '1970-01-01 00:00:00'), map()) + """ + + // Every projected field is deliberately in reverse table order. This + // verifies that target type conversion follows Doris input order while + // PaimonWriteSchema restores canonical table-schema order. + sql """INSERT INTO t_recursive ( + c_deep, c_struct_mixed, c_map_decimal, c_array_timestamp, + c_array_date, c_array_decimal, id + ) VALUES ( + map('reverse', array(named_struct('score', 88, 'label', 'reordered'))), + named_struct( + 'flag', true, + 'amount', CAST(8.800000 AS DECIMAL(18, 6)), + 'event_date', DATE '2025-01-01', + 'event_time', TIMESTAMP '2025-01-01 08:08:08.000008'), + map(CAST(8.80 AS DECIMAL(8, 2)), CAST(9.90 AS DECIMAL(8, 2))), + array(TIMESTAMP '2025-01-01 00:00:00.000008'), + array(DATE '2025-01-01'), + array(CAST(8.800008 AS DECIMAL(18, 6))), + 4 + ) + """ + + // A reordered subset expands to a full table row with NULL in every + // omitted nullable field. + sql """INSERT INTO t_recursive (c_deep, c_array_date, id) VALUES ( + map('partial', array(named_struct('score', 77, 'label', 'subset'))), + array(DATE '2026-01-01'), + 5 + )""" + order_qt_cx_recursive """SELECT * FROM t_recursive ORDER BY id""" + assertTableEquals("t_recursive", "ORDER BY id") + + // Top-level and recursively nested BINARY values exercise both the Arrow + // VarBinaryVector fast path and nested convertVectorValue branches. + sql """INSERT INTO t_binary VALUES + ( + 1, + X'0001FEFF', + [X'41', X'00FF'], + map('payload', X'102030'), + named_struct('label', 'binary_1', 'payload', X'DEADBEEF') + ), + ( + 2, + NULL, + [], + map(), + named_struct('label', 'binary_2', 'payload', CAST(NULL AS VARBINARY)) + ), + (3, X'E4B8ADE69687', NULL, NULL, NULL) + """ + // Reordering binary and nested columns also verifies that their target + // Paimon types are resolved by projected column name rather than position. + sql """INSERT INTO t_binary ( + c_struct_binary, c_map_binary, c_array_binary, c_binary, id + ) VALUES ( + named_struct('label', 'reordered', 'payload', X'ABCD'), + map('payload', X'0102'), + [X'03', X'0405'], + X'060708', + 4 + ) + """ + order_qt_cx_binary """ + SELECT id, + HEX(c_binary), + SIZE(c_array_binary), + HEX(ELEMENT_AT(c_array_binary, 1)), + SIZE(c_map_binary), + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM t_binary + ORDER BY id + """ + def sparkBinaryRows = spark_paimon """ + SELECT id, + HEX(c_binary), + CASE WHEN SIZE(c_array_binary) >= 1 + THEN HEX(ELEMENT_AT(c_array_binary, 1)) END, + CASE WHEN SIZE(c_array_binary) >= 2 + THEN HEX(ELEMENT_AT(c_array_binary, 2)) END, + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM paimon.${dbName}.t_binary + ORDER BY id + """ + def dorisBinaryRows = sql """ + SELECT id, + HEX(c_binary), + CASE WHEN SIZE(c_array_binary) >= 1 + THEN HEX(ELEMENT_AT(c_array_binary, 1)) END, + CASE WHEN SIZE(c_array_binary) >= 2 + THEN HEX(ELEMENT_AT(c_array_binary, 2)) END, + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM t_binary + ORDER BY id + """ + assertSparkDorisResultEquals(sparkBinaryRows, dorisBinaryRows) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_deletion_vector.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_deletion_vector.groovy new file mode 100644 index 00000000000000..62462d1a3b9289 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_deletion_vector.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_deletion_vector", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_dv_catalog" + String dbName = "test_pw_dv_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_dv; + CREATE TABLE paimon.${dbName}.t_dv ( + id INT, payload STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id', + 'deletion-vectors.enabled' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_enable_dv; + CREATE TABLE paimon.${dbName}.t_enable_dv ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id', + 'deletion-vectors.modifiable' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """create database if not exists internal.${dbName}""" + sql """drop table if exists internal.${dbName}.dv_source""" + sql """ + create table internal.${dbName}.dv_source ( + id int, payload string, score int, action string + ) distributed by hash(id) buckets 1 + properties ('replication_num' = '1') + """ + + try { + def deletionVectorEntries = { String tableName -> + def rows = spark_paimon """ + SELECT coalesce(sum(row_count), 0) + FROM paimon.${dbName}.`${tableName}\$table_indexes` + WHERE index_type = 'DELETION_VECTORS' + """ + return rows[0][0].toString().toLong() + } + def assertReaders = { String tag, String tableName, String columns, + String orderBy -> + [true, false].each { boolean forceJni -> + sql """set force_jni_scanner = ${forceJni}""" + String reader = forceJni ? "jni" : "native" + "order_qt_${tag}_${reader}" """ + SELECT ${columns} FROM ${tableName} ${orderBy} + """ + } + def sparkRows = spark_paimon """ + SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy} + """ + sql """set force_jni_scanner = false""" + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Start in MOW mode. Do not set write-only=true here: Paimon implements + // primary-key deletion vectors during lookup compaction, while write-only + // deliberately disables that compaction and leaves new level-0 files + // invisible to the DV-optimized reader until a dedicated compaction runs. + sql """INSERT INTO t_dv VALUES + (1, 'old-1', 10), + (2, 'delete-2', 20), + (3, 'delete-by-merge-3', 30) + """ + spark_paimon_multi """ + CALL paimon.sys.compact( + table => '${dbName}.t_dv', + compact_strategy => 'full'); + """ + sql """REFRESH CATALOG ${catalogName}""" + sql """USE ${dbName}""" + sql """INSERT INTO t_dv VALUES + (1, 'upsert-1', 11), + (4, 'insert-4', 40) + """ + sql """DELETE FROM t_dv WHERE id = 2""" + boolean dvProducedBeforeCompact = deletionVectorEntries("t_dv") > 0L + + sql """INSERT INTO internal.${dbName}.dv_source VALUES + (1, 'merged-1', 12, 'U'), + (3, 'unused-3', 0, 'D'), + (5, 'inserted-5', 50, 'I') + """ + sql """ + MERGE INTO t_dv t + USING internal.${dbName}.dv_source s ON t.id = s.id + WHEN MATCHED AND s.action = 'D' THEN DELETE + WHEN MATCHED THEN UPDATE SET payload = s.payload, score = s.score + WHEN NOT MATCHED THEN INSERT (id, payload, score) + VALUES (s.id, s.payload, s.score) + """ + assertReaders("dv_before_compact", "t_dv", "id, payload, score", "ORDER BY id") + long dvEntriesBeforeCompact = deletionVectorEntries("t_dv") + + // Full compaction must preserve the logical rows while materializing at + // least part of the accumulated deletion-vector state. + spark_paimon_multi """ + CALL paimon.sys.compact( + table => '${dbName}.t_dv', + compact_strategy => 'full' + ); + """ + sql """refresh table t_dv""" + assertReaders("dv_after_compact", "t_dv", "id, payload, score", "ORDER BY id") + assertTrue(deletionVectorEntries("t_dv") <= dvEntriesBeforeCompact, + "Full compaction must not increase retained deletion-vector entries") + + // A writer opened after compaction must restore the current index and + // continue to hide the previous physical row for the same key. + sql """INSERT INTO t_dv VALUES + (1, 'post-compact-1', 13), + (6, 'post-compact-6', 60) + """ + assertReaders("dv_post_compact_write", "t_dv", "id, payload, score", "ORDER BY id") + + // P08/P11 transition: enable MOW after MOR files already exist. The + // next Doris statement must reload the changed table options. + sql """INSERT INTO t_enable_dv VALUES + (1, 'mor-old-1'), + (2, 'mor-delete-2') + """ + spark_paimon_multi """ + CALL paimon.sys.compact( + table => '${dbName}.t_enable_dv', + compact_strategy => 'full'); + ALTER TABLE paimon.${dbName}.t_enable_dv SET TBLPROPERTIES ( + 'deletion-vectors.enabled' = 'true') + """ + sql """refresh catalog ${catalogName}""" + sql """use ${dbName}""" + sql """INSERT INTO t_enable_dv VALUES (1, 'mow-new-1')""" + sql """DELETE FROM t_enable_dv WHERE id = 2""" + // The first MOR-to-MOW lookup compaction may rewrite the old files instead + // of retaining a non-empty DV index, so validate the stable contract here: + // both readers must expose the converted update/delete result. + assertReaders("dv_enabled_after_mor", "t_enable_dv", "id, payload", "ORDER BY id") + assertTrue(dvProducedBeforeCompact, + "Doris UPDATE/DELETE must leave a physical deletion vector before compaction") + } finally { + sql """set force_jni_scanner = false""" + sql """drop catalog if exists ${catalogName}""" + sql """drop table if exists internal.${dbName}.dv_source""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_edge_cases.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_edge_cases.groovy new file mode 100644 index 00000000000000..167ae149ed0bfa --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_edge_cases.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_edge_cases", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_edge_catalog" + String dbName = "test_pw_edge_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_str; + CREATE TABLE paimon.${dbName}.t_edge_str ( + id INT, c_string STRING, c_varchar VARCHAR(10) + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_numeric; + CREATE TABLE paimon.${dbName}.t_edge_numeric ( + id INT, + c_tiny TINYINT, + c_small SMALLINT, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_bool; + CREATE TABLE paimon.${dbName}.t_edge_bool ( + id INT, c_bool BOOLEAN + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_pk_null; + CREATE TABLE paimon.${dbName}.t_edge_pk_null ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ('primary-key' = 'id', 'bucket' = '1', 'bucket-key' = 'id'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_mixed_write; + CREATE TABLE paimon.${dbName}.t_mixed_write ( + id INT, name STRING, score DOUBLE + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-042: Empty string and boundary VARCHAR + sql """INSERT INTO t_edge_str VALUES + (1, '', ''), + (2, 'hello world', 'short_str'), + (3, 'x', 'abcdefghij'), + (4, 'very long string over 100 chars: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890', 'max10chars') + """ + order_qt_edge_str """SELECT id, c_varchar FROM t_edge_str ORDER BY id""" + assertTableEquals("t_edge_str", "*", "ORDER BY id") + + // FT-021: Numeric boundary values (INT_MIN, INT_MAX, etc.) + sql """INSERT INTO t_edge_numeric VALUES + (1, CAST(127 AS TINYINT), CAST(32767 AS SMALLINT), 2147483647, + CAST(9223372036854775807 AS BIGINT), + CAST(3.4028235E38 AS FLOAT), CAST(1.7976931348623157E308 AS DOUBLE)), + (2, CAST(-128 AS TINYINT), CAST(-32768 AS SMALLINT), -2147483648, + CAST(-9223372036854775808 AS BIGINT), + CAST(-3.4028235E38 AS FLOAT), CAST(-1.7976931348623157E308 AS DOUBLE)), + (3, CAST(0 AS TINYINT), CAST(0 AS SMALLINT), 0, + CAST(0 AS BIGINT), + CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE)) + """ + order_qt_edge_numeric """SELECT id, c_tiny, c_small, c_int, c_bigint FROM t_edge_numeric ORDER BY id""" + assertTableEquals("t_edge_numeric", """ + id, c_tiny, c_small, c_int, c_bigint, + c_float / 1.0E38, + c_double / 1.0E308 + """, "ORDER BY id") + + // BOOLEAN with NULL and both true/false + sql """INSERT INTO t_edge_bool VALUES (1, true), (2, false), (3, NULL)""" + order_qt_edge_bool """SELECT id, c_bool FROM t_edge_bool ORDER BY id""" + assertTableEquals("t_edge_bool", "*", "ORDER BY id") + + // FT-041: PK table β€” insert NULL values, then update with non-NULL + sql """INSERT INTO t_edge_pk_null VALUES (1, 'first'), (2, NULL)""" + assertTableEquals("t_edge_pk_null", "*", "ORDER BY id") + + sql """INSERT INTO t_edge_pk_null VALUES (2, 'updated'), (3, 'third')""" + order_qt_edge_pk_null """SELECT id, name FROM t_edge_pk_null ORDER BY id""" + assertTableEquals("t_edge_pk_null", "*", "ORDER BY id") + + // Mixed INSERT patterns: VALUES, then SELECT from self, then single-row VALUES + sql """INSERT INTO t_mixed_write VALUES (1, 'a', 10.0), (2, 'b', 20.0)""" + sql """INSERT INTO t_mixed_write VALUES (3, 'c', 30.0)""" + sql """INSERT INTO t_mixed_write SELECT id + 3, concat(name, '_copy'), score + 30.0 FROM t_mixed_write""" + order_qt_edge_mixed """SELECT id, name, score FROM t_mixed_write ORDER BY id""" + assertTableEquals("t_mixed_write", "*", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy new file mode 100644 index 00000000000000..d12b3e3a751703 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_external_paths", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_external_paths_catalog" + String dbName = "test_pw_external_paths_db" + String pathRoot = "s3://warehouse/paimon-external-paths/${dbName}" + String filesTableSuffix = '$files' + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_round_robin; + CREATE TABLE paimon.${dbName}.t_round_robin ( + pt STRING, id INT, payload STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '1', + 'write-only' = 'true', + 'target-file-size' = '1 kb', + 'data-file.external-paths' = '${pathRoot}/round-a,${pathRoot}/round-b', + 'data-file.external-paths.strategy' = 'round-robin' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_round_robin_roll; + CREATE TABLE paimon.${dbName}.t_round_robin_roll ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-only' = 'true', + 'target-file-size' = '1 kb', + 'data-file.external-paths' = '${pathRoot}/round-roll-a,${pathRoot}/round-roll-b', + 'data-file.external-paths.strategy' = 'round-robin' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_weight_robin; + CREATE TABLE paimon.${dbName}.t_weight_robin ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-only' = 'true', + 'target-file-size' = '1 kb', + 'data-file.external-paths' = '${pathRoot}/weight-a,${pathRoot}/weight-b', + 'data-file.external-paths.strategy' = 'weight-robin', + 'data-file.external-paths.weights' = '1,1' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_specific_fs; + CREATE TABLE paimon.${dbName}.t_specific_fs ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-only' = 'true', + 'data-file.external-paths' = '${pathRoot}/specific-a,${pathRoot}/specific-b', + 'data-file.external-paths.strategy' = 'specific-fs', + 'data-file.external-paths.specific-fs' = 's3' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_none; + CREATE TABLE paimon.${dbName}.t_none ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-only' = 'true', + 'data-file.external-paths' = '${pathRoot}/unused-a,${pathRoot}/unused-b', + 'data-file.external-paths.strategy' = 'none' + ); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + def dataFiles = { String tableName -> + String query = """ + SELECT file_path + FROM paimon.${dbName}.`${tableName}${filesTableSuffix}` + ORDER BY file_path + """ + return spark_paimon(query).collect { row -> row[0].toString() } + } + def assertDorisSparkRows = { String tag, String tableName, + String columns, String orderBy -> + def sparkRows = spark_paimon """ + SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy} + """ + "order_qt_${tag}" """ + SELECT ${columns} FROM ${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Separate statements force separate writer lifecycles. + sql """INSERT INTO t_round_robin VALUES ('p1', 1, 'one')""" + sql """INSERT INTO t_round_robin VALUES ('p1', 2, 'two')""" + sql """INSERT INTO t_round_robin VALUES ('p2', 3, 'three')""" + sql """INSERT INTO t_round_robin VALUES ('p2', 4, 'four')""" + def oldRoundFiles = dataFiles("t_round_robin") + assertFalse(oldRoundFiles.isEmpty()) + // Each lifecycle randomly initializes its round-robin position, so independent + // statements need not hit both paths. Only membership in the configured set is stable. + assertTrue(oldRoundFiles.every { + it.startsWith("${pathRoot}/round-a/") || + it.startsWith("${pathRoot}/round-b/") + }) + assertDorisSparkRows("external_round_robin_initial", "t_round_robin", + "pt, id, length(payload)", "ORDER BY pt, id") + + // Isolate the round-robin oracle from the independent writers above. One fixed bucket and + // one pipeline task keep all rows in a single Paimon writer. Paimon checks file rolling + // every 1000 rows; 4000 deterministic high-entropy rows leave enough margin to roll + // repeatedly and therefore visit both roots regardless of its random start. + sql """SET parallel_pipeline_task_num = 1""" + try { + sql """ + INSERT INTO t_round_robin_roll + SELECT CAST(number AS INT), + concat(md5(CAST(number AS STRING)), + md5(CAST(number + 100000 AS STRING))) + FROM numbers("number" = "4000") + """ + } finally { + sql """SET parallel_pipeline_task_num = 0""" + } + def rolledFiles = dataFiles("t_round_robin_roll") + assertTrue(rolledFiles.size() >= 2) + assertTrue(rolledFiles.every { + it.startsWith("${pathRoot}/round-roll-a/") || + it.startsWith("${pathRoot}/round-roll-b/") + }) + assertTrue(rolledFiles.any { it.startsWith("${pathRoot}/round-roll-a/") }) + assertTrue(rolledFiles.any { it.startsWith("${pathRoot}/round-roll-b/") }) + assertDorisSparkRows("external_round_robin_roll", "t_round_robin_roll", + "count(*), sum(id), min(length(payload)), max(length(payload))", "") + + spark_paimon """ + ALTER TABLE paimon.${dbName}.t_round_robin SET TBLPROPERTIES ( + 'data-file.external-paths' = '${pathRoot}/round-c,${pathRoot}/round-d' + ) + """ + sql """REFRESH CATALOG ${catalogName}""" + sql """USE ${dbName}""" + sql """INSERT INTO t_round_robin VALUES ('p3', 5, 'five')""" + sql """INSERT INTO t_round_robin VALUES ('p3', 6, 'six')""" + sql """ + INSERT INTO t_round_robin + SELECT 'p-new-bulk', CAST(number + 200 AS INT), repeat('y', 2048) + FROM numbers("number" = "16") + """ + def changedRoundFiles = dataFiles("t_round_robin") + boolean oldRoundFilesRetained = changedRoundFiles.containsAll(oldRoundFiles) + def newRoundFiles = changedRoundFiles - oldRoundFiles + assertTrue(oldRoundFilesRetained) + assertFalse(newRoundFiles.isEmpty()) + // Round-robin selection is scoped to a writer lifecycle, so a small + // number of independent Doris statements need not hit both paths. The + // stable contract is that every new file uses the refreshed path set. + assertTrue(newRoundFiles.every { + it.startsWith("${pathRoot}/round-c/") || + it.startsWith("${pathRoot}/round-d/") + }) + assertDorisSparkRows("external_round_robin_changed", "t_round_robin", + "pt, id, length(payload)", "ORDER BY pt, id") + + (1..6).each { id -> + sql """INSERT INTO t_weight_robin VALUES (${id}, 'weight-${id}')""" + } + def weightedFiles = dataFiles("t_weight_robin") + assertFalse(weightedFiles.isEmpty()) + assertTrue(weightedFiles.every { + it.startsWith("${pathRoot}/weight-a/") || + it.startsWith("${pathRoot}/weight-b/") + }) + assertDorisSparkRows("external_weight_robin", "t_weight_robin", + "id, payload", "ORDER BY id") + + sql """INSERT INTO t_specific_fs VALUES (1, 'specific-1')""" + sql """INSERT INTO t_specific_fs VALUES (2, 'specific-2')""" + def specificFiles = dataFiles("t_specific_fs") + assertFalse(specificFiles.isEmpty()) + assertTrue(specificFiles.every { it.startsWith("${pathRoot}/specific-") }) + assertDorisSparkRows("external_specific_fs", "t_specific_fs", + "id, payload", "ORDER BY id") + + sql """INSERT INTO t_none VALUES (1, 'default-path')""" + def defaultFiles = dataFiles("t_none") + assertFalse(defaultFiles.isEmpty()) + assertTrue(defaultFiles.every { !it.startsWith(pathRoot) }) + assertDorisSparkRows("external_default_path", "t_none", + "id, payload", "ORDER BY id") + } finally { + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_failures.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_failures.groovy new file mode 100644 index 00000000000000..12c7b1316cb9a0 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_failures.groovy @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_failures", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_failure_catalog" + String dbName = "test_pw_failure_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_atomic_append; + CREATE TABLE paimon.${dbName}.t_atomic_append ( + id INT NOT NULL, + payload STRING NOT NULL, + dt STRING NOT NULL + ) USING paimon + PARTITIONED BY (dt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_not_null; + CREATE TABLE paimon.${dbName}.t_pk_not_null ( + id INT NOT NULL, + payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertAtomicAppendState = { long expectedRows, long expectedSnapshots -> + assertEquals(expectedRows, + (sql """SELECT COUNT(*) FROM t_atomic_append""")[0][0] as long) + assertEquals(expectedSnapshots, + (sql """SELECT COUNT(*) FROM t_atomic_append\$snapshots""")[0][0] as long) + } + + // A failure after an earlier row has already entered the JNI writer must + // abort the whole statement, including data for a different partition. + sql """INSERT INTO t_atomic_append VALUES (1, 'baseline', 'p0')""" + order_qt_failure_atomic_before """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_atomic_snapshot_before """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + test { + sql """INSERT INTO t_atomic_append VALUES + (2, 'accepted_before_error', 'p1'), + (3, NULL, 'p2')""" + exception "Cannot write null to non-null column(payload)" + } + assertAtomicAppendState(1L, 1L) + order_qt_failure_atomic_after """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_atomic_snapshot_after """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + // An omitted field without a Paimon default remains NULL and is validated + // against the real Paimon schema by the Paimon writer. + test { + sql """INSERT INTO t_atomic_append (id, dt) VALUES (4, 'p4')""" + exception "Cannot write null to non-null column(payload)" + } + + // Partition columns follow the same Paimon nullability contract. + test { + sql """INSERT INTO t_atomic_append VALUES (4, 'bad_partition', NULL)""" + exception "Cannot write null to non-null column(dt)" + } + assertAtomicAppendState(1L, 1L) + + // These errors are rejected during target-column and partition binding and + // therefore must not create a writer or a new Paimon snapshot. + test { + sql """INSERT INTO t_atomic_append (id, payload, dt, missing) + VALUES (5, 'unknown_column', 'p5', 1)""" + exception "Unknown column 'missing' in target table" + } + test { + sql """INSERT INTO t_atomic_append (id, payload, dt) + VALUES (5, 'too_few_values')""" + exception "Column count doesn't match value count" + } + test { + sql """INSERT OVERWRITE TABLE t_atomic_append + PARTITION (payload = 'not_a_partition') VALUES (5, 'p5')""" + exception "is not a partition column of Paimon table" + } + assertAtomicAppendState(1L, 1L) + + // A successful statement after several failures verifies that failed JNI + // writers and transactions do not poison subsequent writes. + sql """INSERT INTO t_atomic_append VALUES (5, 'recovered', 'p5')""" + assertAtomicAppendState(2L, 2L) + order_qt_failure_recovered """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_recovered_snapshot """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + // A failed overwrite must not publish its replacement files or remove the + // data referenced by the previous committed snapshot. + test { + sql """INSERT OVERWRITE TABLE t_atomic_append VALUES + (10, 'would_replace', 'p10'), + (11, NULL, 'p11')""" + exception "Cannot write null to non-null column(payload)" + } + assertAtomicAppendState(2L, 2L) + order_qt_failure_overwrite_after """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_overwrite_snapshot_after """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + assertTableEquals("t_atomic_append", "ORDER BY id") + + // Primary-key nullability is checked before bucket routing. A rejected row + // must publish no snapshot, and the table remains writable afterwards. + test { + sql """INSERT INTO t_pk_not_null VALUES (NULL, 'invalid_key')""" + exception "Cannot write null to non-null column(id)" + } + assertEquals(0L, + (sql """SELECT COUNT(*) FROM t_pk_not_null\$snapshots""")[0][0] as long) + + sql """INSERT INTO t_pk_not_null VALUES (1, 'valid_after_failure')""" + order_qt_failure_pk_recovered """ + SELECT id, payload FROM t_pk_not_null ORDER BY id + """ + qt_failure_pk_snapshot """ + SELECT COUNT(*) FROM t_pk_not_null\$snapshots + """ + assertTableEquals("t_pk_not_null", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_key_dynamic_memory_negative.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_key_dynamic_memory_negative.groovy new file mode 100644 index 00000000000000..222562d73f8981 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_key_dynamic_memory_negative.groovy @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.DriverManager +import java.util.concurrent.atomic.AtomicReference + +suite("test_paimon_write_key_dynamic_memory_negative", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + // This opt-in case intentionally puts sustained pressure on the embedded JVM. + String knownBugTestEnabled = context.config.otherConfigs.get("enablePaimonKnownBugTest") + if (knownBugTestEnabled == null || !knownBugTestEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Paimon known-bug resource regression") + return + } + + long stressRows = (context.config.otherConfigs.get("paimonKeyDynamicStressRows") + ?: "4000000").toLong() + long queryMemoryLimit = 128L * 1024 * 1024 + long allowedJvmGrowth = queryMemoryLimit + 64L * 1024 * 1024 + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_key_dynamic_memory_catalog" + String dbName = "test_pw_key_dynamic_memory_db" + + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort) + def backendEndpoints = backendIdToIp.collectEntries { backendId, ip -> + [(backendId): [ip.toString(), backendIdToHttpPort[backendId].toString()]] + } + assertFalse(backendEndpoints.isEmpty()) + def heapUsed = { + backendEndpoints.collectEntries { backendId, endpoint -> + [(backendId): (get_be_metric(endpoint[0], endpoint[1], + "jvm_heap_size_bytes", "used") as long)] + } + } + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_memory; + CREATE TABLE paimon.${dbName}.t_key_dynamic_memory ( + pt STRING, id STRING, payload STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '10000', + 'dynamic-bucket.max-buckets' = '64', + 'write-buffer-size' = '16 mb', + 'page-size' = '64 kb', + 'write-buffer-spillable' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + sql """INSERT INTO t_key_dynamic_memory VALUES ('warmup', 'warmup', 'warmup')""" + sleep(3000) + def baseline = heapUsed() + def peak = new LinkedHashMap(baseline) + def writeFailure = new AtomicReference() + def activeStatement = new AtomicReference() + + Thread writerThread = Thread.start("paimon-key-dynamic-memory-writer") { + try (def connection = DriverManager.getConnection(context.config.jdbcUrl, + context.config.jdbcUser, context.config.jdbcPassword); + def statement = connection.createStatement()) { + activeStatement.set(statement) + statement.execute("SET exec_mem_limit = ${queryMemoryLimit}") + statement.execute("SWITCH ${catalogName}") + statement.execute("USE ${dbName}") + statement.execute(""" + INSERT INTO t_key_dynamic_memory + SELECT concat('p', CAST(number % 64 AS STRING)), + concat(lpad(CAST(number AS STRING), 20, '0'), repeat('k', 76)), + repeat('v', 32) + FROM numbers("number" = "${stressRows}") + """) + } catch (Throwable t) { + writeFailure.set(t) + } finally { + activeStatement.set(null) + } + } + + long deadline = System.currentTimeMillis() + 20L * 60 * 1000 + while (writerThread.isAlive() && System.currentTimeMillis() < deadline) { + sleep(1000) + heapUsed().each { backendId, used -> + peak[backendId] = Math.max(peak[backendId], used) + } + } + writerThread.join(10000) + if (writerThread.isAlive()) { + // Cancel the stress query before failing so a timeout cannot leave its + // JDBC writer running after the regression suite has already finished. + activeStatement.get()?.cancel() + writerThread.join(10000) + } + assertFalse(writerThread.isAlive(), "KEY_DYNAMIC stress insert did not finish within 20 minutes") + + def growth = peak.collectEntries { backendId, used -> + [(backendId): used - baseline[backendId]] + } + def failureMessages = [] + Throwable failure = writeFailure.get() + while (failure != null && !failureMessages.contains(failure.toString())) { + failureMessages.add(failure.toString()) + failure = failure.getCause() + } + String failureMessage = failureMessages.join(" caused by ") + logger.info("Paimon KEY_DYNAMIC memory result: rows=${stressRows}, baseline=${baseline}, " + + "peak=${peak}, growth=${growth}, failure=${failureMessage}") + + // A valid query may be rejected by a memory limit, but the embedded JVM + // must not be the component that exhausts memory outside Doris accounting. + assertFalse(failureMessage.contains("OutOfMemoryError"), + "KEY_DYNAMIC write exhausted the embedded JVM: ${failureMessage}") + assertTrue(growth.values().every { delta -> delta <= allowedJvmGrowth }, + "KEY_DYNAMIC Java heap growth escaped the query memory limit: " + + "limit=${queryMemoryLimit}, allowed=${allowedJvmGrowth}, growth=${growth}") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_engine.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_engine.groovy new file mode 100644 index 00000000000000..a0dbdd944c4870 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_engine.groovy @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_merge_engine", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_merge_engine_catalog" + String dbName = "test_pw_merge_engine_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_partial_update; + CREATE TABLE paimon.${dbName}.t_partial_update ( + id INT, name STRING, score DOUBLE, note STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_first_row; + CREATE TABLE paimon.${dbName}.t_first_row ( + id INT, name STRING, score DOUBLE + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'first-row' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_aggregation; + CREATE TABLE paimon.${dbName}.t_aggregation ( + id INT, total BIGINT, highest DOUBLE, label STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'fields.highest.aggregate-function' = 'max' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Partial-update accepts both full rows and arbitrary value-column subsets. + sql """INSERT INTO t_partial_update VALUES + (1, 'alice', 10.0, 'created'), + (2, 'bob', 20.0, 'created') + """ + sql """INSERT INTO t_partial_update (score, id) VALUES (15.5, 1)""" + sql """INSERT INTO t_partial_update (note, id) VALUES ('score_updated', 1)""" + sql """INSERT INTO t_partial_update (id, name) VALUES (1, NULL)""" + sql """INSERT INTO t_partial_update (name, id) VALUES ('charlie', 3)""" + sql """INSERT INTO t_partial_update VALUES (2, 'bob_full', 25.0, 'full_update')""" + order_qt_partial_update """SELECT id, name, score, note + FROM t_partial_update ORDER BY id""" + assertTableEquals("t_partial_update", "ORDER BY id") + + // An omitted primary key reaches the SDK as NULL in the complete table row. + // Let Paimon's real NOT NULL schema enforce the primary-key requirement. + test { + sql """INSERT INTO t_partial_update (name, score) VALUES ('missing_pk', 1.0)""" + exception "Cannot write null to non-null column(id)" + } + + // First-row keeps the first value observed for each primary key across writes. + sql """INSERT INTO t_first_row VALUES + (1, 'first_1', 10.0), + (2, 'first_2', 20.0) + """ + sql """INSERT INTO t_first_row VALUES + (2, 'second_2', 21.0), + (1, 'second_1', 11.0), + (3, 'first_3', 30.0) + """ + sql """INSERT INTO t_first_row VALUES (1, 'third_1', 12.0)""" + order_qt_first_row """SELECT id, name, score FROM t_first_row ORDER BY id""" + assertTableEquals("t_first_row", "ORDER BY id") + + test { + sql """INSERT INTO t_first_row (id, name) VALUES (4, 'partial')""" + exception "table uses merge-engine=first-row" + } + + // Aggregation applies the configured function per value field. Fields without + // an explicit function use Paimon's default last_non_null_value aggregation. + sql """INSERT INTO t_aggregation VALUES + (1, 10, 90.0, 'first_1'), + (2, 5, 70.0, 'first_2') + """ + sql """INSERT INTO t_aggregation VALUES + (1, 20, 85.0, NULL), + (2, 3, 80.0, NULL), + (3, 7, 60.0, 'first_3') + """ + sql """INSERT INTO t_aggregation VALUES (1, 7, 95.0, 'latest_1')""" + order_qt_aggregation """SELECT id, total, highest, label + FROM t_aggregation ORDER BY id""" + assertTableEquals("t_aggregation", "ORDER BY id") + + test { + sql """INSERT INTO t_aggregation (id, total) VALUES (4, 100)""" + exception "table uses merge-engine=aggregation" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_semantics.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_semantics.groovy new file mode 100644 index 00000000000000..2fe05e130df8d8 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_semantics.groovy @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_merge_semantics", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_merge_semantics_catalog" + String dbName = "test_pw_merge_semantics_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_merge; + CREATE TABLE paimon.${dbName}.t_merge ( + id INT, + score INT, + payload STRUCT, + status STRING, + required_value STRING NOT NULL + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'num-sorted-run.compaction-trigger' = '100' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """create database if not exists internal.${dbName}""" + sql """drop table if exists internal.${dbName}.merge_source""" + sql """ + create table internal.${dbName}.merge_source ( + id int, + delta int, + new_x int, + new_y string, + action string, + required_value string + ) distributed by hash(id) buckets 1 + properties ('replication_num' = '1') + """ + + try { + def latestSnapshotId = { + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`t_merge\$snapshots` + """ + return rows[0][0] == null ? 0L : rows[0][0].toString().toLong() + } + def activeFileCount = { + def rows = spark_paimon """ + SELECT count(*) FROM paimon.${dbName}.`t_merge\$files` + """ + return rows[0][0].toString().toLong() + } + def assertCrossEngine = { + def sparkRows = spark_paimon """ + SELECT id, score, payload.x, payload.y, status, required_value + FROM paimon.${dbName}.t_merge ORDER BY id + """ + def dorisRows = sql """ + SELECT id, score, payload.x, payload.y, status, required_value + FROM t_merge ORDER BY id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """INSERT INTO t_merge VALUES + (1, 10, named_struct('x', 1, 'y', 'base-1'), 'old', 'required-1'), + (2, 20, named_struct('x', 2, 'y', 'base-2'), 'old', 'required-2'), + (3, 30, named_struct('x', 3, 'y', 'base-3'), 'stable', 'required-3') + """ + sql """INSERT INTO internal.${dbName}.merge_source VALUES + (1, 5, 11, 'source-1', 'U', 'required-1-new'), + (2, 0, 22, 'source-2', 'D', 'required-2-new'), + (4, 40, 44, 'source-4', 'I', 'required-4') + """ + + long beforeMerge = latestSnapshotId() + sql """ + MERGE INTO t_merge t + USING internal.${dbName}.merge_source s + ON t.id = s.id + WHEN MATCHED AND s.action = 'U' THEN UPDATE SET + score = t.score + s.delta, + payload = named_struct( + 'x', s.new_x, + 'y', concat(t.payload.y, '-', s.new_y)), + status = 'updated', + required_value = s.required_value + WHEN MATCHED THEN DELETE + WHEN NOT MATCHED AND s.action = 'I' THEN INSERT + (required_value, status, payload, score, id) + VALUES ( + s.required_value, + 'inserted', + named_struct('x', s.new_x, 'y', s.new_y), + s.delta, + s.id) + """ + assertEquals(beforeMerge + 1L, latestSnapshotId()) + order_qt_merge_semantics_result """ + SELECT id, score, payload.x, payload.y, status, required_value + FROM t_merge ORDER BY id + """ + assertCrossEngine() + + // An empty source is a true no-op: it must not publish an empty Paimon + // snapshot or alter the active file set. + sql """TRUNCATE TABLE internal.${dbName}.merge_source""" + long beforeEmptySnapshot = latestSnapshotId() + long beforeEmptyFiles = activeFileCount() + sql """ + MERGE INTO t_merge t + USING internal.${dbName}.merge_source s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET score = t.score + s.delta + WHEN NOT MATCHED THEN INSERT + (id, score, payload, status, required_value) + VALUES (s.id, s.delta, + named_struct('x', s.new_x, 'y', s.new_y), + 'inserted', s.required_value) + """ + assertEquals(beforeEmptySnapshot, latestSnapshotId()) + assertEquals(beforeEmptyFiles, activeFileCount()) + + // P09 failure atomicity: a forbidden key update and a missing required + // insert column both fail before a snapshot or file becomes visible. + sql """INSERT INTO internal.${dbName}.merge_source VALUES + (1, 1, 100, 'invalid-key-update', 'U', 'still-required') + """ + long beforeFailureSnapshot = latestSnapshotId() + long beforeFailureFiles = activeFileCount() + test { + sql """ + MERGE INTO t_merge t + USING internal.${dbName}.merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET id = s.id + 100 + """ + exception "primary-key" + } + assertEquals(beforeFailureSnapshot, latestSnapshotId()) + assertEquals(beforeFailureFiles, activeFileCount()) + + sql """TRUNCATE TABLE internal.${dbName}.merge_source""" + sql """INSERT INTO internal.${dbName}.merge_source VALUES + (9, 9, 9, 'missing-required', 'I', NULL) + """ + test { + sql """ + MERGE INTO t_merge t + USING internal.${dbName}.merge_source s ON t.id = s.id + WHEN NOT MATCHED THEN INSERT (id, score, payload, status) + VALUES (s.id, s.delta, + named_struct('x', s.new_x, 'y', s.new_y), 'invalid') + """ + exception "requires values for every table column" + } + assertEquals(beforeFailureSnapshot, latestSnapshotId()) + assertEquals(beforeFailureFiles, activeFileCount()) + + // A legal MERGE after both failures proves that no stale writer or + // transaction state poisoned the table. + sql """TRUNCATE TABLE internal.${dbName}.merge_source""" + sql """INSERT INTO internal.${dbName}.merge_source VALUES + (5, 50, 55, 'recovery', 'I', 'required-5') + """ + sql """ + MERGE INTO t_merge t + USING internal.${dbName}.merge_source s ON t.id = s.id + WHEN NOT MATCHED THEN INSERT + (id, score, payload, status, required_value) + VALUES (s.id, s.delta, + named_struct('x', s.new_x, 'y', s.new_y), + 'recovered', s.required_value) + """ + assertEquals(beforeFailureSnapshot + 1L, latestSnapshotId()) + order_qt_merge_semantics_recovered """ + SELECT id, status FROM t_merge WHERE id = 5 ORDER BY id + """ + assertCrossEngine() + } finally { + sql """drop catalog if exists ${catalogName}""" + sql """drop table if exists internal.${dbName}.merge_source""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_partition_delete.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_partition_delete.groovy new file mode 100644 index 00000000000000..3334a6cf005ccd --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_partition_delete.groovy @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_partition_delete", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_partition_delete_catalog" + String dbName = "test_pw_partition_delete_db" + String snapshotsTableSuffix = '$snapshots' + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_partition_delete; + CREATE TABLE paimon.${dbName}.t_partition_delete ( + pt STRING, id INT, score INT, payload STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'write-only' = 'true' + ); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + def latestSnapshot = { + String query = """ + SELECT snapshot_id, commit_kind + FROM paimon.${dbName}.`t_partition_delete${snapshotsTableSuffix}` + ORDER BY snapshot_id DESC LIMIT 1 + """ + def rows = spark_paimon(query) + return rows.isEmpty() ? [0L, null] : [ + rows[0][0].toString().toLong(), rows[0][1].toString().toUpperCase()] + } + def assertRows = { String tag -> + "order_qt_${tag}" """ + SELECT pt, id, score, payload + FROM t_partition_delete ORDER BY id + """ + def dorisRows = sql """ + SELECT pt, id, score, payload + FROM t_partition_delete ORDER BY id + """ + def sparkRows = spark_paimon """ + SELECT pt, id, score, payload + FROM paimon.${dbName}.t_partition_delete ORDER BY id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + def assertDeleteCommit = { List before -> + def after = latestSnapshot() + assertEquals(before[0] + 1L, after[0]) + assertEquals("APPEND", after[1]) + return after + } + + sql """INSERT INTO t_partition_delete VALUES + ('p1', 1, 10, 'p1-a'), + ('p1', 2, 20, 'p1-b'), + ('p2', 3, 30, 'p2-a'), + ('p2', 4, 40, 'p2-b'), + ('p3', 5, 50, 'p3-a'), + (NULL, 6, 60, 'default-partition') + """ + + // A predicate covering the complete partition must not affect any + // other partition, including the default partition represented by NULL. + def snapshot = latestSnapshot() + sql """DELETE FROM t_partition_delete WHERE pt = 'p1'""" + snapshot = assertDeleteCommit(snapshot) + assertRows("partition_delete_full_partition") + + // A partial-partition predicate is evaluated row by row. + sql """DELETE FROM t_partition_delete WHERE pt = 'p2' AND score >= 40""" + snapshot = assertDeleteCommit(snapshot) + assertRows("partition_delete_partial_partition") + + // A non-convertible partition expression must retain its exact SQL + // semantics instead of expanding into a full-partition delete. + sql """DELETE FROM t_partition_delete WHERE upper(pt) = 'P2' AND id = 3""" + snapshot = assertDeleteCommit(snapshot) + assertRows("partition_delete_expression") + + // UNKNOWN predicates match no rows and must not create empty commits. + sql """DELETE FROM t_partition_delete WHERE pt = NULL""" + assertEquals(snapshot, latestSnapshot()) + sql """DELETE FROM t_partition_delete WHERE pt NOT IN ('p3', NULL)""" + assertEquals(snapshot, latestSnapshot()) + sql """DELETE FROM t_partition_delete WHERE EXISTS (SELECT 1 WHERE FALSE)""" + assertEquals(snapshot, latestSnapshot()) + + // NOT EXISTS is true here and is combined with a target predicate so + // only the intended row is removed. + sql """ + DELETE FROM t_partition_delete + WHERE id = 5 AND NOT EXISTS (SELECT 1 WHERE FALSE) + """ + snapshot = assertDeleteCommit(snapshot) + assertRows("partition_delete_not_exists") + + // IS NULL addresses the default partition explicitly. + sql """DELETE FROM t_partition_delete WHERE pt IS NULL""" + snapshot = assertDeleteCommit(snapshot) + assertRows("partition_delete_default_partition") + + // A second no-match delete proves the empty-table path also avoids a + // metadata-only snapshot. + sql """DELETE FROM t_partition_delete WHERE id = 999""" + assertEquals(snapshot, latestSnapshot()) + } finally { + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_pk.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_pk.groovy new file mode 100644 index 00000000000000..4034cab882c411 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_pk.groovy @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_pk", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_pk_catalog" + String dbName = "test_pw_pk_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_dedup; + CREATE TABLE paimon.${dbName}.t_pk_dedup ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_bucket4; + CREATE TABLE paimon.${dbName}.t_pk_bucket4 ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '4', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_composite; + CREATE TABLE paimon.${dbName}.t_pk_composite ( + user_id INT, event_time BIGINT, event_type STRING, value DOUBLE + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'user_id,event_time', + 'bucket' = '2', + 'bucket-key' = 'user_id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_string_bucket; + CREATE TABLE paimon.${dbName}.t_pk_string_bucket ( + user_key STRING, event_id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'user_key,event_id', + 'bucket' = '4', + 'bucket-key' = 'user_key' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_writer_scaling; + CREATE TABLE paimon.${dbName}.t_pk_writer_scaling ( + id INT, version BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // Prepare an internal OLAP source table for INSERT INTO ... SELECT + sql """create database if not exists internal.${dbName}""" + sql """drop table if exists internal.${dbName}.t_source""" + sql """ + CREATE TABLE internal.${dbName}.t_source ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1'); + """ + sql """INSERT INTO internal.${dbName}.t_source VALUES + (1, 'alice', 95.5, 1000), + (1, 'alice_updated', 99.0, 2000), + (2, 'bob', 87.0, 1000), + (3, 'charlie', 92.3, 1000), + (3, 'charlie_v2', 88.0, 1500), + (4, 'diana', 91.0, 1000), + (5, 'eve', 85.0, 1000) + """ + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-004: PK table, deduplicate β€” duplicate keys merged by Paimon SDK + sql """INSERT INTO t_pk_dedup SELECT id, name, score, ts FROM internal.${dbName}.t_source""" + order_qt_pk_dedup """SELECT id, name, score FROM t_pk_dedup ORDER BY id""" + assertTableEquals("t_pk_dedup", "ORDER BY id") + + // FT-005: Interleaved duplicate keys verify that SDK routing preserves the + // input order within each bucket and the last row for each key wins. + sql """INSERT INTO t_pk_dedup VALUES + (100, 'key100_v1', 10.0, 1000), + (200, 'key200_v1', 20.0, 1000), + (100, 'key100_v2', 11.0, 2000), + (200, 'key200_v2', 21.0, 2000), + (100, 'key100_v3', 12.0, 3000) + """ + order_qt_pk_interleaved """SELECT id, name, score, ts FROM t_pk_dedup + WHERE id >= 100 ORDER BY id""" + assertTableEquals("t_pk_dedup", "ORDER BY id") + + // FT-003: Fixed bucket table with multiple buckets + for (int i = 0; i < 20; i++) { + sql """INSERT INTO t_pk_bucket4 VALUES (${i}, 'row${i}')""" + } + order_qt_pk_bucket """SELECT id, name FROM t_pk_bucket4 ORDER BY id""" + assertTableEquals("t_pk_bucket4", "ORDER BY id") + + // PK table with composite primary key + sql """INSERT INTO t_pk_composite VALUES + (1, 100, 'click', 1.0), + (1, 200, 'view', 2.0), + (2, 100, 'click', 3.0), + (1, 100, 'click_updated', 99.0) + """ + // (1,100) duplicated β†’ 3 unique PKs: (1,100), (1,200), (2,100) + order_qt_pk_composite """SELECT user_id, event_time, event_type, value FROM t_pk_composite ORDER BY user_id, event_time""" + assertTableEquals("t_pk_composite", "ORDER BY user_id, event_time") + + // FT-006: A string bucket key must use Paimon's string hash and preserve + // UTF-8 values while routing rows to fixed buckets. + sql """INSERT INTO t_pk_string_bucket VALUES + ('alpha', 1, 'alpha_v1'), + ('beta', 2, 'δΈ­ζ–‡_payload'), + ('emoji_πŸ˜€', 3, 'emoji_payload'), + ('alpha', 1, 'alpha_v2') + """ + order_qt_pk_string_bucket """SELECT user_key, event_id, payload + FROM t_pk_string_bucket ORDER BY event_id""" + assertTableEquals("t_pk_string_bucket", "ORDER BY event_id") + + // FT-045: The bucket-aware Exchange may use multiple writers, but every + // (partition, bucket) remains owned by exactly one writer. + sql """SET parallel_pipeline_task_num = 4""" + sql """SET enable_strict_consistency_dml = false""" + qt_pk_writer_scaling_plan """EXPLAIN SHAPE PLAN + INSERT INTO t_pk_writer_scaling + SELECT 1, number, repeat('x', 4096) + FROM numbers("number" = "10000")""" + sql """INSERT INTO t_pk_writer_scaling + SELECT 1, number, repeat('x', 4096) + FROM numbers("number" = "10000") + ORDER BY number""" + qt_pk_writer_scaling """SELECT COUNT(*), MIN(id), MAX(id), + MIN(LENGTH(payload)), MAX(LENGTH(payload)) FROM t_pk_writer_scaling""" + assertTableEquals("t_pk_writer_scaling", "ORDER BY id") + sql """SET parallel_pipeline_task_num = 0""" + sql """SET enable_strict_consistency_dml = true""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_level_dml.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_level_dml.groovy new file mode 100644 index 00000000000000..ec6f1304ef63d6 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_level_dml.groovy @@ -0,0 +1,663 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_row_level_dml", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_row_dml_catalog" + String dbName = "test_pw_row_dml_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_dml; + CREATE TABLE paimon.${dbName}.t_dml ( + id INT, name STRING, score INT, status STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_only; + CREATE TABLE paimon.${dbName}.t_append_only ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_first_row; + CREATE TABLE paimon.${dbName}.t_first_row ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'first-row' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_ignore_delete; + CREATE TABLE paimon.${dbName}.t_ignore_delete ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'ignore-delete' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_partial_update; + CREATE TABLE paimon.${dbName}.t_partial_update ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'partial-update.remove-record-on-delete' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_partial_update_sequence_group; + CREATE TABLE paimon.${dbName}.t_partial_update_sequence_group ( + id INT, name STRING, seq INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'fields.seq.sequence-group' = 'name', + 'partial-update.remove-record-on-sequence-group' = 'seq' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_aggregation_no_delete; + CREATE TABLE paimon.${dbName}.t_aggregation_no_delete ( + id INT, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.score.aggregate-function' = 'sum' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_aggregation_delete; + CREATE TABLE paimon.${dbName}.t_aggregation_delete ( + id INT, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.score.aggregate-function' = 'sum', + 'aggregation.remove-record-on-delete' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_sequence_ascending; + CREATE TABLE paimon.${dbName}.t_sequence_ascending ( + id INT, seq1 INT, seq2 INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'sequence.field' = 'seq1,seq2', + 'sequence.field.sort-order' = 'ascending' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_sequence_descending; + CREATE TABLE paimon.${dbName}.t_sequence_descending ( + id INT, seq INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'sequence.field' = 'seq', + 'sequence.field.sort-order' = 'descending' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_rowkind_field; + CREATE TABLE paimon.${dbName}.t_rowkind_field ( + id INT, row_kind STRING, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'rowkind.field' = 'row_kind' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_input_changelog; + CREATE TABLE paimon.${dbName}.t_input_changelog ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'changelog-producer' = 'input' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_cross_partition_fixed; + CREATE TABLE paimon.${dbName}.t_cross_partition_fixed ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_cross_partition_ttl; + CREATE TABLE paimon.${dbName}.t_cross_partition_ttl ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'cross-partition-upsert.index-ttl' = '1 h' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_cross_partition_ignore_delete; + CREATE TABLE paimon.${dbName}.t_cross_partition_ignore_delete ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'ignore-delete' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_same_name; + CREATE TABLE paimon.${dbName}.t_same_name ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_binary; + CREATE TABLE paimon.${dbName}.t_binary ( + id INT, payload BINARY + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'enable.mapping.varbinary' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """create database if not exists internal.${dbName}""" + sql """drop table if exists internal.${dbName}.t_merge_source""" + sql """ + CREATE TABLE internal.${dbName}.t_merge_source ( + id INT, name STRING, score INT, action STRING + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (1, 'Alice_merged', 100, 'U'), + (2, 'ignored', 0, 'D'), + (5, 'Eve', 50, 'I') + """ + sql """drop table if exists internal.${dbName}.t_delete_source""" + sql """ + CREATE TABLE internal.${dbName}.t_delete_source ( + id INT + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + sql """ + INSERT INTO internal.${dbName}.t_delete_source + SELECT 10 FROM numbers("number" = "1024") + """ + sql """drop table if exists internal.${dbName}.t_same_name""" + sql """ + CREATE TABLE internal.${dbName}.t_same_name ( + id INT, name STRING + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1') + """ + + try { + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + assertEquals(1, rows.size()) + assertTrue(rows[0][0] != null) + return rows[0][0].toString() + } + + def incrementalAuditLog = { tableName, columns, beforeSnapshot, afterSnapshot -> + spark_paimon """ + SELECT ${columns} + FROM paimon_incremental_query( + 'paimon.${dbName}.`${tableName}\$audit_log`', + '${beforeSnapshot}', + '${afterSnapshot}' + ) + ORDER BY id, rowkind + """ + } + + sql """INSERT INTO t_dml VALUES + (1, 'Alice', 10, 'active'), + (2, 'Bob', 20, 'active'), + (3, 'Charlie', 30, 'active'), + (4, 'Diana', 40, 'active') + """ + sql """INSERT INTO t_sequence_ascending VALUES (1, 10, 20, 'ascending')""" + sql """INSERT INTO t_sequence_descending VALUES (1, 10, 'descending')""" + + test { + sql """UPDATE t_sequence_ascending SET seq2 = 30 WHERE id = 1""" + exception "Paimon UPDATE cannot modify sequence-field column 'seq2'" + } + test { + sql """ + MERGE INTO t_sequence_descending t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET seq = s.score, name = s.name + """ + exception "Paimon UPDATE cannot modify sequence-field column 'seq'" + } + test { + sql """UPDATE t_rowkind_field SET name = 'updated' WHERE id = 1""" + exception "Paimon UPDATE is not supported when rowkind.field is configured" + } + test { + sql """DELETE FROM t_rowkind_field WHERE id = 1""" + exception "Paimon DELETE is not supported when rowkind.field is configured" + } + test { + sql """ + MERGE INTO t_rowkind_field t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN DELETE + """ + exception "Paimon MERGE is not supported when rowkind.field is configured" + } + test { + sql """UPDATE t_input_changelog SET name = 'updated' WHERE id = 1""" + exception "Paimon UPDATE is not supported when changelog-producer=input" + } + test { + sql """ + MERGE INTO t_input_changelog t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = s.name + """ + exception "Paimon UPDATE is not supported when changelog-producer=input" + } + test { + sql """UPDATE t_cross_partition_fixed SET pt = 'new_pt' WHERE id = 1""" + exception "Paimon UPDATE cannot modify partition column 'pt' unless bucket=-1" + } + test { + sql """ + MERGE INTO t_cross_partition_fixed t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET pt = 'new_pt', name = s.name + """ + exception "Paimon UPDATE cannot modify partition column 'pt' unless bucket=-1" + } + test { + sql """UPDATE t_cross_partition_ttl SET name = 'updated' WHERE id = 1""" + exception "cross-partition-upsert.index-ttl is configured" + } + test { + sql """DELETE FROM t_cross_partition_ttl WHERE id = 1""" + exception "cross-partition-upsert.index-ttl is configured" + } + test { + sql """ + MERGE INTO t_cross_partition_ttl t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN DELETE + """ + exception "cross-partition-upsert.index-ttl is configured" + } + test { + sql """ + UPDATE t_cross_partition_ignore_delete + SET pt = 'new_pt' WHERE id = 1 + """ + exception "cannot modify partition column 'pt' when ignore-delete=true" + } + test { + sql """ + MERGE INTO t_cross_partition_ignore_delete t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET pt = 'new_pt', name = s.name + """ + exception "cannot modify partition column 'pt' when ignore-delete=true" + } + + sql """INSERT INTO t_input_changelog VALUES (10, 'delete-me')""" + String deleteInputBefore = latestSnapshotId("t_input_changelog") + sql """ + DELETE FROM t_input_changelog t + USING internal.${dbName}.t_delete_source s + WHERE t.id = s.id + """ + String deleteInputAfter = latestSnapshotId("t_input_changelog") + assertEquals([ + ["-D", 10, "delete-me"] + ], incrementalAuditLog("t_input_changelog", "rowkind, id, name", + deleteInputBefore, deleteInputAfter)) + assertEquals([[0L]], sql("SELECT count(*) FROM t_input_changelog")) + + sql """INSERT INTO t_binary VALUES (1, CAST('delete-me' AS VARBINARY))""" + sql """DELETE FROM t_binary WHERE id = 1""" + assertEquals([[0L]], sql("SELECT count(*) FROM t_binary")) + + sql """INSERT INTO t_same_name VALUES (1, 'old')""" + sql """INSERT INTO internal.${dbName}.t_same_name VALUES (1, 'from-update')""" + sql """ + UPDATE ${catalogName}.${dbName}.t_same_name + SET name = internal.${dbName}.t_same_name.name + FROM internal.${dbName}.t_same_name + WHERE ${catalogName}.${dbName}.t_same_name.id + = internal.${dbName}.t_same_name.id + """ + assertEquals([[1, "from-update"]], sql("SELECT * FROM t_same_name")) + sql """TRUNCATE TABLE internal.${dbName}.t_same_name""" + sql """INSERT INTO internal.${dbName}.t_same_name VALUES (1, 'from-merge')""" + sql """ + MERGE INTO t_same_name + USING internal.${dbName}.t_same_name + ON ${catalogName}.${dbName}.t_same_name.id + = internal.${dbName}.t_same_name.id + WHEN MATCHED THEN UPDATE + SET name = internal.${dbName}.t_same_name.name + """ + assertEquals([[1, "from-merge"]], sql("SELECT * FROM t_same_name")) + sql """ + DELETE FROM ${catalogName}.${dbName}.t_same_name + USING internal.${dbName}.t_same_name + WHERE ${catalogName}.${dbName}.t_same_name.id + = internal.${dbName}.t_same_name.id + """ + assertEquals([[0L]], sql("SELECT count(*) FROM t_same_name")) + + sql """ + UPDATE t_dml + SET name = concat(name, '_updated'), score = score + 1 + WHERE id IN (1, 2) + """ + order_qt_paimon_update """SELECT * FROM t_dml ORDER BY id""" + + sql """DELETE FROM t_dml WHERE id = 3""" + order_qt_paimon_delete """SELECT * FROM t_dml ORDER BY id""" + + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = s.id + WHEN MATCHED AND s.action = 'D' THEN DELETE + WHEN MATCHED THEN UPDATE SET + name = s.name, + score = s.score, + status = 'merged' + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'inserted') + """ + order_qt_paimon_merge """SELECT * FROM t_dml ORDER BY id""" + + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (1, 'priority_update', 101, 'U'), + (6, 'priority_insert', 60, 'I') + """ + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = s.id + WHEN MATCHED AND s.action = 'U' THEN UPDATE SET + name = s.name, score = s.score, status = 'first-matched' + WHEN MATCHED THEN DELETE + WHEN NOT MATCHED AND s.action = 'I' THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'first-not-matched') + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'fallback') + """ + order_qt_paimon_merge_branch_priority """SELECT * FROM t_dml ORDER BY id""" + + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (1, 'duplicate_1', 201, 'U'), + (1, 'duplicate_2', 202, 'U') + """ + test { + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = s.name, score = s.score + """ + exception "Paimon MERGE matched one target row with multiple source rows" + } + order_qt_paimon_merge_duplicate_unchanged """SELECT * FROM t_dml ORDER BY id""" + + test { + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id + 1, s.name, s.score, 'invalid-insert-key') + """ + exception "each INSERT to use the corresponding deterministic source expression" + } + test { + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = s.id AND t.status = 'not-present' + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'invalid-extra-predicate') + """ + exception "Paimon MERGE with NOT MATCHED INSERT requires ON to contain only equality predicates" + } + + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (1000, 'duplicate_insert_1', 301, 'I'), + (1000, 'duplicate_insert_2', 302, 'I') + """ + test { + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = s.id + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'duplicate-insert') + """ + exception "Paimon MERGE attempted to insert multiple rows with the same primary key" + } + order_qt_paimon_merge_duplicate_insert_unchanged """SELECT * FROM t_dml ORDER BY id""" + + test { + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s + ON t.id = CAST(RAND() * 1000000 AS INT) + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (CAST(RAND() * 1000000 AS INT), s.name, s.score, 'random-key') + """ + exception "each INSERT to use the corresponding deterministic source expression" + } + + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """ + INSERT INTO internal.${dbName}.t_merge_source + SELECT CAST(number + 2000 AS INT), CONCAT('unmatched_', number), + CAST(number AS INT), 'I' + FROM numbers("number" = "1024") + """ + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'many-unmatched') + """ + qt_paimon_merge_many_unmatched """SELECT count(*) FROM t_dml WHERE id >= 2000""" + + test { + sql """UPDATE t_append_only SET name = 'x' WHERE id = 1""" + exception "Paimon UPDATE requires a primary-key table" + } + test { + sql """DELETE FROM t_append_only WHERE id = 1""" + exception "Paimon DELETE requires a primary-key table" + } + test { + sql """UPDATE t_first_row SET name = 'x' WHERE id = 1""" + exception "Paimon UPDATE only supports merge-engine=deduplicate" + } + test { + sql """ + MERGE INTO t_append_only t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN DELETE + """ + exception "Paimon MERGE requires a primary-key table" + } + + sql """INSERT INTO t_ignore_delete VALUES (10, 'keep')""" + test { + sql """DELETE FROM t_ignore_delete WHERE id = 10""" + exception "Paimon DELETE is not supported when ignore-delete=true" + } + test { + sql """ + MERGE INTO t_ignore_delete t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN DELETE + """ + exception "Paimon DELETE is not supported when ignore-delete=true" + } + order_qt_paimon_ignore_delete_unchanged """SELECT * FROM t_ignore_delete ORDER BY id""" + + sql """INSERT INTO t_partial_update VALUES (20, 'partial', 20)""" + test { + sql """UPDATE t_partial_update SET name = NULL WHERE id = 20""" + exception "Paimon UPDATE only supports merge-engine=deduplicate" + } + test { + sql """ + MERGE INTO t_partial_update t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = NULL + """ + exception "Paimon UPDATE only supports merge-engine=deduplicate" + } + sql """DELETE FROM t_partial_update WHERE id = 20""" + qt_paimon_partial_update_delete """SELECT count(*) FROM t_partial_update""" + + sql """INSERT INTO t_partial_update_sequence_group VALUES (21, 'keep', NULL)""" + test { + sql """DELETE FROM t_partial_update_sequence_group WHERE id = 21""" + exception "partial-update.remove-record-on-delete=true" + } + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (21, 'keep', 0, 'D') + """ + test { + sql """ + MERGE INTO t_partial_update_sequence_group t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN DELETE + """ + exception "partial-update.remove-record-on-delete=true" + } + + sql """INSERT INTO t_aggregation_no_delete VALUES (30, 30)""" + test { + sql """DELETE FROM t_aggregation_no_delete WHERE id = 30""" + exception "Paimon DELETE does not support merge-engine=aggregation" + } + order_qt_paimon_aggregation_no_delete_unchanged """ + SELECT * FROM t_aggregation_no_delete ORDER BY id + """ + + sql """INSERT INTO t_aggregation_delete VALUES (40, 40)""" + sql """DELETE FROM t_aggregation_delete WHERE id = 40""" + qt_paimon_aggregation_delete """SELECT count(*) FROM t_aggregation_delete""" + + test { + sql """ + MERGE INTO t_first_row t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = s.name + """ + exception "Paimon UPDATE only supports merge-engine=deduplicate" + } + + sql """set short_circuit_evaluation = false""" + sql """TRUNCATE TABLE internal.${dbName}.t_merge_source""" + sql """INSERT INTO internal.${dbName}.t_merge_source VALUES + (1, 'short_update', 901, 'U'), + (7000, 'short_insert', 902, 'I') + """ + sql """ + MERGE INTO t_dml t + USING internal.${dbName}.t_merge_source s ON t.id = s.id + WHEN MATCHED AND s.id <=> 1 THEN UPDATE SET + name = s.name, + score = IF(assert_true(s.id = 1, 'inactive assignment evaluated'), s.score, -1) + WHEN MATCHED AND assert_true(s.id < 0, 'later predicate evaluated') THEN DELETE + WHEN NOT MATCHED THEN INSERT (id, name, score, status) + VALUES (s.id, s.name, s.score, 'short-circuit') + """ + assertEquals([["short_update"], ["short_insert"]], + sql("SELECT name FROM t_dml WHERE id IN (1, 7000) ORDER BY id")) + } finally { + sql """drop catalog if exists ${catalogName}""" + sql """drop table if exists internal.${dbName}.t_merge_source""" + sql """drop table if exists internal.${dbName}.t_delete_source""" + sql """drop table if exists internal.${dbName}.t_same_name""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.groovy new file mode 100644 index 00000000000000..cdea849f37aff0 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.groovy @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_row_tracking_evolution", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_row_tracking_catalog" + String dbName = "test_pw_row_tracking_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_tracking; + CREATE TABLE paimon.${dbName}.t_tracking ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'bucket' = '-1', + 'row-tracking.enabled' = 'true', + 'compaction.min.file-num' = '2' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_evolution; + CREATE TABLE paimon.${dbName}.t_evolution ( + id INT, b INT, c INT + ) USING paimon + TBLPROPERTIES ( + 'bucket' = '-1', + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_evolution_source; + CREATE TABLE paimon.${dbName}.t_evolution_source ( + id INT, b INT, c INT + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def trackingRows = { String tableName -> + return sql(""" + SELECT id, _ROW_ID, _SEQUENCE_NUMBER + FROM `${tableName}\$row_tracking` + ORDER BY id + """) + } + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + return rows[0][0] == null ? 0L : rows[0][0].toString().toLong() + } + + // Doris assigns row ids through the Paimon committer. Their exact + // values are not assumed, but they must be unique and stable. + sql """INSERT INTO t_tracking VALUES + (1, 'one'), (2, 'two'), (3, 'three') + """ + def initialTracking = trackingRows("t_tracking") + assertEquals(3, initialTracking.size()) + assertEquals(3, initialTracking.collect { it[1] }.toSet().size()) + Map initialRowIds = initialTracking.collectEntries { row -> + [(row[0].toString().toInteger()): row[1].toString().toLong()] + } + Map initialSequences = initialTracking.collectEntries { row -> + [(row[0].toString().toInteger()): row[2].toString().toLong()] + } + + // Spark performs row-level changes which Doris does not expose for an + // append table. This verifies that rows originally written by Doris have + // valid tracking metadata for every upstream operation. + spark_paimon_multi """ + UPDATE paimon.${dbName}.t_tracking + SET payload = 'two-updated' WHERE id = 2; + DELETE FROM paimon.${dbName}.t_tracking WHERE id = 3; + MERGE INTO paimon.${dbName}.t_tracking t + USING (SELECT 1 AS id, 'one-merged' AS payload + UNION ALL + SELECT 4 AS id, 'four' AS payload) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET payload = s.payload + WHEN NOT MATCHED THEN INSERT (id, payload) VALUES (s.id, s.payload); + """ + sql """refresh table t_tracking""" + order_qt_row_tracking_after_spark_changes """ + SELECT id, payload FROM t_tracking ORDER BY id + """ + def changedTracking = trackingRows("t_tracking") + Map changedRowIds = changedTracking.collectEntries { row -> + [(row[0].toString().toInteger()): row[1].toString().toLong()] + } + Map changedSequences = changedTracking.collectEntries { row -> + [(row[0].toString().toInteger()): row[2].toString().toLong()] + } + assertEquals(initialRowIds[1], changedRowIds[1]) + assertEquals(initialRowIds[2], changedRowIds[2]) + assertFalse(initialRowIds.values().contains(changedRowIds[4])) + assertTrue(changedSequences[1] > initialSequences[1]) + assertTrue(changedSequences[2] > initialSequences[2]) + + spark_paimon """ + CALL paimon.sys.compact( + table => '${dbName}.t_tracking', + compact_strategy => 'full') + """ + sql """refresh table t_tracking""" + def compactedTracking = trackingRows("t_tracking") + Map compactedRowIds = compactedTracking.collectEntries { row -> + [(row[0].toString().toInteger()): row[1].toString().toLong()] + } + assertEquals(changedRowIds, compactedRowIds) + + sql """INSERT INTO t_tracking VALUES (5, 'five-after-compact')""" + order_qt_row_tracking_after_compact_write """ + SELECT id, payload FROM t_tracking ORDER BY id + """ + def afterDorisReopen = trackingRows("t_tracking") + assertEquals(4, afterDorisReopen.collect { it[1] }.toSet().size()) + assertFalse(compactedRowIds.values().contains( + afterDorisReopen.find { it[0].toString().toInteger() == 5 }[1] + .toString().toLong())) + + // Data evolution accepts Doris full and partial INSERTs. Spark MERGE + // then updates only selected columns and keeps the original row ids. + sql """INSERT INTO t_evolution VALUES (1, 10, 100), (2, 20, 200)""" + sql """INSERT INTO t_evolution (id, b) VALUES (3, 30)""" + def evolutionBefore = trackingRows("t_evolution") + Map evolutionRowIds = evolutionBefore.collectEntries { row -> + [(row[0].toString().toInteger()): row[1].toString().toLong()] + } + spark_paimon_multi """ + INSERT INTO paimon.${dbName}.t_evolution_source VALUES + (1, 11, 111), (2, 22, 222), (4, 44, 444); + MERGE INTO paimon.${dbName}.t_evolution t + USING paimon.${dbName}.t_evolution_source s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET b = s.b + WHEN NOT MATCHED THEN INSERT (id, b, c) + VALUES (s.id, s.b, s.c); + """ + sql """refresh table t_evolution""" + order_qt_data_evolution_after_spark_merge """ + SELECT * FROM t_evolution ORDER BY id + """ + def evolutionAfter = trackingRows("t_evolution") + Map evolutionAfterIds = evolutionAfter.collectEntries { row -> + [(row[0].toString().toInteger()): row[1].toString().toLong()] + } + assertEquals(evolutionRowIds[1], evolutionAfterIds[1]) + assertEquals(evolutionRowIds[2], evolutionAfterIds[2]) + assertEquals(evolutionRowIds[3], evolutionAfterIds[3]) + assertFalse(evolutionRowIds.values().contains(evolutionAfterIds[4])) + + // Paimon 1.4.2 does not support ordinary UPDATE/DELETE on a data + // evolution table. Doris currently rejects them at its append-table + // boundary; either way no Paimon snapshot may be committed. + long evolutionSnapshot = latestSnapshotId("t_evolution") + test { + sql """UPDATE t_evolution SET b = 999 WHERE id = 1""" + exception "primary-key table" + } + assertEquals(evolutionSnapshot, latestSnapshotId("t_evolution")) + test { + sql """DELETE FROM t_evolution WHERE id = 1""" + exception "primary-key table" + } + assertEquals(evolutionSnapshot, latestSnapshotId("t_evolution")) + test { + sql """ + MERGE INTO t_evolution t + USING (SELECT 1 AS id, 999 AS b) s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET b = s.b + """ + exception "primary-key table" + } + assertEquals(evolutionSnapshot, latestSnapshotId("t_evolution")) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy new file mode 100644 index 00000000000000..9882ec73560824 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy @@ -0,0 +1,1097 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_schema_change", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_schema_change_catalog" + String dbName = "test_pw_schema_change_db" + String appendTable = "t_schema_change_append" + String typeTable = "t_schema_change_types" + String explicitTypeTable = "t_schema_change_explicit_types" + String primaryKeyTable = "t_schema_change_pk" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + sql """SET show_column_comment_in_describe = true""" + + try { + def assertTableEquals = { String table, String columns, String orderBy -> + spark_paimon """ + REFRESH TABLE paimon.${dbName}.${table} + """ + def sparkRows = spark_paimon """ + SELECT ${columns} + FROM paimon.${dbName}.${table} + ${orderBy} + """ + def dorisRows = sql """ + SELECT ${columns} + FROM `${table}` + ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // ------------------------------------------------------------------ + // Append-only partitioned table: every supported column evolution is + // followed by reading historical rows and writing with the new schema. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${appendTable}` ( + id INT NULL, + required_value BIGINT NOT NULL, + name STRING NULL, + score INT NULL, + amount DECIMAL(8, 2) NULL, + obsolete STRING NULL, + dt STRING NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'disable-explicit-type-casting' = 'true' + ) + """ + + sql """ + INSERT INTO `${appendTable}` VALUES + (1, 100, 'alice', 10, 1.10, 'old-a', '2026-07-01'), + (2, 200, 'bob', 20, 2.20, 'old-b', '2026-07-02') + """ + order_qt_sc_append_initial """ + SELECT id, required_value, name, score, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, amount, obsolete, dt", + "ORDER BY id") + + // ADD COLUMN with DEFAULT, COMMENT and AFTER. Historical rows remain + // readable and explicit values can immediately be written. + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN added_after STRING NULL DEFAULT 'unknown' + COMMENT 'added after score' AFTER score + """ + order_qt_sc_add_after_before_insert """ + SELECT id, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (added_after, dt, id, obsolete, amount, name, required_value, score) + VALUES + ('added-3', '2026-07-03', 3, 'old-c', 3.30, 'carol', 300, 30), + (NULL, '2026-07-01', 4, 'old-d', 4.40, 'dave', 400, 40) + """ + order_qt_sc_add_after_after_insert """ + SELECT id, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, required_value, name, score, amount, obsolete, dt) + VALUES (100, 10000, 'default-value', 100, 100.00, 'old-default', '2026-07-10') + """ + order_qt_sc_add_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 100 + """ + + // ADD COLUMN FIRST. All historical rows expose NULL for the new column. + sql """ALTER TABLE `${appendTable}` ADD COLUMN first_col BIGINT NULL FIRST""" + order_qt_sc_add_first_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, first_col, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (first_col, id, required_value, name, score, added_after, amount, obsolete, dt) + VALUES + (5000, 5, 500, 'erin', 50, 'added-first', 5.50, 'old-e', '2026-07-04') + """ + order_qt_sc_add_first_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, first_col, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + // ADD COLUMNS validates that a batch of columns is visible atomically + // to both the reader and writer. + sql """ + ALTER TABLE `${appendTable}` ADD COLUMN ( + tiny_col TINYINT NULL, + small_col SMALLINT NULL COMMENT 'small integer' + ) + """ + order_qt_sc_add_columns_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (small_col, tiny_col, dt, id, first_col, required_value, + name, score, added_after, amount, obsolete) + VALUES + (600, 6, '2026-07-05', 6, 6000, 600, + 'frank', 60, 'added-columns', 6.60, 'old-f') + """ + // Omit the newly added nullable columns while keeping added_after + // explicit because its omitted-default behavior is tracked above. + sql """ + INSERT INTO `${appendTable}` + (id, required_value, name, score, added_after, amount, obsolete, dt) + VALUES + (60, 6000, 'partial-columns', 600, 'explicit-default-column', + 60.60, 'old-partial', '2026-07-05') + """ + order_qt_sc_add_columns_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + """, + "ORDER BY id") + + // DROP a populated non-key column. + sql """ALTER TABLE `${appendTable}` DROP COLUMN obsolete""" + order_qt_sc_drop_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (tiny_col, small_col, dt, id, first_col, required_value, + name, score, added_after, amount) + VALUES + (7, 700, '2026-07-06', 7, 7000, 700, + 'grace', 70, 'after-drop', 7.70) + """ + order_qt_sc_drop_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + // RENAME resolves the old name case-insensitively while preserving the + // Paimon field id and all historical values. + sql """ALTER TABLE `${appendTable}` RENAME COLUMN NAME full_name""" + order_qt_sc_rename_before_insert """ + SELECT id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (full_name, id, score, added_after, dt, amount, + required_value, first_col, tiny_col, small_col) + VALUES + ('heidi', 8, 80, 'after-rename', '2026-07-02', 8.80, + 800, 8000, 8, 800) + """ + order_qt_sc_rename_after_insert """ + SELECT id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + // MODIFY type: INT -> BIGINT. The new value exceeds the INT range. + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN score BIGINT NULL""" + order_qt_sc_modify_bigint_before_insert """ + SELECT id, full_name, score, amount, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, required_value, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (9, 9000, 900, 'ivan', CAST(3000000000 AS BIGINT), + 'after-bigint', 9.90, 9, 900, '2026-07-07') + """ + order_qt_sc_modify_bigint_after_insert """ + SELECT id, full_name, score, amount, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, required_value, dt", + "ORDER BY id") + + // MODIFY type: widen DECIMAL precision without changing the scale. + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN amount DECIMAL(12, 2) NULL""" + order_qt_sc_modify_decimal_before_insert """ + SELECT id, full_name, score, amount, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (10, 10000, 1000, 'judy', 100, + 'after-decimal', 1234567890.12, 10, 1000, '2026-07-08') + """ + order_qt_sc_modify_decimal_after_insert """ + SELECT id, full_name, score, amount, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, dt", + "ORDER BY id") + + // MODIFY nullability: NOT NULL -> NULL, then actually write NULL. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN required_value BIGINT NULL + """ + order_qt_sc_modify_nullable_before_insert """ + SELECT id, full_name, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, required_value, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (11, 11000, NULL, 'kate', 110, + 'after-nullable', 11.11, 11, 1100, '2026-07-09') + """ + order_qt_sc_modify_nullable_after_insert """ + SELECT id, full_name, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, required_value, dt", + "ORDER BY id") + + // MODIFY DEFAULT, COMMENT and position together. DESC checks metadata; + // data checks field-id preservation after moving the column. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN added_after STRING NULL DEFAULT 'changed-default' + COMMENT 'changed comment' FIRST + """ + qt_sc_modify_metadata_desc """DESC `${appendTable}`""" + order_qt_sc_modify_metadata_before_insert """ + SELECT id, full_name, added_after, score, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, added_after, score, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (added_after, id, first_col, required_value, full_name, + score, amount, tiny_col, small_col, dt) + VALUES + ('after-metadata', 12, 12000, 1200, 'leo', + 120, 12.12, 12, 1200, '2026-07-10') + """ + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, + score, amount, tiny_col, small_col, dt) + VALUES + (120, 120000, 12000, 'modified-default', + 1200, 120.00, 12, 1200, '2026-07-10') + """ + order_qt_sc_modify_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 120 + """ + order_qt_sc_modify_metadata_after_insert """ + SELECT id, full_name, added_after, score, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, added_after, score, dt", + "ORDER BY id") + + // Omitting DEFAULT and COMMENT removes both, while AFTER moves the same + // field again. Explicit writes must continue to map to the correct id. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN added_after STRING NULL AFTER score + """ + qt_sc_modify_remove_metadata_desc """DESC `${appendTable}`""" + order_qt_sc_modify_remove_metadata_before_insert """ + SELECT id, full_name, score, added_after, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, added_after, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (13, 13000, 1300, 'mallory', 130, + 'after-remove-metadata', 13.13, 13, 1300, '2026-07-11') + """ + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + amount, tiny_col, small_col, dt) + VALUES + (130, 130000, 13000, 'removed-default', 1300, + 130.00, 13, 1300, '2026-07-11') + """ + order_qt_sc_remove_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 130 + """ + order_qt_sc_modify_remove_metadata_after_insert """ + SELECT id, full_name, score, added_after, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, added_after, dt", + "ORDER BY id") + + // Reorder every column, then use INSERT VALUES without a target list. + // This catches stale FE and JNI writer physical-column ordering. + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + ) + """ + order_qt_sc_reorder_before_insert """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` VALUES + (14, 'nick', 140, 14.14, 1400, + 'after-reorder', 14000, 14, 1400, '2026-07-12') + """ + order_qt_sc_reorder_after_insert """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Failed schema changes must be atomic and must not poison a subsequent + // writer created from the still-current schema. + test { + sql """ + ALTER TABLE `${appendTable}` ADD COLUMN ( + batch_ok INT NULL, + batch_bad INT NOT NULL DEFAULT '1' + ) + """ + exception "cannot specify NOT NULL" + } + test { + sql """ALTER TABLE `${appendTable}` ADD COLUMN ID INT NULL""" + exception "conflicts with an existing Paimon column" + } + test { + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN bad_position INT NULL AFTER missing_col + """ + exception "does not exist in Paimon table" + } + test { + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN multi_a INT NULL, + ADD COLUMN multi_b INT NULL + """ + exception "External table does not support multiple ALTER clauses" + } + test { + sql """ALTER TABLE `${appendTable}` DROP COLUMN missing_col""" + exception "does not exist in Paimon table" + } + test { + sql """ + ALTER TABLE `${appendTable}` + RENAME COLUMN full_name id + """ + exception "conflicts with an existing Paimon column" + } + test { + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN score INT NULL""" + exception "cannot be converted" + } + test { + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN required_value BIGINT NOT NULL + """ + exception "nullable to non nullable" + } + test { + sql """ALTER TABLE `${appendTable}` DROP COLUMN dt""" + exception "Cannot drop partition key or primary key" + } + test { + sql """ + ALTER TABLE `${appendTable}` + RENAME COLUMN dt partition_col + """ + exception "Cannot rename partition column" + } + test { + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN dt INT NULL + """ + exception "Cannot update partition column" + } + test { + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score + ) + """ + exception "must contain every Paimon column exactly once" + } + test { + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, id + ) + """ + exception "Duplicate column in reorder columns" + } + + sql """ + INSERT INTO `${appendTable}` VALUES + (15, 'olivia', 150, 15.15, NULL, + 'after-failed-alters', 15000, 15, 1500, '2026-07-13') + """ + order_qt_sc_after_failed_alters """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Paimon 1.4.2 has no partition-key evolution in SchemaChange. + // Doris therefore rejects ADD, DROP and REPLACE before catalog mutation. + test { + sql """ + ALTER TABLE `${appendTable}` + ADD PARTITION KEY bucket(4, id) AS id_bucket + """ + exception "ADD PARTITION KEY is only supported for Iceberg tables" + } + test { + sql """ALTER TABLE `${appendTable}` DROP PARTITION KEY dt""" + exception "DROP PARTITION KEY is only supported for Iceberg tables" + } + test { + sql """ + ALTER TABLE `${appendTable}` + REPLACE PARTITION KEY dt WITH bucket(4, id) AS id_bucket + """ + exception "REPLACE PARTITION KEY is only supported for Iceberg tables" + } + + // Rejected partition evolution must not mutate the current schema or + // prevent a new writer from committing another physical partition. + sql """ + INSERT INTO `${appendTable}` VALUES + (16, 'peggy', 160, 16.16, 1600, + 'after-partition-evolution-failures', 16000, 16, 1600, '2026-07-14') + """ + order_qt_sc_after_partition_evolution_failures """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Regular writes create new partition values before and after schema + // evolution. This is dynamic partition creation, not partition evolution. + def sparkPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`${appendTable}\$partitions` + ORDER BY `partition` + """ + def dorisPartitions = sql """ + SELECT `partition`, record_count + FROM `${appendTable}\$partitions` + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + order_qt_sc_append_partitions """ + SELECT `partition`, record_count + FROM `${appendTable}\$partitions` + ORDER BY `partition` + """ + + // ------------------------------------------------------------------ + // Type-evolution matrix: verify widening conversions on existing data, + // then write values which cannot fit in the original types. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${typeTable}` ( + id INT NULL, + c_tiny TINYINT NULL, + c_small SMALLINT NULL, + c_int INT NULL, + c_float FLOAT NULL, + c_decimal DECIMAL(8, 2) NULL + ) ENGINE=paimon + PROPERTIES ( + 'disable-explicit-type-casting' = 'true' + ) + """ + sql """ + INSERT INTO `${typeTable}` VALUES + (1, 100, 30000, 2000000000, 1.5, 123456.78) + """ + order_qt_sc_types_initial """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_tiny SMALLINT NULL""" + order_qt_sc_types_tiny_to_small_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (2, 200, 30001, 2000000001, 2.5, 123456.79) + """ + order_qt_sc_types_tiny_to_small_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_small INT NULL""" + order_qt_sc_types_small_to_int_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (3, 201, 40000, 2000000002, 3.5, 123456.80) + """ + order_qt_sc_types_small_to_int_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_int BIGINT NULL""" + order_qt_sc_types_int_to_bigint_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (4, 202, 40001, 3000000000, 4.5, 123456.81) + """ + order_qt_sc_types_int_to_bigint_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_float DOUBLE NULL""" + order_qt_sc_types_float_to_double_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (5, 203, 40002, 3000000001, 1.0E40, 123456.82) + """ + order_qt_sc_types_float_to_double_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${typeTable}` + MODIFY COLUMN c_decimal DECIMAL(12, 2) NULL + """ + order_qt_sc_types_decimal_widen_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (6, 204, 40003, 3000000002, 2.0E40, 1234567890.12) + """ + order_qt_sc_types_decimal_widen_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + // A narrowing conversion fails with data present; the old writer schema + // remains usable after the failed ALTER. + test { + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_int INT NULL""" + exception "cannot be converted" + } + sql """ + INSERT INTO `${typeTable}` VALUES + (7, 205, 40004, 3000000003, 3.0E40, 1234567890.13) + """ + order_qt_sc_types_after_failed_narrow """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + // By default Paimon also permits explicit casts. Use values which fit + // in the target type to cover BIGINT -> INT and then INT -> STRING. + sql """ + CREATE TABLE `${explicitTypeTable}` ( + id INT NULL, + explicit_value BIGINT NULL + ) ENGINE=paimon + """ + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (1, 100), + (2, 200) + """ + order_qt_sc_explicit_types_initial """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${explicitTypeTable}` + MODIFY COLUMN explicit_value INT NULL + """ + order_qt_sc_explicit_bigint_to_int_before_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (3, 300) + """ + order_qt_sc_explicit_bigint_to_int_after_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${explicitTypeTable}` + MODIFY COLUMN explicit_value STRING NULL + """ + order_qt_sc_explicit_int_to_string_before_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (4, 'after-explicit-cast') + """ + order_qt_sc_explicit_int_to_string_after_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + // ------------------------------------------------------------------ + // Primary-key table: repeat the core evolutions around merge-tree data + // and verify key/partition-column restrictions do not affect later writes. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${primaryKeyTable}` ( + id INT NOT NULL, + dt STRING NOT NULL, + metric INT NULL, + legacy STRING NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'primary-key' = 'id,dt', + 'bucket' = '2', + 'merge-engine' = 'partial-update' + ) + """ + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, '2026-08-01', 10, 'pk-a'), + (2, '2026-08-01', 20, 'pk-b') + """ + order_qt_sc_pk_initial """ + SELECT * FROM `${primaryKeyTable}` ORDER BY dt, id + """ + assertTableEquals(primaryKeyTable, "*", "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + ADD COLUMN note STRING NULL DEFAULT 'default-note' AFTER metric + """ + order_qt_sc_pk_add_before_insert """ + SELECT id, dt, metric, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, '2026-08-01', 11, 'updated-after-add', 'pk-a2'), + (3, '2026-08-02', 30, 'new-after-add', 'pk-c') + """ + order_qt_sc_pk_add_after_insert """ + SELECT id, dt, metric, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric, note, legacy", + "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + RENAME COLUMN metric metric_value + """ + order_qt_sc_pk_rename_before_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note, legacy) + VALUES + (2, '2026-08-01', 22, 'updated-after-rename', 'pk-b2'), + (4, '2026-08-02', 40, 'new-after-rename', 'pk-d') + """ + order_qt_sc_pk_rename_after_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + MODIFY COLUMN metric_value BIGINT NULL + """ + order_qt_sc_pk_type_before_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note, legacy) + VALUES + (5, '2026-08-03', 3000000000, 'new-after-type', 'pk-e') + """ + order_qt_sc_pk_type_after_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ALTER TABLE `${primaryKeyTable}` DROP COLUMN legacy""" + order_qt_sc_pk_drop_before_insert """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + // A PK partial-update writer also distinguishes omitted fields from + // explicit NULL using the evolved remote schema. A later partial row + // which omits note applies its schema default again. + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value) + VALUES + (8, '2026-08-05', 80) + """ + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, note) + VALUES + (8, '2026-08-05', 'explicit-note'), + (9, '2026-08-05', NULL) + """ + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value) + VALUES + (8, '2026-08-05', 81) + """ + order_qt_sc_pk_partial_default """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + WHERE id IN (8, 9) + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (note, metric_value, dt, id) + VALUES + ('new-after-drop', 60, '2026-08-03', 6) + """ + order_qt_sc_pk_drop_after_insert """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + // Primary and partition keys cannot be dropped, renamed or have their + // types changed. All failures must leave the merge-tree writer usable. + test { + sql """ALTER TABLE `${primaryKeyTable}` DROP COLUMN id""" + exception "Cannot drop partition key or primary key" + } + test { + sql """ + ALTER TABLE `${primaryKeyTable}` + RENAME COLUMN dt partition_col + """ + exception "Cannot rename partition column" + } + test { + sql """ + ALTER TABLE `${primaryKeyTable}` + MODIFY COLUMN id BIGINT NOT NULL + """ + exception "Cannot update primary key" + } + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note) + VALUES + (7, '2026-08-04', 70, 'after-key-failures') + """ + order_qt_sc_pk_after_key_failures """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + } finally { + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + sql """DROP TABLE IF EXISTS `${explicitTypeTable}`""" + sql """DROP TABLE IF EXISTS `${typeTable}`""" + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_group.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_group.groovy new file mode 100644 index 00000000000000..1da8ddca25af58 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_group.groovy @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_sequence_group", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_sequence_group_catalog" + String dbName = "test_pw_sequence_group_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi_group; + CREATE TABLE paimon.${dbName}.t_multi_group ( + id INT, + profile_name STRING, + profile_city STRING, + profile_seq INT, + total BIGINT, + peak INT, + metric_seq INT, + note STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'fields.profile_seq.sequence-group' = 'profile_name,profile_city', + 'fields.metric_seq.sequence-group' = 'total,peak', + 'fields.total.aggregate-function' = 'sum', + 'fields.peak.aggregate-function' = 'max', + 'fields.note.aggregate-function' = 'last_non_null_value' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_remove_on_delete; + CREATE TABLE paimon.${dbName}.t_remove_on_delete ( + id INT, a STRING, b STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'partial-update.remove-record-on-delete' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_group_remove_on_delete; + CREATE TABLE paimon.${dbName}.t_group_remove_on_delete ( + id INT, + a STRING, + seq_a INT, + b STRING, + seq_b INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'fields.seq_a.sequence-group' = 'a', + 'fields.seq_b.sequence-group' = 'b', + 'partial-update.remove-record-on-sequence-group' = 'seq_a' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_property_change; + CREATE TABLE paimon.${dbName}.t_property_change ( + id INT, a STRING, seq INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertSparkEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """ + SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // P01: Each sequence group advances independently. Aggregations only + // consume a row when the sequence of their own group is accepted. + sql """INSERT INTO t_multi_group VALUES + (1, 'alice', 'shanghai', 10, 5, 80, 10, 'base') + """ + sql """INSERT INTO t_multi_group VALUES + (1, 'stale-profile', 'beijing', 9, 7, 90, 11, NULL) + """ + order_qt_sequence_group_stale_profile """ + SELECT * FROM t_multi_group ORDER BY id + """ + + sql """INSERT INTO t_multi_group VALUES + (1, 'alice-new', 'shenzhen', 12, 100, 99, 10, 'new-note') + """ + order_qt_sequence_group_new_profile """ + SELECT * FROM t_multi_group ORDER BY id + """ + + // A NULL sequence does not advance its group. A different group in the + // same row can still advance and apply its aggregate functions. + sql """INSERT INTO t_multi_group VALUES + (1, 'null-sequence', 'hangzhou', NULL, 3, 88, 13, NULL) + """ + order_qt_sequence_group_null_sequence """ + SELECT * FROM t_multi_group ORDER BY id + """ + assertSparkEquals("t_multi_group", "*", "ORDER BY id") + + // P02: remove-record-on-delete must discard the old partial row. A + // later partial insert creates a new row and must not revive old fields. + sql """INSERT INTO t_remove_on_delete VALUES (1, 'old-a', 'old-b')""" + sql """DELETE FROM t_remove_on_delete WHERE id = 1""" + qt_remove_on_delete_empty """SELECT count(*) FROM t_remove_on_delete""" + sql """INSERT INTO t_remove_on_delete (id, b) VALUES (1, 'new-b')""" + order_qt_remove_on_delete_partial """ + SELECT id, a, b FROM t_remove_on_delete ORDER BY id + """ + sql """INSERT INTO t_remove_on_delete (id, a) VALUES (1, 'new-a')""" + order_qt_remove_on_delete_complete """ + SELECT id, a, b FROM t_remove_on_delete ORDER BY id + """ + assertSparkEquals("t_remove_on_delete", "*", "ORDER BY id") + + // Paimon makes remove-record-on-delete and sequence groups mutually + // exclusive. Doris must reject a whole-row DELETE on this legal + // sequence-group configuration without changing its accumulated row. + sql """INSERT INTO t_group_remove_on_delete VALUES + (1, 'old-a', 100, 'old-b', 100) + """ + test { + sql """DELETE FROM t_group_remove_on_delete WHERE id = 1""" + exception "partial-update.remove-record-on-delete=true" + } + order_qt_group_remove_on_delete_unchanged """ + SELECT * FROM t_group_remove_on_delete ORDER BY id + """ + sql """INSERT INTO t_group_remove_on_delete VALUES + (1, 'low-a', 1, 'low-b', 1) + """ + sql """INSERT INTO t_group_remove_on_delete (id, a, seq_a) VALUES + (1, 'high-a', 101) + """ + order_qt_group_remove_on_delete_sequence """ + SELECT * FROM t_group_remove_on_delete ORDER BY id + """ + assertSparkEquals("t_group_remove_on_delete", "*", "ORDER BY id") + + // P03: Invalid writer properties fail as metadata changes and must not + // poison the last valid schema. A legal sequence group takes effect for + // the next writer without recreating the catalog. + sql """INSERT INTO t_property_change VALUES (1, 'base', 10)""" + long propertySnapshot = (sql """ + SELECT max(snapshot_id) FROM t_property_change\$snapshots + """)[0][0] as long + String invalidPropertyError = null + try { + spark_paimon """ + ALTER TABLE paimon.${dbName}.t_property_change SET TBLPROPERTIES ( + 'fields.missing.sequence-group' = 'a') + """ + } catch (Exception e) { + invalidPropertyError = e.getMessage() + } + assertNotNull(invalidPropertyError) + assertTrue(invalidPropertyError.toLowerCase().contains("missing")) + assertEquals(propertySnapshot, (sql """ + SELECT max(snapshot_id) FROM t_property_change\$snapshots + """)[0][0] as long) + sql """INSERT INTO t_property_change (id, a) VALUES (1, 'still-writable')""" + order_qt_property_change_still_writable """ + SELECT * FROM t_property_change ORDER BY id + """ + + spark_paimon """ + ALTER TABLE paimon.${dbName}.t_property_change SET TBLPROPERTIES ( + 'fields.seq.sequence-group' = 'a') + """ + sql """REFRESH CATALOG ${catalogName}""" + sql """USE ${dbName}""" + sql """INSERT INTO t_property_change VALUES (1, 'high', 20)""" + sql """INSERT INTO t_property_change VALUES (1, 'low-must-lose', 15)""" + order_qt_property_change_sequence_group """ + SELECT * FROM t_property_change ORDER BY id + """ + assertSparkEquals("t_property_change", "*", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.groovy new file mode 100644 index 00000000000000..b33c981e716442 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.groovy @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_sequence_rowkind", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_sequence_rowkind_catalog" + String dbName = "test_pw_sequence_rowkind_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_sequence_asc; + CREATE TABLE paimon.${dbName}.t_sequence_asc ( + id INT, seq1 INT, seq2 INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'sequence.field' = 'seq1,seq2', + 'sequence.field.sort-order' = 'ascending' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_sequence_desc; + CREATE TABLE paimon.${dbName}.t_sequence_desc ( + id INT, seq INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'sequence.field' = 'seq', + 'sequence.field.sort-order' = 'descending' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_rowkind; + CREATE TABLE paimon.${dbName}.t_rowkind ( + id INT, row_kind STRING, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'rowkind.field' = 'row_kind', + 'changelog-producer' = 'input' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + return rows[0][0] == null ? 0L : rows[0][0].toString().toLong() + } + def activeFileCount = { String tableName -> + def rows = spark_paimon """ + SELECT count(*) FROM paimon.${dbName}.`${tableName}\$files` + """ + return rows[0][0].toString().toLong() + } + def assertSparkEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """ + SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Multi-column ascending sequences compare lexicographically across + // Doris commits. A NULL tuple loses once a concrete sequence arrives. + sql """INSERT INTO t_sequence_asc VALUES + (1, 10, 20, 'base'), + (2, NULL, NULL, 'null-base') + """ + sql """INSERT INTO t_sequence_asc VALUES + (1, 9, 99, 'stale-first-field'), + (2, 1, 0, 'nonnull-wins') + """ + sql """INSERT INTO t_sequence_asc VALUES + (1, 10, 21, 'new-second-field'), + (2, NULL, NULL, 'null-must-not-return') + """ + order_qt_sequence_ascending """ + SELECT * FROM t_sequence_asc ORDER BY id + """ + + // Equal sequences fall back to input order. Keep the writer single-task + // so this oracle checks Paimon's tie rule instead of scheduler ordering. + sql """set parallel_pipeline_task_num = 1""" + sql """INSERT INTO t_sequence_asc VALUES + (3, 7, 7, 'first-equal'), + (3, 7, 7, 'second-equal') + """ + order_qt_sequence_equal """ + SELECT id, payload FROM t_sequence_asc WHERE id = 3 ORDER BY id + """ + assertSparkEquals("t_sequence_asc", "*", "ORDER BY id") + + // Descending order reverses priority: a smaller sequence supersedes the + // current row while a larger sequence is ignored. + sql """INSERT INTO t_sequence_desc VALUES (1, 10, 'base')""" + sql """INSERT INTO t_sequence_desc VALUES (1, 20, 'larger-is-stale')""" + sql """INSERT INTO t_sequence_desc VALUES (1, 5, 'smaller-wins')""" + order_qt_sequence_descending """ + SELECT * FROM t_sequence_desc ORDER BY id + """ + assertSparkEquals("t_sequence_desc", "*", "ORDER BY id") + + // rowkind.field turns ordinary INSERT rows into an input changelog. + sql """INSERT INTO t_rowkind VALUES + (1, '+I', 'old-1'), + (2, '+I', 'old-2') + """ + long rowkindBefore = latestSnapshotId("t_rowkind") + sql """INSERT INTO t_rowkind VALUES + (1, '+U', 'new-1'), + (2, '-D', 'old-2'), + (3, '+I', 'new-3') + """ + long rowkindAfter = latestSnapshotId("t_rowkind") + order_qt_rowkind_changelog """ + SELECT id, payload FROM t_rowkind ORDER BY id + """ + + def auditRows = spark_paimon """ + SELECT rowkind, id, payload + FROM paimon_incremental_query( + 'paimon.${dbName}.`t_rowkind\$audit_log`', + '${rowkindBefore}', '${rowkindAfter}') + ORDER BY id + """ + assertEquals([ + ["+U", 1, "new-1"], + ["-D", 2, "old-2"], + ["+I", 3, "new-3"] + ], auditRows) + assertSparkEquals("t_rowkind", "id, payload", "ORDER BY id") + + // Invalid or omitted row kinds fail atomically. The following valid + // changelog record must still be accepted by a newly opened writer. + long beforeInvalidSnapshot = latestSnapshotId("t_rowkind") + long beforeInvalidFiles = activeFileCount("t_rowkind") + test { + sql """INSERT INTO t_rowkind VALUES (9, 'XX', 'invalid')""" + exception "row kind" + } + assertEquals(beforeInvalidSnapshot, latestSnapshotId("t_rowkind")) + assertEquals(beforeInvalidFiles, activeFileCount("t_rowkind")) + + test { + sql """INSERT INTO t_rowkind VALUES (9, NULL, 'missing-kind')""" + exception "cannot be null" + } + assertEquals(beforeInvalidSnapshot, latestSnapshotId("t_rowkind")) + assertEquals(beforeInvalidFiles, activeFileCount("t_rowkind")) + + sql """INSERT INTO t_rowkind VALUES (4, '+I', 'recovered')""" + order_qt_rowkind_recovered """ + SELECT id, payload FROM t_rowkind WHERE id = 4 ORDER BY id + """ + assertSparkEquals("t_rowkind", "id, payload", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_snapshot_refs.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_snapshot_refs.groovy new file mode 100644 index 00000000000000..1184895f2ad422 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_snapshot_refs.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_snapshot_refs", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_snapshot_refs_catalog" + String dbName = "test_pw_snapshot_refs_db" + String tableName = "t_refs" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.${tableName}; + CREATE TABLE paimon.${dbName}.${tableName} ( + id INT, + payload STRING, + amount DECIMAL(18, 2), + event_time TIMESTAMP_NTZ + ) USING paimon + TBLPROPERTIES ( + 'bucket' = '-1', + 'write-only' = 'true', + 'file.format' = 'parquet' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'meta.cache.paimon.table.ttl-second' = '0' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // Doris creates the snapshot which becomes the immutable tag and branch base. + sql """ + INSERT INTO ${tableName} VALUES + (1, 'base', 10.25, '2026-07-01 10:11:12.123456') + """ + long baselineSnapshot = (sql """ + SELECT MAX(snapshot_id) FROM ${tableName}\$snapshots + """)[0][0] as long + + spark_paimon """REFRESH TABLE paimon.${dbName}.${tableName}""" + spark_paimon_multi """ + CALL paimon.sys.create_tag( + table => '${dbName}.${tableName}', + tag => 'baseline_tag', + snapshot => ${baselineSnapshot} + ); + CALL paimon.sys.create_branch( + '${dbName}.${tableName}', + 'audit_branch', + 'baseline_tag' + ); + """ + + sql """ + INSERT INTO ${tableName} VALUES + (2, 'latest', 20.50, '2026-07-02 10:11:12.654321') + """ + sql """refresh table ${tableName}""" + + def baseline = [[1, "base", "10.25", "2026-07-01 10:11:12.123456"]] + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName} FOR VERSION AS OF ${baselineSnapshot} + ORDER BY id + """)) + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName}@tag(baseline_tag) + ORDER BY id + """)) + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName}@branch(audit_branch) + ORDER BY id + """)) + + // A historical source relation must keep its own schema/snapshot while + // the sink is rebound to the latest writable table generation. + sql """ + INSERT INTO ${tableName} + SELECT id + 100, concat(payload, '-snapshot-copy'), amount + 1, event_time + FROM ${tableName} FOR VERSION AS OF ${baselineSnapshot} + """ + sql """refresh table ${tableName}""" + assertEquals([[1, "base"], [2, "latest"], [101, "base-snapshot-copy"]], sql(""" + SELECT id, payload FROM ${tableName} ORDER BY id + """)) + + // Branches are valid sources even though Doris does not currently expose + // a Paimon branch sink. + sql """ + INSERT INTO ${tableName} + SELECT id + 200, concat(payload, '-branch-copy'), amount + 2, event_time + FROM ${tableName}@branch(audit_branch) + """ + sql """refresh table ${tableName}""" + assertEquals([[1, "base"], [2, "latest"], [101, "base-snapshot-copy"], + [201, "base-branch-copy"]], sql(""" + SELECT id, payload FROM ${tableName} ORDER BY id + """)) + + // Keep this unsupported boundary explicit. A rejected branch sink must + // not fall back to the main branch or mutate the referenced branch. + test { + sql """ + INSERT INTO ${tableName}@branch(audit_branch) + VALUES (9, 'branch-write', 9.00, '2026-07-09 00:00:00') + """ + exception "Only support insert data into iceberg table's branch" + } + test { + sql """ + INSERT OVERWRITE TABLE ${tableName}@branch(audit_branch) + VALUES (9, 'branch-overwrite', 9.00, '2026-07-09 00:00:00') + """ + exception "Only support insert overwrite into iceberg table's branch" + } + + assertEquals(4L, (sql """SELECT COUNT(*) FROM ${tableName}""")[0][0] as long) + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName}@tag(baseline_tag) + ORDER BY id + """)) + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName}@branch(audit_branch) + ORDER BY id + """)) + assertEquals(baseline, sql(""" + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName} FOR VERSION AS OF ${baselineSnapshot} + ORDER BY id + """)) + + spark_paimon """REFRESH TABLE paimon.${dbName}.${tableName}""" + def sparkRows = spark_paimon """ + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, 'yyyy-MM-dd HH:mm:ss.SSSSSS') + FROM paimon.${dbName}.${tableName} + ORDER BY id + """ + def dorisRows = sql """ + SELECT id, payload, CAST(amount AS STRING), + DATE_FORMAT(event_time, '%Y-%m-%d %H:%i:%s.%f') + FROM ${tableName} + ORDER BY id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_source_models.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_source_models.groovy new file mode 100644 index 00000000000000..64f4272c1c38d6 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_source_models.groovy @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_source_models", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_source_models_catalog" + String dbName = "test_pw_source_models_db" + String internalDb = "test_pw_source_models_internal_db" + + sql """drop database if exists internal.${internalDb} force""" + sql """create database internal.${internalDb}""" + + // Keep the source layouts deliberately different. The sink must consume the + // source query result, not raw source rows hidden by each OLAP table model. + sql """ + create table internal.${internalDb}.source_duplicate ( + id int, + category varchar(20), + amount bigint + ) + duplicate key(id) + distributed by random buckets 3 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.source_duplicate values + (1, 'A', 10), + (1, 'A', 11), + (2, null, 20) + """ + + sql """ + create table internal.${internalDb}.source_unique_mow ( + id int, + category varchar(20), + amount bigint + ) + unique key(id, category) + partition by list(category) ( + partition p_ab values in ('A', 'B'), + partition p_null values in (null) + ) + distributed by hash(id) buckets auto + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """ + insert into internal.${internalDb}.source_unique_mow values + (10, 'A', 100), (11, null, 110) + """ + sql """ + insert into internal.${internalDb}.source_unique_mow values + (10, 'A', 101) + """ + + sql """ + create table internal.${internalDb}.source_unique_mor ( + id int, + category varchar(20), + amount bigint + ) + unique key(id) + partition by range(id) ( + partition p_lt_20 values less than (20), + partition p_max values less than maxvalue + ) + distributed by hash(id) buckets 2 + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ) + """ + sql """ + insert into internal.${internalDb}.source_unique_mor values + (20, 'C', 200), (21, 'D', 210) + """ + sql """ + insert into internal.${internalDb}.source_unique_mor values + (20, 'C', 201) + """ + + sql """ + create table internal.${internalDb}.source_aggregate ( + id int, + category varchar(20), + amount bigint sum + ) + aggregate key(id, category) + partition by range(id) ( + partition p_lt_40 values less than (40), + partition p_max values less than maxvalue + ) + distributed by hash(id, category) buckets 4 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.source_aggregate values + (30, 'E', 300), + (30, 'E', 3), + (31, 'F', 310) + """ + + sql """ + create table internal.${internalDb}.source_complex ( + id int, + metrics array, + attributes map, + profile struct, + flags array, + nested_payload map>>, + event_date date, + event_time datetime(6) + ) + duplicate key(id) + distributed by hash(id) buckets 3 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.source_complex values + ( + 1, + array(cast(1.25 as decimal(10, 2)), cast(null as decimal(10, 2))), + map('alpha', 10, 'nullable', null), + named_struct('name', 'alice', 'active', true), + array(true, false, cast(null as boolean)), + map('term', array( + named_struct('score', 90, 'label', 'good'), + named_struct('score', cast(null as int), 'label', null) + )), + date '2024-02-29', + timestamp '2024-02-29 12:34:56.123456' + ), + ( + 2, + array(), + map(), + named_struct('name', cast(null as string), + 'active', cast(null as boolean)), + array(), + map('empty', array()), + date '1970-01-01', + timestamp '1970-01-01 00:00:00.000001' + ), + (3, null, null, null, null, null, null, null) + """ + + spark_paimon_multi """ + SET spark.sql.timestampType=TIMESTAMP_NTZ; + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.source_model_sink; + CREATE TABLE paimon.${dbName}.source_model_sink ( + source_model STRING NOT NULL, + id INT, + category STRING, + amount BIGINT + ) USING paimon + PARTITIONED BY (source_model) + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.complex_sink; + CREATE TABLE paimon.${dbName}.complex_sink ( + id INT, + metrics ARRAY, + attributes MAP, + profile STRUCT, + flags ARRAY, + nested_payload MAP>>, + event_date DATE, + event_time TIMESTAMP_NTZ + ) USING paimon + TBLPROPERTIES ('file.format' = 'orc'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + sql """ + insert into source_model_sink + select 'duplicate', id, category, amount + from internal.${internalDb}.source_duplicate + """ + sql """ + insert into source_model_sink + select 'unique_mow', id, category, amount + from internal.${internalDb}.source_unique_mow + """ + sql """ + insert into source_model_sink + select 'unique_mor', id, category, amount + from internal.${internalDb}.source_unique_mor + """ + sql """ + insert into source_model_sink + select 'aggregate', id, category, amount + from internal.${internalDb}.source_aggregate + """ + + def sourceRows = sql """ + select 'duplicate', id, category, amount + from internal.${internalDb}.source_duplicate + union all + select 'unique_mow', id, category, amount + from internal.${internalDb}.source_unique_mow + union all + select 'unique_mor', id, category, amount + from internal.${internalDb}.source_unique_mor + union all + select 'aggregate', id, category, amount + from internal.${internalDb}.source_aggregate + order by 1, 2, 3, 4 + """ + def sinkRows = sql """ + select source_model, id, category, amount + from source_model_sink + order by 1, 2, 3, 4 + """ + assertEquals(sourceRows, sinkRows) + assertEquals(4L, + (sql """select count(*) from source_model_sink\$snapshots""")[0][0] as long) + + def sparkModelRows = spark_paimon """ + select source_model, id, category, amount + from paimon.${dbName}.source_model_sink + order by source_model, id, category, amount + """ + assertSparkDorisResultEquals(sparkModelRows, sinkRows) + + // Complex values now cross the OLAP scanner and an INSERT SELECT + // projection before reaching the Paimon Arrow writer. + sql """ + insert into complex_sink + select id, metrics, attributes, profile, flags, nested_payload, + event_date, event_time + from internal.${internalDb}.source_complex + """ + def complexRows = sql """ + select id, metrics, attributes, profile, flags, nested_payload, + event_date, event_time + from complex_sink + order by id + """ + def sparkComplexRows = spark_paimon """ + select id, metrics, attributes, profile, flags, nested_payload, + event_date, event_time + from paimon.${dbName}.complex_sink + order by id + """ + assertSparkDorisResultEquals(sparkComplexRows, complexRows) + assertEquals(3L, complexRows.size() as long) + assertEquals(1L, + (sql """select count(*) from complex_sink\$snapshots""")[0][0] as long) + } finally { + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop database if exists internal.${internalDb} force""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_thread_lifecycle.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_thread_lifecycle.groovy new file mode 100644 index 00000000000000..e5f34149a76191 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_thread_lifecycle.groovy @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Http is a framework utility class, not an injected Suite DSL property. +import org.apache.doris.regression.util.Http + +suite("test_paimon_write_thread_lifecycle", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_thread_lifecycle_catalog" + String dbName = "test_pw_thread_lifecycle_db" + + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort) + def backendEndpoints = backendIdToIp.collectEntries { backendId, ip -> + [(backendId): [ip.toString(), backendIdToHttpPort[backendId].toString()]] + } + assertFalse(backendEndpoints.isEmpty()) + + def jvmThreadCounts = { + backendEndpoints.collectEntries { backendId, endpoint -> + [(backendId): (get_be_metric(endpoint[0], endpoint[1], "jvm_thread", "count") as long)] + } + } + def processThreadCounts = { + backendEndpoints.collectEntries { backendId, endpoint -> + def body = Http.GET("http://${endpoint[0]}:${endpoint[1]}/api/be_process_thread_num", + false, false).toString() + def item = parseJson(body).find { row -> row[0].toString() == "total_thread_count" } + assertNotNull(item) + [(backendId): item[1].toString().toLong()] + } + } + def minimumThreadCounts = { counter -> + def minimums = null + for (int sample = 0; sample < 5; sample++) { + def counts = counter() + minimums = minimums == null ? counts : counts.collectEntries { backendId, count -> + [(backendId): Math.min(minimums[backendId], count)] + } + sleep(1000) + } + minimums + } + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_thread_lifecycle; + CREATE TABLE paimon.${dbName}.t_thread_lifecycle ( + id BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '3', + 'bucket-key' = 'id', + 'num-sorted-run.compaction-trigger' = '2', + 'target-file-size' = '1 gb' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // Warm all writer and metrics paths before taking the baseline. This keeps + // one-time JVM attachment and SDK class initialization out of the leak oracle. + for (int round = 0; round < 12; round++) { + sql """ + INSERT INTO t_thread_lifecycle + SELECT number + ${round * 1000}, repeat('w', 32) + FROM numbers("number" = "1000") + """ + } + sleep(3000) + + def jvmBefore = minimumThreadCounts(jvmThreadCounts) + def processBefore = minimumThreadCounts(processThreadCounts) + logger.info("Paimon thread baseline: jvm=${jvmBefore}, process=${processBefore}") + + def writePhase = { int firstRound -> + for (int round = firstRound; round < firstRound + 12; round++) { + sql """ + INSERT INTO t_thread_lifecycle + SELECT number + ${round * 1000}, repeat('x', 32) + FROM numbers("number" = "1000") + """ + } + } + + def jvmPhases = [] + def processPhases = [] + for (int phase = 0; phase < 4; phase++) { + writePhase(12 + phase * 12) + sleep(5000) + jvmPhases.add(minimumThreadCounts(jvmThreadCounts)) + processPhases.add(minimumThreadCounts(processThreadCounts)) + logger.info("Paimon thread phase ${phase + 1}: jvm=${jvmPhases[-1]}, " + + "process=${processPhases[-1]}") + } + + assertEquals(60000L, + (sql """SELECT COUNT(*) FROM t_thread_lifecycle""")[0][0] as long) + + backendEndpoints.keySet().each { backendId -> + // Warm-up performs the same workload as every measured phase. Judge persistent growth + // from the actual pre-phase baseline and phase low-water marks instead of failing on + // an isolated background-thread spike: a leaked thread cannot disappear in a later + // phase, while an unrelated transient thread can. + def jvmCounts = jvmPhases.collect { sample -> sample[backendId] as long } + def processCounts = processPhases.collect { sample -> sample[backendId] as long } + def earlyJvmFloor = jvmCounts.take(2).min() + def lateJvmFloor = jvmCounts.drop(2).min() + def earlyProcessFloor = processCounts.take(2).min() + def lateProcessFloor = processCounts.drop(2).min() + + assertTrue(jvmCounts.min() <= jvmBefore[backendId] + 2, + "JVM threads never returned to the warm-up baseline on backend ${backendId}: " + + "baseline=${jvmBefore[backendId]}, phases=${jvmCounts}") + assertTrue(lateJvmFloor <= earlyJvmFloor + 2, + "JVM threads kept growing on backend ${backendId}: phases=${jvmCounts}") + assertTrue(processCounts.min() <= processBefore[backendId] + 4, + "Process threads never returned to the warm-up baseline on backend ${backendId}: " + + "baseline=${processBefore[backendId]}, phases=${processCounts}") + assertTrue(lateProcessFloor <= earlyProcessFloor + 4, + "Process threads kept growing on backend ${backendId}: phases=${processCounts}") + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy new file mode 100644 index 00000000000000..bfdeffa376bbe6 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_transaction", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_txn_catalog" + String dbName = "test_pw_txn_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit; + CREATE TABLE paimon.${dbName}.t_commit ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit_batch; + CREATE TABLE paimon.${dbName}.t_commit_batch ( + id INT, val DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite; + CREATE TABLE paimon.${dbName}.t_overwrite ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite_part; + CREATE TABLE paimon.${dbName}.t_overwrite_part ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite_part_case; + CREATE TABLE paimon.${dbName}.t_overwrite_part_case ( + id INT, name STRING, Region STRING + ) USING paimon + PARTITIONED BY (Region); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_multi; + CREATE TABLE paimon.${dbName}.t_static_multi ( + id INT, name STRING, pt0 INT, pt1 STRING + ) USING paimon + PARTITIONED BY (pt0, pt1) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_default; + CREATE TABLE paimon.${dbName}.t_static_default ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true', + 'partition.default-name' = '__CUSTOM_DEFAULT_PARTITION__' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_boundary; + CREATE TABLE paimon.${dbName}.t_static_boundary ( + id INT, name STRING, region STRING, dt DATE + ) USING paimon + PARTITIONED BY (region, dt) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true', + 'partition.default-name' = '__CUSTOM_DEFAULT_PARTITION__' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_dynamic_multi; + CREATE TABLE paimon.${dbName}.t_dynamic_multi ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi; + CREATE TABLE paimon.${dbName}.t_multi ( + id INT, name STRING, score DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_parallel; + CREATE TABLE paimon.${dbName}.t_parallel ( + id BIGINT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi_block; + CREATE TABLE paimon.${dbName}.t_multi_block ( + id BIGINT, + group_id INT, + payload STRING, + nullable_value BIGINT, + pt STRING + ) USING paimon + PARTITIONED BY (pt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_spill; + CREATE TABLE paimon.${dbName}.t_spill ( + id BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-buffer-size' = '256 kb', + 'page-size' = '64 kb', + 'write-buffer-spillable' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_failed_write; + CREATE TABLE paimon.${dbName}.t_failed_write ( + id BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-buffer-size' = '256 kb', + 'page-size' = '64 kb', + 'write-buffer-spillable' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-010: Basic commit β€” INSERT INTO, commit, read back + sql """INSERT INTO t_commit VALUES (1, 'alice'), (2, 'bob')""" + assertTableEquals("t_commit", "ORDER BY id") + + sql """INSERT INTO t_commit VALUES (3, 'charlie'), (4, 'diana')""" + order_qt_txn_commit """SELECT id, name FROM t_commit ORDER BY id""" + assertTableEquals("t_commit", "ORDER BY id") + + // FT-011: Batch INSERT β€” 10 rows, then self-copy via SELECT + sql """INSERT INTO t_commit_batch VALUES + (1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0), (5, 5.0), + (6, 6.0), (7, 7.0), (8, 8.0), (9, 9.0), (10, 10.0)""" + sql """INSERT INTO t_commit_batch SELECT id + 10, val + 10.0 FROM t_commit_batch""" + order_qt_txn_batch """SELECT id, val FROM t_commit_batch ORDER BY id""" + assertTableEquals("t_commit_batch", "ORDER BY id") + + // FT-012: Full-table overwrite must remove every row from the previous snapshot. + sql """INSERT INTO t_overwrite VALUES (1, 'old1'), (2, 'old2'), (3, 'old3')""" + assertTableEquals("t_overwrite", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_overwrite VALUES (10, 'new1'), (20, 'new2')""" + order_qt_txn_overwrite """SELECT id, name FROM t_overwrite ORDER BY id""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // FT-013: An overwrite with an empty input still commits an empty snapshot. + sql """INSERT OVERWRITE TABLE t_overwrite SELECT 1, 'unused' WHERE 1 = 0""" + qt_txn_empty_overwrite """SELECT COUNT(*) FROM t_overwrite""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // A direct PhysicalEmptyRelation must still publish the empty overwrite snapshot. + sql """INSERT INTO t_overwrite VALUES (30, 'old_for_limit_zero')""" + sql """INSERT OVERWRITE TABLE t_overwrite SELECT 1, 'unused' LIMIT 0""" + qt_txn_empty_overwrite_limit_zero """SELECT COUNT(*) FROM t_overwrite""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // FT-015: Static partition overwrite replaces only the requested partition. + sql """INSERT INTO t_overwrite_part VALUES + (1, 'east_old', 'east'), (2, 'west_old', 'west')""" + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region = 'east') VALUES (10, 'east_new')""" + order_qt_txn_static_partition """SELECT id, name, region FROM t_overwrite_part ORDER BY id""" + assertTableEquals("t_overwrite_part", "ORDER BY id") + + // Static partition names are case-insensitive in Doris and canonicalized + // to the exact Paimon schema field name before commit. + sql """INSERT INTO t_overwrite_part_case VALUES + (1, 'east_old', 'east'), (2, 'west_old', 'west')""" + sql """INSERT OVERWRITE TABLE t_overwrite_part_case + PARTITION (region = 'east') VALUES (10, 'east_new')""" + order_qt_txn_static_partition_case """SELECT id, name, Region + FROM t_overwrite_part_case ORDER BY id""" + assertTableEquals("t_overwrite_part_case", "ORDER BY id") + + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region = 'east', REGION = 'west') VALUES (20, 'ambiguous')""" + exception "Duplicate partition column: REGION" + } + + // A static partial spec uses static overwrite semantics even when the + // table default is dynamic partition overwrite. It replaces every + // matching subpartition, including those absent from the new input. + sql """INSERT INTO t_static_multi VALUES + (1, 'old_a', 1, 'A'), (2, 'old_b', 1, 'B'), (3, 'keep', 2, 'C')""" + sql """INSERT OVERWRITE TABLE t_static_multi + PARTITION (pt0 = 1) VALUES (10, 'new_a', 'A')""" + order_qt_txn_static_partial """SELECT id, name, pt0, pt1 + FROM t_static_multi ORDER BY id""" + assertTableEquals("t_static_multi", "ORDER BY id") + + // Empty static overwrite still removes the complete matching spec. + sql """INSERT OVERWRITE TABLE t_static_multi + PARTITION (pt0 = 1) SELECT 1, 'unused', 'A' LIMIT 0""" + order_qt_txn_static_partial_empty """SELECT id, name, pt0, pt1 + FROM t_static_multi ORDER BY id""" + assertTableEquals("t_static_multi", "ORDER BY id") + + // NULL must use the table's actual configurable default partition + // name, while the literal string "null" and blank strings remain + // distinct typed partition values. + sql """INSERT INTO t_static_default VALUES + (1, 'null_old', NULL), + (2, 'literal_null', 'null'), + (3, 'blank_old', ''), + (4, 'east_old', 'east')""" + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = NULL) VALUES (10, 'null_new')""" + order_qt_txn_static_null """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = 'east') SELECT 1, 'unused' LIMIT 0""" + order_qt_txn_static_empty """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = '') VALUES (30, 'blank_new')""" + order_qt_txn_static_blank """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + // A partial static specification must use typed partition identity. + // NULL, blank, the literal "null", escaped path characters, and DATE + // subpartitions must remain distinct even when display paths overlap. + sql """INSERT INTO t_static_boundary VALUES + (1, 'null_d1', NULL, '2026-07-01'), + (2, 'null_d2', NULL, '2026-07-02'), + (3, 'blank', '', '2026-07-01'), + (4, 'literal_null', 'null', '2026-07-01'), + (5, 'special_d1', 'a/b=c%20', '2026-07-01'), + (6, 'special_d2', 'a/b=c%20', '2026-07-02'), + (7, 'keep', 'keep', '2026-07-01') + """ + sql """INSERT OVERWRITE TABLE t_static_boundary + PARTITION (region = NULL) + VALUES (10, 'null_new', '2026-07-03')""" + def staticNullRows = sql """ + SELECT id, name, IF(region = '', '', region), CAST(dt AS STRING) + FROM t_static_boundary ORDER BY id + """ + assertEquals([ + [3, "blank", "", "2026-07-01"], + [4, "literal_null", "null", "2026-07-01"], + [5, "special_d1", "a/b=c%20", "2026-07-01"], + [6, "special_d2", "a/b=c%20", "2026-07-02"], + [7, "keep", "keep", "2026-07-01"], + [10, "null_new", null, "2026-07-03"] + ], staticNullRows) + + sql """INSERT OVERWRITE TABLE t_static_boundary + PARTITION (region = 'a/b=c%20') + VALUES (50, 'special_new', '2026-07-04')""" + order_qt_txn_static_typed_boundaries """ + SELECT id, name, IF(region = '', '', region) AS region, dt + FROM t_static_boundary ORDER BY id + """ + assertTableEquals("t_static_boundary", "ORDER BY id") + def sparkBoundaryPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_static_boundary\$partitions` + ORDER BY `partition` + """ + def dorisBoundaryPartitions = sql """ + SELECT `partition`, record_count + FROM t_static_boundary\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkBoundaryPartitions, dorisBoundaryPartitions) + + // Dynamic overwrite replaces all partitions present in one input batch, + // preserves untouched partitions, and publishes one overwrite snapshot. + sql """INSERT INTO t_dynamic_multi VALUES + (1, 'p1_old_a', 'p1'), (2, 'p1_old_b', 'p1'), + (3, 'p2_old', 'p2'), (4, 'p3_keep', 'p3'), (5, 'p4_keep', 'p4') + """ + sql """INSERT OVERWRITE TABLE t_dynamic_multi VALUES + (10, 'p1_new', 'p1'), + (20, 'p2_new_a', 'p2'), + (21, 'p2_new_b', 'p2') + """ + order_qt_txn_dynamic_multi """ + SELECT id, name, region FROM t_dynamic_multi ORDER BY id + """ + assertTableEquals("t_dynamic_multi", "ORDER BY id") + assertEquals(2L, + (sql """SELECT COUNT(*) FROM t_dynamic_multi\$snapshots""")[0][0] as long) + def sparkDynamicPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_dynamic_multi\$partitions` + ORDER BY `partition` + """ + def dorisDynamicPartitions = sql """ + SELECT `partition`, record_count + FROM t_dynamic_multi\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkDynamicPartitions, dorisDynamicPartitions) + + // FT-016: Dynamic partition overwrite replaces the partitions present in the + // input while preserving existing partitions that are not touched. + sql """INSERT OVERWRITE TABLE t_overwrite_part VALUES (30, 'south_new', 'south')""" + order_qt_txn_dynamic_partition """SELECT id, name, region FROM t_overwrite_part ORDER BY id""" + assertTableEquals("t_overwrite_part", "ORDER BY id") + + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region) VALUES (40, 'bare_partition', 'east')""" + exception "Paimon tables do not support PARTITION name lists" + } + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + TEMPORARY PARTITION (region) VALUES (40, 'temporary_partition', 'east')""" + exception "Paimon tables do not support temporary partitions" + } + order_qt_txn_unsupported_partition_syntax """ + SELECT id, name, region FROM t_overwrite_part ORDER BY id + """ + assertTableEquals("t_overwrite_part", "ORDER BY id") + + // FT-017: Multiple pipeline tasks create multiple LocalState-scoped writers. + // FE must aggregate every writer's commit payload into one Paimon snapshot. + sql """SET parallel_pipeline_task_num = 4""" + sql """INSERT INTO t_parallel + SELECT number, concat('row_', CAST(number AS STRING)) + FROM numbers("number" = "256")""" + qt_txn_parallel_writers """SELECT COUNT(*), MIN(id), MAX(id), SUM(id) FROM t_parallel""" + assertTableEquals("t_parallel", "ORDER BY id") + sql """SET parallel_pipeline_task_num = 0""" + + // A single JNI writer receives many native Blocks, each serialized as an + // independent Arrow stream. Every row must survive repeated write() calls, + // partition routing and prepareCommit(). + sql """SET parallel_pipeline_task_num = 1""" + sql """INSERT INTO t_multi_block + SELECT number, + CAST(number % 97 AS INT), + concat('payload_', CAST(number AS STRING)), + IF(number % 11 = 0, NULL, number * 3), + concat('p', CAST(number % 8 AS STRING)) + FROM numbers("number" = "16384")""" + + // Opening a fresh writer in the next Doris transaction must append to the + // existing snapshot without losing or duplicating the first transaction. + sql """INSERT INTO t_multi_block + SELECT number + 16384, + CAST((number + 16384) % 97 AS INT), + concat('payload_', CAST(number + 16384 AS STRING)), + IF((number + 16384) % 11 = 0, NULL, (number + 16384) * 3), + concat('p', CAST((number + 16384) % 8 AS STRING)) + FROM numbers("number" = "4096")""" + sql """SET parallel_pipeline_task_num = 0""" + + qt_txn_multi_block """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM t_multi_block + """ + order_qt_txn_multi_block_samples """ + SELECT id, group_id, payload, nullable_value, pt + FROM t_multi_block + WHERE id IN (0, 4095, 4096, 8191, 8192, 16383, 16384, 20479) + ORDER BY id + """ + qt_txn_multi_block_snapshots """ + SELECT COUNT(*) FROM t_multi_block\$snapshots + """ + def sparkMultiBlock = spark_paimon """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM paimon.${dbName}.t_multi_block + """ + def dorisMultiBlock = sql """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM t_multi_block + """ + assertSparkDorisResultEquals(sparkMultiBlock, dorisMultiBlock) + + // FT-018: The payload is larger than the 256 KB write buffer while each + // individual row still fits, forcing Paimon's spillable buffer path. + sql """INSERT INTO t_spill + SELECT number, repeat('spill_payload_', 80) + FROM numbers("number" = "2048")""" + qt_txn_spill """SELECT COUNT(*), MIN(id), MAX(id), SUM(id) FROM t_spill""" + assertTableEquals("t_spill", "ORDER BY id") + + // FT-019: A row larger than the complete write buffer fails inside the + // Paimon writer. The failed statement must not publish rows or a snapshot. + sql """INSERT INTO t_failed_write VALUES (1, 'committed_before_failure')""" + qt_txn_failed_write_before """SELECT id, payload FROM t_failed_write ORDER BY id""" + qt_txn_failed_snapshot_before """SELECT COUNT(*) FROM t_failed_write\$snapshots""" + test { + sql """INSERT INTO t_failed_write VALUES + (2, 'accepted_before_error'), + (3, repeat('x', 1048576))""" + exception "The record exceeds the maximum size of a sort buffer" + } + qt_txn_failed_write_after """SELECT id, payload FROM t_failed_write ORDER BY id""" + qt_txn_failed_snapshot_after """SELECT COUNT(*) FROM t_failed_write\$snapshots""" + assertTableEquals("t_failed_write", "ORDER BY id") + + // Multi-row VALUES β€” verify all 20 rows committed + sql """ + INSERT INTO t_multi VALUES + (1, 'a', 10.0), (2, 'b', 20.0), (3, 'c', 30.0), (4, 'd', 40.0), (5, 'e', 50.0), + (6, 'f', 60.0), (7, 'g', 70.0), (8, 'h', 80.0), (9, 'i', 90.0), (10, 'j', 100.0), + (11, 'k', 110.0),(12, 'l', 120.0),(13, 'm', 130.0),(14, 'n', 140.0),(15, 'o', 150.0), + (16, 'p', 160.0),(17, 'q', 170.0),(18, 'r', 180.0),(19, 's', 190.0),(20, 't', 200.0) + """ + order_qt_txn_multi """SELECT id, name, score FROM t_multi ORDER BY id""" + assertTableEquals("t_multi", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_types.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_types.groovy new file mode 100644 index 00000000000000..5d9d8f80e42481 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_types.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_types_catalog" + String dbName = "test_pw_types_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types; + CREATE TABLE paimon.${dbName}.t_types ( + c_boolean BOOLEAN, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE, + c_decimal DECIMAL(10,2), + c_string STRING, + c_varchar VARCHAR(100), + c_date DATE, + c_datetime TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_null; + CREATE TABLE paimon.${dbName}.t_types_null ( + id INT, + c_int INT, + c_string STRING, + c_double DOUBLE, + c_boolean BOOLEAN + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_decimal; + CREATE TABLE paimon.${dbName}.t_types_decimal ( + id INT, + d2 DECIMAL(2,1), + d10 DECIMAL(10,2), + d18 DECIMAL(18,6), + d38 DECIMAL(38,10) + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_dt; + CREATE TABLE paimon.${dbName}.t_types_dt ( + d DATE, dt TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_timezone; + CREATE TABLE paimon.${dbName}.t_types_timezone ( + id INT, event_time TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_ltz_schema; + CREATE TABLE paimon.${dbName}.t_types_ltz_schema ( + event_time TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_ntz; + CREATE TABLE paimon.${dbName}.t_types_ntz ( + id INT, event_time TIMESTAMP_NTZ + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + def originalTimeZone = sql """SELECT @@time_zone""" + + try { + def assertTableEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-020~027: Basic types with boundary values + sql """ + INSERT INTO t_types VALUES + (true, 1, 100, CAST(1.5 AS FLOAT), CAST(2.71828 AS DOUBLE), + CAST(99.99 AS DECIMAL(10,2)), 'hello', 'short', + DATE '2024-01-15', TIMESTAMP '2024-01-15 10:30:00'), + (false, 2147483647, 9223372036854775807, + CAST(3.4E38 AS FLOAT), CAST(1.7E308 AS DOUBLE), + CAST(0.00 AS DECIMAL(10,2)), '', '', + DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (false, -2147483648, -9223372036854775808, + CAST(-3.4E38 AS FLOAT), CAST(-1.7E308 AS DOUBLE), + CAST(-1.50 AS DECIMAL(10,2)), 'long_string_1234567890', 'max_varchar', + DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59'), + (true, 0, 0, CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE), + CAST(12345678.90 AS DECIMAL(10,2)), 'hello', 'fixed_len', + DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00.123456') + """ + order_qt_types_basic """SELECT * FROM t_types ORDER BY c_int""" + assertTableEquals("t_types", """ + c_boolean, c_int, c_bigint, + c_float / 1.0E38, + c_double / 1.0E308, + c_decimal, c_string, c_varchar, c_date, c_datetime + """, "ORDER BY c_int") + + // FT-040: NULL handling + sql """INSERT INTO t_types_null VALUES (1, 100, 'data', 1.5, true)""" + sql """INSERT INTO t_types_null VALUES (2, NULL, NULL, NULL, NULL)""" + sql """INSERT INTO t_types_null VALUES (3, NULL, 'partial', 2.0, false)""" + order_qt_types_null """SELECT id, c_int, c_string, c_double, c_boolean FROM t_types_null ORDER BY id""" + assertTableEquals("t_types_null", "*", "ORDER BY id") + + // FT-043: Decimal precision + sql """ + INSERT INTO t_types_decimal VALUES + (1, CAST(1.5 AS DECIMAL(2,1)), CAST(12345678.90 AS DECIMAL(10,2)), + CAST(123456789012.123456 AS DECIMAL(18,6)), + CAST(1234567890123456789012345678.1234567890 AS DECIMAL(38,10))), + (2, CAST(-1.5 AS DECIMAL(2,1)), CAST(-0.01 AS DECIMAL(10,2)), + CAST(-1.000001 AS DECIMAL(18,6)), + CAST(0.0000000001 AS DECIMAL(38,10))), + (3, CAST(0.0 AS DECIMAL(2,1)), CAST(0.00 AS DECIMAL(10,2)), + CAST(0.000000 AS DECIMAL(18,6)), + CAST(0.0000000000 AS DECIMAL(38,10))) + """ + order_qt_types_decimal """SELECT id, d2, d10, d18, d38 FROM t_types_decimal ORDER BY id""" + assertTableEquals("t_types_decimal", "*", "ORDER BY id") + + // DATE / DATETIME boundary + sql """ + INSERT INTO t_types_dt VALUES + (DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00'), + (DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59.999999') + """ + order_qt_types_dt """SELECT d, dt FROM t_types_dt ORDER BY d""" + assertTableEquals("t_types_dt", "*", "ORDER BY d") + + // FT-027: Spark TIMESTAMP maps to Paimon's local-zoned timestamp. Values + // written in different Doris session timezones must represent the same instant + // in UTC while preserving their independent microsecond fractions. + qt_desc_types_timezone """DESC t_types_ltz_schema""" + sql """SET time_zone = 'Asia/Shanghai'""" + sql """INSERT INTO t_types_timezone VALUES + (1, TIMESTAMP '2024-01-15 10:30:00.123456')""" + sql """SET time_zone = 'UTC'""" + sql """INSERT INTO t_types_timezone VALUES + (2, TIMESTAMP '2024-01-15 02:30:00.654321')""" + order_qt_types_timezone_utc """SELECT id, event_time FROM t_types_timezone ORDER BY id""" + + // Reading the same snapshot in Asia/Shanghai must apply the session offset + // to both rows without losing their microsecond fractions. + sql """SET time_zone = 'Asia/Shanghai'""" + order_qt_types_timezone_shanghai """SELECT id, event_time FROM t_types_timezone ORDER BY id""" + + // Paimon TIMESTAMP_NTZ stores civil fields. A value in a DST gap and a + // Doris-supported short timezone alias must therefore survive without + // an instant conversion or Java ZoneId parsing. + sql """SET time_zone = 'America/Los_Angeles'""" + sql """INSERT INTO t_types_ntz VALUES + (1, TIMESTAMP '2024-03-10 02:30:00.123456')""" + sql """SET time_zone = 'CST'""" + sql """INSERT INTO t_types_ntz VALUES + (2, TIMESTAMP '2024-01-15 10:30:00.654321')""" + sql """SET time_zone = 'UTC'""" + order_qt_types_ntz """SELECT id, event_time FROM t_types_ntz ORDER BY id""" + assertTableEquals("t_types_ntz", "*", "ORDER BY id") + } finally { + sql """SET time_zone = '${originalTimeZone[0][0]}'""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant.groovy new file mode 100644 index 00000000000000..5a6818ba4b96e2 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant.groovy @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_catalog" + String dbName = "test_pw_variant_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_basic; + CREATE TABLE paimon.${dbName}.t_variant_basic ( + id INT, + payload VARIANT, + secondary VARIANT + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + // Paimon Variant writes are deliberately V2-only. + setFeConfigTemporary([enable_variant_v2: false]) { + assertFalse(getFeConfig("enable_variant_v2").toBoolean()) + test { + sql """INSERT INTO t_variant_basic VALUES + (0, parse_to_variant('{"disabled":true}'), NULL)""" + exception "set FE config enable_variant_v2=true" + } + } + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + sql """SET force_jni_scanner = true""" + + // JSON containers, JSON null and SQL NULL are different logical values. + sql """ + INSERT INTO t_variant_basic VALUES + (1, parse_to_variant(CONCAT( + '{"object":{"name":"doris","address":{"city":"Hangzhou"}},"array":[1,true,null,"x"],"emptyObject":{},"emptyArray":[],"explicitNull":null,"escaped":"line', + CHAR(92), 'nquote', CHAR(92), CHAR(34), '","unicode":"δΈ­ζ–‡πŸ˜€"}')), + parse_to_variant('{"second":2}')), + (2, parse_to_variant('{}'), parse_to_variant('[]')), + (3, parse_to_variant('[]'), parse_to_variant('{}')), + (4, parse_to_variant('null'), parse_to_variant('null')), + (5, CAST(NULL AS VARIANT), CAST(NULL AS VARIANT)) + """ + + // Typed scalar values exercise every V2 primitive family used by Doris. + sql """ + INSERT INTO t_variant_basic VALUES + (10, CAST(TRUE AS VARIANT), CAST(FALSE AS VARIANT)), + (11, CAST(CAST(-128 AS TINYINT) AS VARIANT), + CAST(CAST(32767 AS SMALLINT) AS VARIANT)), + (12, CAST(CAST(-2147483648 AS INT) AS VARIANT), + CAST(CAST(9223372036854775807 AS BIGINT) AS VARIANT)), + (13, CAST(CAST(1.25 AS FLOAT) AS VARIANT), + CAST(CAST(-2.5 AS DOUBLE) AS VARIANT)), + (14, CAST(CAST(123456.789 AS DECIMAL(12, 3)) AS VARIANT), + CAST(CAST(-0.000001 AS DECIMAL(18, 6)) AS VARIANT)), + (15, CAST(CAST('plain-string' AS VARCHAR(32)) AS VARIANT), + CAST(CAST('δΈ­ζ–‡πŸ˜€' AS STRING) AS VARIANT)), + (16, CAST(DATE '2024-02-29' AS VARIANT), + CAST(CAST('2024-02-29 12:34:56.123456' AS DATETIMEV2(6)) AS VARIANT)), + (17, CAST(REPEAT('long-value-', 4096) AS VARIANT), + parse_to_variant('{"batch":"large-string"}')) + """ + + // Every VALUES row must reach Variant coercion before the inline table chooses a common + // type. In particular, the integer in the first row must not become the string "1". + sql """ + INSERT INTO t_variant_basic VALUES + (20, 1, 'row-one'), + (21, 'row-two', 2) + """ + order_qt_variant_heterogeneous """ + SELECT id, payload, secondary + FROM t_variant_basic + WHERE id IN (20, 21) + ORDER BY id + """ + + order_qt_variant_object """ + SELECT + CAST(payload['object']['name'] AS STRING), + CAST(payload['object']['address']['city'] AS STRING), + CAST(payload['array'][1] AS INT), + CAST(payload['array'][2] AS BOOLEAN), + CAST(payload['emptyObject'] AS STRING), + CAST(payload['emptyArray'] AS STRING), + CAST(payload['unicode'] AS STRING), + CAST(secondary['second'] AS INT) + FROM t_variant_basic + WHERE id = 1 + """ + + order_qt_variant_nulls """ + SELECT id, payload, payload IS NULL, payload['missing'] IS NULL + FROM t_variant_basic + WHERE id IN (4, 5) + ORDER BY id + """ + + order_qt_variant_scalars """ + SELECT id, payload, secondary + FROM t_variant_basic + WHERE id BETWEEN 10 AND 16 + ORDER BY id + """ + + qt_variant_long_string """ + SELECT LENGTH(CAST(payload AS STRING)), CAST(secondary['batch'] AS STRING) + FROM t_variant_basic + WHERE id = 17 + """ + + // Refresh metadata and verify that all Doris-written rows remain readable through the + // Paimon JNI Variant reader. + sql """REFRESH TABLE t_variant_basic""" + qt_variant_row_count """SELECT COUNT(*) FROM t_variant_basic""" + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_dml.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_dml.groovy new file mode 100644 index 00000000000000..19ab351a83f767 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_dml.groovy @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant_dml", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_dml_catalog" + String dbName = "test_pw_variant_dml_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_dml; + CREATE TABLE paimon.${dbName}.t_variant_dml ( + id INT, + payload VARIANT, + note STRING NOT NULL DEFAULT 'default-note', + pt STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_overwrite; + CREATE TABLE paimon.${dbName}.t_variant_overwrite ( + id INT, + payload VARIANT, + pt STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + sql """SET force_jni_scanner = true""" + // INSERT SELECT preserves the V2 value and metadata buffers through the Paimon sink. + sql """ + INSERT INTO t_variant_dml (id, payload, pt) + SELECT 1, parse_to_variant('{"source":"direct","n":1}'), 'p1' + UNION ALL + SELECT 2, parse_to_variant('["direct",2]'), 'p1' + UNION ALL + SELECT 3, parse_to_variant('null'), 'p2' + UNION ALL + SELECT 4, CAST(NULL AS VARIANT), 'p2' + """ + + // Reordered columns, partial columns and writer-side defaults. + sql """ + INSERT INTO t_variant_dml (pt, note, payload, id) VALUES + ('p3', 'reordered', parse_to_variant('{"mode":"reordered"}'), 10) + """ + sql """ + INSERT INTO t_variant_dml (pt, payload, id) VALUES + ('p3', parse_to_variant('{"mode":"default"}'), 11) + """ + sql """ + INSERT INTO t_variant_dml (pt, id) VALUES ('p3', 12) + """ + + // CTE, UNION ALL, expression-generated Variant and an empty input. + sql """ + INSERT INTO t_variant_dml + WITH source AS ( + SELECT 20 AS id, parse_to_variant('{"mode":"cte"}') AS payload, + 'cte-note' AS note, 'p4' AS pt + ) + SELECT id, payload, note, pt FROM source + """ + sql """ + INSERT INTO t_variant_dml + SELECT 21, parse_to_variant('{"mode":"union-a"}'), 'union', 'p4' + UNION ALL + SELECT 22, CAST(CAST(22 AS BIGINT) AS VARIANT), 'union', 'p4' + """ + sql """ + INSERT INTO t_variant_dml + SELECT 30 + number, + parse_to_variant(CONCAT('{"generated":', number, '}')), + 'generated', + 'p5' + FROM numbers("number" = "8") + """ + sql """ + INSERT INTO t_variant_dml + SELECT 100, parse_to_variant('{"unused":true}'), 'empty', 'p0' + WHERE 1 = 0 + """ + + // Static and dynamic partition writes. + sql """ + INSERT INTO t_variant_dml PARTITION (pt = 'static') + VALUES (50, parse_to_variant('{"partition":"static"}'), 'static-note') + """ + sql """ + INSERT INTO t_variant_dml VALUES + (51, parse_to_variant('{"partition":"dynamic-a"}'), 'dynamic', 'dynamic-a'), + (52, parse_to_variant('{"partition":"dynamic-b"}'), 'dynamic', 'dynamic-b') + """ + + order_qt_variant_dml_rows """ + SELECT id, payload, note, pt, payload IS NULL + FROM t_variant_dml + WHERE id IN (1, 2, 3, 4, 10, 11, 12, 20, 21, 22, 30, 37, 50, 51, 52) + ORDER BY id + """ + + // Static-partition overwrite exercises the overwrite writer with Variant V2 rows. + sql """ + INSERT INTO t_variant_overwrite VALUES + (1, parse_to_variant('{"state":"old-east"}'), 'east'), + (2, parse_to_variant('{"state":"old-west"}'), 'west') + """ + sql """ + INSERT OVERWRITE TABLE t_variant_overwrite + PARTITION (pt = 'east') + VALUES (10, parse_to_variant('{"state":"new-east"}')) + """ + order_qt_variant_partition_overwrite """ + SELECT id, payload, pt + FROM t_variant_overwrite + ORDER BY id + """ + + // A full-table overwrite on a partitioned table must remove untouched old partitions. + sql """ + INSERT OVERWRITE TABLE t_variant_overwrite VALUES + (20, parse_to_variant('{"state":"full-a"}'), 'all'), + (21, parse_to_variant('{"state":"full-b"}'), 'all') + """ + order_qt_variant_full_overwrite """ + SELECT id, payload, pt + FROM t_variant_overwrite + ORDER BY id + """ + + // Empty full-table overwrite must still commit an empty snapshot. + sql """ + INSERT OVERWRITE TABLE t_variant_overwrite + SELECT 30, parse_to_variant('{"state":"unused"}'), 'empty' + WHERE 1 = 0 + """ + qt_variant_empty_overwrite """SELECT COUNT(*) FROM t_variant_overwrite""" + + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy new file mode 100644 index 00000000000000..f2876e2b85df12 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_errors_catalog" + String dbName = "test_pw_variant_errors_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_error; + CREATE TABLE paimon.${dbName}.t_variant_error ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_nested_error; + CREATE TABLE paimon.${dbName}.t_variant_nested_error ( + id INT, + payloads ARRAY + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_coercion_source; + CREATE TABLE paimon.${dbName}.t_variant_coercion_source ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + // Both top-level and nested targets fail during analysis when V2 is disabled. + setFeConfigTemporary([enable_variant_v2: false]) { + assertFalse(getFeConfig("enable_variant_v2").toBoolean()) + test { + sql """INSERT INTO t_variant_error VALUES + (1, parse_to_variant('{"disabled":"top"}'))""" + exception "set FE config enable_variant_v2=true" + } + test { + sql """INSERT INTO t_variant_nested_error VALUES + (1, CAST(NULL AS ARRAY))""" + exception "set FE config enable_variant_v2=true" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 2, parse_to_variant('{"disabled":"select"}') + """ + exception "set FE config enable_variant_v2=true" + } + } + + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + sql """SET force_jni_scanner = true""" + + sql """ + INSERT INTO t_variant_coercion_source VALUES + (1, parse_to_variant('{"kind":"object-source"}')), + (2, parse_to_variant('["array-source",2]')) + """ + + // UNION, IF and CASE can choose a scalar common type before sink binding. The sink must + // reject that lossy implicit cast instead of encoding an object Variant as SQL NULL. + test { + sql """ + INSERT INTO t_variant_error + SELECT id + 29, payload FROM t_variant_coercion_source WHERE id = 1 + UNION ALL + SELECT 31, 1 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT id + 30, payload FROM t_variant_coercion_source WHERE id = 2 + UNION ALL + SELECT 33, 1 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 34, IF(TRUE, payload, 1) + FROM t_variant_coercion_source WHERE id = 1 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 35, IF(TRUE, payload, 1) + FROM t_variant_coercion_source WHERE id = 2 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 36, CASE WHEN TRUE THEN payload ELSE 1 END + FROM t_variant_coercion_source WHERE id = 1 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 37, CASE WHEN TRUE THEN payload ELSE 1 END + FROM t_variant_coercion_source WHERE id = 2 + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + WITH RECURSIVE source AS ( + SELECT IF(TRUE, payload, 1) AS payload + FROM t_variant_coercion_source WHERE id = 1 + UNION ALL + SELECT CAST(1 AS DECIMAL(38, 9)) FROM source + ) + SELECT 38, payload FROM source + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + test { + sql """ + INSERT INTO t_variant_error + SELECT 39, generated_payload + FROM (SELECT payload FROM t_variant_coercion_source WHERE id = 1) source + LATERAL VIEW explode(ARRAY(IF(TRUE, payload, 1))) generated AS generated_payload + """ + exception "Paimon VARIANT write cannot safely convert input column 'payload'" + } + + // Valid V2 writes still work after Config-gated analysis failures. + sql """ + INSERT INTO t_variant_error VALUES + (20, parse_to_variant('{"recovered":true}')), + (21, try_parse_to_variant('not-json')), + (22, CAST(CAST('{"typed":"string"}' AS STRING) AS VARIANT)) + """ + // Primitive-to-Variant coercion remains supported for both inline VALUES and ordinary SELECT. + sql """ + INSERT INTO t_variant_error VALUES + (23, 7), + (24, 'values-string'), + (25, TRUE), + (26, ARRAY(1, 2)) + """ + sql """INSERT INTO t_variant_error SELECT 27, 8""" + sql """INSERT INTO t_variant_error SELECT 28, 'select-string'""" + sql """INSERT INTO t_variant_error SELECT 29, FALSE""" + sql """INSERT INTO t_variant_error SELECT 30, ARRAY(3, 4)""" + + spark_paimon """REFRESH TABLE paimon.${dbName}.t_variant_error""" + def sparkPrimitiveRows = spark_paimon """ + SELECT id, to_json(payload) + FROM paimon.${dbName}.t_variant_error + WHERE id BETWEEN 23 AND 30 + ORDER BY id + """ + assertEquals([ + ["23", "7"], + ["24", '"values-string"'], + ["25", "true"], + ["26", "[1,2]"], + ["27", "8"], + ["28", '"select-string"'], + ["29", "false"], + ["30", "[3,4]"] + ], sparkPrimitiveRows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + }) + + // Invalid JSON is preserved as a Variant string unless the global + // throw-on-invalid-JSON option is enabled. + order_qt_variant_after_errors """ + SELECT id, payload IS NULL, + CAST(payload['recovered'] AS BOOLEAN), + payload + FROM t_variant_error + ORDER BY id + """ + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_nested.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_nested.groovy new file mode 100644 index 00000000000000..2e883a63dc5b44 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_nested.groovy @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant_nested", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_nested_catalog" + String dbName = "test_pw_variant_nested_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_nested; + CREATE TABLE paimon.${dbName}.t_variant_nested ( + id INT, + variants ARRAY, + variant_map MAP, + variant_struct STRUCT, + first_payload VARIANT, + second_payload VARIANT + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_deep; + CREATE TABLE paimon.${dbName}.t_variant_deep ( + id INT, + deep STRUCT< + level1:ARRAY< + MAP> + > + > + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + sql """SET force_jni_scanner = true""" + // ARRAY, MAP, STRUCT and multiple Variant columns in one Arrow batch. + sql """ + INSERT INTO t_variant_nested VALUES + ( + 1, + array( + parse_to_variant('{"kind":"array-object","n":1}'), + parse_to_variant('null'), + CAST(NULL AS VARIANT), + CAST(CAST(7 AS INT) AS VARIANT) + ), + map( + 'object', parse_to_variant('{"kind":"map-object","n":2}'), + 'json_null', parse_to_variant('null'), + 'sql_null', CAST(NULL AS VARIANT) + ), + named_struct( + 'label', 'struct-value', + 'payload', parse_to_variant('{"kind":"struct-object","n":3}') + ), + parse_to_variant('{"column":"first"}'), + parse_to_variant('["second",2]') + ), + ( + 2, + array(), + map(), + named_struct('label', 'empty', 'payload', parse_to_variant('{}')), + parse_to_variant('[]'), + CAST(NULL AS VARIANT) + ), + (3, NULL, NULL, NULL, NULL, NULL) + """ + + order_qt_variant_nested_values """ + SELECT + CAST(variants[1]['kind'] AS STRING), + variants[2], + variants[3], + CAST(variants[4] AS INT), + CAST(variant_map['object']['n'] AS INT), + variant_map['json_null'], + variant_map['sql_null'], + CAST(variant_struct.payload['kind'] AS STRING), + CAST(first_payload['column'] AS STRING), + CAST(second_payload[1] AS STRING) + FROM t_variant_nested + WHERE id = 1 + """ + + order_qt_variant_nested_containers """ + SELECT id, + variants IS NULL, SIZE(variants), + variant_map IS NULL, SIZE(variant_map), + variant_struct IS NULL + FROM t_variant_nested + WHERE id IN (2, 3) + ORDER BY id + """ + + // Deep nesting is P0: STRUCT -> ARRAY -> MAP -> STRUCT -> VARIANT. + sql """ + INSERT INTO t_variant_deep VALUES + ( + 1, + named_struct( + 'level1', + array( + map( + 'outer', + named_struct( + 'note', 'depth-1', + 'payload', parse_to_variant( + '{"level2":{"level3":{"level4":{"value":"deep-ok"}}}}') + ) + ) + ) + ) + ), + ( + 2, + named_struct( + 'level1', + array( + map( + 'null-leaf', + named_struct( + 'note', 'depth-null', + 'payload', CAST(NULL AS VARIANT) + ) + ) + ) + ) + ) + """ + + order_qt_variant_deep_value """ + SELECT id, + deep.level1[1]['outer'].note, + CAST(deep.level1[1]['outer'].payload['level2']['level3']['level4']['value'] + AS STRING) + FROM t_variant_deep + WHERE id = 1 + """ + + order_qt_variant_deep_null """ + SELECT deep.level1[1]['null-leaf'].payload IS NULL + FROM t_variant_deep + WHERE id = 2 + """ + + // Refreshing metadata must not affect nested Variant reads. + sql """REFRESH TABLE t_variant_deep""" + qt_variant_deep_count """SELECT COUNT(*) FROM t_variant_deep""" + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy new file mode 100644 index 00000000000000..45cb26295d1d87 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_shredding.groovy @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant_shredding", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_shredding_catalog" + String dbName = "test_pw_variant_shredding_db" + String shreddingSchema = + '{"type":"ROW","fields":[{"name":"payload","type":{"type":"ROW","fields":[' + + '{"name":"age","type":"INT"},' + + '{"name":"city","type":"STRING"},' + + '{"name":"active","type":"BOOLEAN"},' + + '{"name":"profile","type":{"type":"ROW","fields":[' + + '{"name":"name","type":"STRING"},' + + '{"name":"scores","type":{"type":"ARRAY","element":"INT"}}' + + ']}}]}}]}' + + // TODO: Use variant.shreddingSchema after Paimon passes the global option to its Parquet + // builder. In 1.4.2 the builder still requires the fallback spelling used below. + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_shredded; + CREATE TABLE paimon.${dbName}.t_variant_shredded ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ( + 'file.format' = 'parquet', + 'write-only' = 'true', + 'parquet.variant.shreddingSchema' = '${shreddingSchema}', + 'variant.inferShreddingSchema' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_inferred; + CREATE TABLE paimon.${dbName}.t_variant_inferred ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ( + 'file.format' = 'parquet', + 'write-only' = 'true', + 'variant.inferShreddingSchema' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_mixed; + CREATE TABLE paimon.${dbName}.t_variant_mixed ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ( + 'file.format' = 'parquet', + 'write-only' = 'true' + ); + """ + + def createDorisCatalog = { + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + sql """SET force_jni_scanner = true""" + } + + def sparkValues = { rows -> + rows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + } + } + + String filesTableSuffix = '$files' + def dataFiles = { String tableName -> + String filesQuery = """ + SELECT file_path + FROM paimon.${dbName}.`${tableName}${filesTableSuffix}` + ORDER BY file_path + """ + spark_paimon(filesQuery).collect { row -> row[0].toString() } + } + + // Read the data file as ordinary Parquet, bypassing Paimon's logical Variant reader. This + // proves that shredding produced typed_value columns rather than merely round-tripping the + // original value/metadata pair. + def rawParquetSource = { String path -> + return """S3( + "uri" = "${path}", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true", + "format" = "parquet" + )""" + } + def rawPayloadType = { String path -> + def columns = sql """DESC FUNCTION ${rawParquetSource(path)}""" + def payloadColumn = columns.find { it[0].toString().equalsIgnoreCase("payload") } + assertTrue(payloadColumn != null, "No payload column in Paimon data file ${path}") + return payloadColumn[1].toString().toLowerCase() + } + + createDorisCatalog() + try { + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + // Cover typed fields, residual object fields, type mismatch fallback, nested ROW/ARRAY, + // root scalars, empty objects, Variant null, and SQL null. + sql """ + INSERT INTO t_variant_shredded VALUES + (1, parse_to_variant('{"age":27,"city":"Beijing","active":true,"profile":{"name":"alice","scores":[10,20]},"other":"kept"}')), + (2, parse_to_variant('{"age":28}')), + (3, parse_to_variant('{"age":"29","active":"true"}')), + (4, parse_to_variant('"scalar"')), + (5, parse_to_variant('{}')), + (6, parse_to_variant('null')), + (7, CAST(NULL AS VARIANT)), + (8, parse_to_variant('{"profile":{"name":"bob","scores":[30],"extra":"nested-kept"},"other":"root-kept"}')) + """ + + // Doris's Paimon reader must unshred typed and residual components back into one logical + // Variant value. + order_qt_variant_explicit_shredding """ + SELECT id, + CAST(payload['age'] AS STRING), + CAST(payload['city'] AS STRING), + CAST(payload['active'] AS BOOLEAN), + CAST(payload['profile']['name'] AS STRING), + CAST(payload['profile']['scores'][1] AS INT), + CAST(payload['profile']['extra'] AS STRING), + CAST(payload['other'] AS STRING), + payload IS NULL + FROM t_variant_shredded + ORDER BY id + """ + + def shreddedFiles = dataFiles("t_variant_shredded") + assertTrue(!shreddedFiles.isEmpty()) + def physicalRows = [] + shreddedFiles.each { filePath -> + String payloadType = rawPayloadType(filePath) + assertTrue(payloadType.contains("metadata:text")) + assertTrue(payloadType.contains("value:text")) + assertTrue(payloadType.contains("typed_value:struct")) + assertTrue(payloadType.contains("age:struct")) + assertTrue(payloadType.contains("profile:struct")) + // The explicit schema wins over inference; residual-only fields must not be promoted. + assertFalse(payloadType.contains("other:struct")) + + physicalRows.addAll(sql(""" + SELECT id, + payload.typed_value.age.typed_value, + CAST(payload.typed_value.age.value IS NOT NULL AS INT), + payload.typed_value.city.typed_value, + CAST(payload.typed_value.active.typed_value AS INT), + payload.typed_value.profile.typed_value.name.typed_value, + payload.typed_value.profile.typed_value.scores.typed_value[1].typed_value, + CAST(payload.value IS NOT NULL AS INT), + CAST(payload.metadata IS NOT NULL AS INT) + FROM ${rawParquetSource(filePath)} + WHERE id IN (1, 3) + """)) + } + physicalRows.sort { left, right -> + Integer.parseInt(left[0].toString()) <=> Integer.parseInt(right[0].toString()) + } + assertEquals([ + ["1", "27", "0", "Beijing", "1", "alice", "10", "1", "1"], + ["3", null, "1", null, null, null, null, "0", "1"] + ], sparkValues(physicalRows)) + + // First create ordinary value/metadata files, then enable shredding for the same table. + // Paimon's reader detects the physical schema per file and must read both layouts together. + sql """ + INSERT INTO t_variant_mixed VALUES + (100, parse_to_variant('{"age":100,"city":"old"}')), + (101, parse_to_variant('{"legacy":"residual"}')) + """ + def unshreddedFiles = dataFiles("t_variant_mixed") + assertTrue(!unshreddedFiles.isEmpty()) + unshreddedFiles.each { filePath -> + String payloadType = rawPayloadType(filePath) + assertTrue(payloadType.contains("value:text")) + assertTrue(payloadType.contains("metadata:text")) + assertFalse(payloadType.contains("typed_value")) + } + + spark_paimon """ + ALTER TABLE paimon.${dbName}.t_variant_mixed + SET TBLPROPERTIES ('parquet.variant.shreddingSchema' = '${shreddingSchema}') + """ + // Reload the serialized Paimon table used by the JNI writer so the next write observes + // the new file-format option. + createDorisCatalog() + sql """ + INSERT INTO t_variant_mixed VALUES + (200, parse_to_variant('{"age":200,"city":"new","extra":"kept"}')), + (201, parse_to_variant('{"age":"201"}')) + """ + + def mixedFiles = dataFiles("t_variant_mixed") + def newlyShreddedFiles = mixedFiles.findAll { !unshreddedFiles.contains(it) } + assertTrue(!newlyShreddedFiles.isEmpty()) + newlyShreddedFiles.each { filePath -> + assertTrue(rawPayloadType(filePath).contains("typed_value:struct")) + } + + order_qt_variant_mixed_layout """ + SELECT id, + CAST(payload['age'] AS STRING), + CAST(payload['city'] AS STRING), + CAST(payload['legacy'] AS STRING), + CAST(payload['extra'] AS STRING) + FROM t_variant_mixed + ORDER BY id + """ + + // Paimon 1.4 can infer one shredding schema per file writer. Doris still sends the same + // logical value/metadata pair; the SDK buffers the rows, chooses typed fields, and writes + // typed_value without a caller-provided schema. + sql """ + INSERT INTO t_variant_inferred VALUES + (300, parse_to_variant('{"age":30,"profile":{"name":"alice"},"extra":"first"}')), + (301, parse_to_variant('{"age":31,"profile":{"name":"bob"},"extra":"second"}')) + """ + def firstInferredFiles = dataFiles("t_variant_inferred") + assertTrue(!firstInferredFiles.isEmpty()) + firstInferredFiles.each { filePath -> + String payloadType = rawPayloadType(filePath) + assertTrue(payloadType.contains("metadata:text")) + assertTrue(payloadType.contains("value:text")) + assertTrue(payloadType.contains("typed_value:struct")) + assertTrue(payloadType.contains("age:struct")) + assertTrue(payloadType.contains("profile:struct")) + assertFalse(payloadType.contains("active:struct")) + assertFalse(payloadType.contains("city:struct")) + } + + // A later Doris statement opens a new Paimon file writer and may infer a different schema. + // Keep write-only enabled so compaction cannot hide the per-file schema difference. + sql """ + INSERT INTO t_variant_inferred VALUES + (400, parse_to_variant('{"city":"Hangzhou","active":true,"extra":"third"}')), + (401, parse_to_variant('{"city":"Shanghai","active":false,"extra":"fourth"}')) + """ + def allInferredFiles = dataFiles("t_variant_inferred") + def secondInferredFiles = allInferredFiles.findAll { !firstInferredFiles.contains(it) } + assertTrue(!secondInferredFiles.isEmpty()) + secondInferredFiles.each { filePath -> + String payloadType = rawPayloadType(filePath) + assertTrue(payloadType.contains("typed_value:struct")) + assertTrue(payloadType.contains("active:struct")) + assertTrue(payloadType.contains("city:struct")) + assertFalse(payloadType.contains("age:struct")) + assertFalse(payloadType.contains("profile:struct")) + } + + // Unshredding is a reader responsibility. It must use each file's physical schema and + // combine typed fields with residual values into one logical Variant column. + order_qt_variant_inferred_shredding """ + SELECT id, + CAST(payload['age'] AS INT), + CAST(payload['profile']['name'] AS STRING), + CAST(payload['city'] AS STRING), + CAST(payload['active'] AS BOOLEAN), + CAST(payload['extra'] AS STRING) + FROM t_variant_inferred + ORDER BY id + """ + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.groovy new file mode 100644 index 00000000000000..35d78c42f26389 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_table_modes.groovy @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_variant_table_modes", "p0,external,paimon,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_variant_modes_catalog" + String dbName = "test_pw_variant_modes_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_pk; + CREATE TABLE paimon.${dbName}.t_variant_pk ( + id INT, + payload VARIANT, + version BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'file.format' = 'parquet' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_dynamic_bucket; + CREATE TABLE paimon.${dbName}.t_variant_dynamic_bucket ( + id INT, + payload VARIANT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'file.format' = 'parquet' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_schema; + CREATE TABLE paimon.${dbName}.t_variant_schema ( + id INT, + name STRING + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_non_variant; + CREATE TABLE paimon.${dbName}.t_non_variant ( + id INT, + payload STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_variant_required; + CREATE TABLE paimon.${dbName}.t_variant_required ( + id INT, + payload VARIANT NOT NULL + ) USING paimon + TBLPROPERTIES ('file.format' = 'parquet'); + """ + + sql """DROP CATALOG IF EXISTS ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + + try { + setFeConfigTemporary([enable_variant_v2: true]) { + assertTrue(getFeConfig("enable_variant_v2").toBoolean()) + sql """SET force_jni_scanner = true""" + // Fixed-bucket primary-key table: later rows replace the same key. + sql """ + INSERT INTO t_variant_pk VALUES + (1, parse_to_variant('{"state":"v1","n":1}'), 1), + (2, parse_to_variant('{"state":"stable","n":2}'), 1), + (1, parse_to_variant('{"state":"v2","n":10}'), 2) + """ + sql """ + INSERT INTO t_variant_pk VALUES + (1, parse_to_variant('{"state":"v3","n":100}'), 3) + """ + order_qt_variant_pk """ + SELECT id, + CAST(payload['state'] AS STRING), + CAST(payload['n'] AS INT), + version + FROM t_variant_pk + ORDER BY id + """ + + // Dynamic bucket routing with Variant values. + sql """ + INSERT INTO t_variant_dynamic_bucket + SELECT number, + parse_to_variant(CONCAT('{"bucket":"dynamic","id":', number, '}')) + FROM numbers("number" = "32") + """ + order_qt_variant_dynamic_bucket """ + SELECT COUNT(*), + SUM(CAST(payload['id'] AS INT)), + SUM(CASE WHEN id <> CAST(payload['id'] AS INT) + THEN 1 ELSE 0 END) + FROM t_variant_dynamic_bucket + """ + + // Schema evolution: add Variant, write it, then add a normal column and continue writing. + sql """INSERT INTO t_variant_schema VALUES (1, 'before')""" + sql """ALTER TABLE t_variant_schema ADD COLUMN payload VARIANT NULL AFTER name""" + sql """ + INSERT INTO t_variant_schema (payload, name, id) VALUES + (parse_to_variant('{"schema":"added"}'), 'after-add', 2) + """ + sql """ALTER TABLE t_variant_schema ADD COLUMN note STRING NULL DEFAULT 'default-note'""" + sql """ + INSERT INTO t_variant_schema (id, name, payload) VALUES + (3, 'after-normal-column', parse_to_variant('{"schema":"continued"}')) + """ + sql """REFRESH TABLE t_variant_schema""" + order_qt_variant_schema_evolution """ + SELECT id, name, + CAST(payload['schema'] AS STRING), + note + FROM t_variant_schema + ORDER BY id + """ + + // Non-Variant Paimon writes remain unchanged while the FE config enables V2. + sql """INSERT INTO t_non_variant VALUES (1, '{"plain":"string"}')""" + order_qt_non_variant """SELECT id, payload FROM t_non_variant ORDER BY id""" + + // Paimon's real NOT NULL schema is enforced by the SDK. + test { + sql """INSERT INTO t_variant_required VALUES (1, CAST(NULL AS VARIANT))""" + exception "Cannot write null to non-null column(payload)" + } + } + } finally { + sql """SET force_jni_scanner = false""" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } +}