diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 9a780f8f04c794..093bc8092c254f 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -39,7 +39,7 @@ jobs: github.event.issue.user.login == 'github-actions[bot]') outputs: thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }} - arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }} + focused_test_changes: ${{ steps.filter.outputs.focused_test_changes }} steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 @@ -55,36 +55,27 @@ jobs: thirdparty_changes: - 'thirdparty/**' - 'env.sh' - arrow_paimon_lifecycle_changes: - - 'build.sh' - - 'env.sh' - - 'thirdparty/arrow-paimon-vars.sh' - - 'thirdparty/vars.sh' - - 'thirdparty/build-thirdparty.sh' - - 'thirdparty/download-thirdparty.sh' - - 'thirdparty/paimon-cpp-cache.cmake' - - 'thirdparty/patches/apache-arrow-*.patch' - - 'thirdparty/patches/paimon-cpp-*.patch' + focused_test_changes: - 'regression-test/pipeline/external/conf/fe.conf' - - 'thirdparty/test/arrow-paimon-lifecycle-test.sh' - 'thirdparty/test/adbc-jni-config-test.sh' - - 'thirdparty/test/paimon-codec-dependency-test.sh' - '.github/workflows/build-thirdparty.yml' - arrow_paimon_lifecycle_test: - name: Arrow/Paimon Lifecycle Test + script_test: + name: Thirdparty Script Test needs: changes - if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }} + if: ${{ needs.changes.outputs.thirdparty_changes == 'true' || needs.changes.outputs.focused_test_changes == 'true' }} runs-on: ubuntu-22.04 steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 - - name: Test focused thirdparty lifecycle + - name: Test download-thirdparty.sh + run: | + bash thirdparty/test/download-thirdparty-fallback-test.sh + + - name: Test thirdparty configuration run: | - thirdparty/test/arrow-paimon-lifecycle-test.sh thirdparty/test/adbc-jni-config-test.sh - thirdparty/test/paimon-codec-dependency-test.sh build_linux: name: Build Third Party Libraries (Linux) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 26b0055f3cf536..7808bab13d6bd0 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -147,8 +147,6 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF) message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") -option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) -set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -166,14 +164,6 @@ if (ENABLE_VARIANT_NESTED_GROUP AND "${VARIANT_NESTED_GROUP_MODULE_DIR}" STREQUA message(FATAL_ERROR "ENABLE_VARIANT_NESTED_GROUP requires VARIANT_NESTED_GROUP_MODULE_DIR") endif() -# Allow env to override when reconfiguring (avoid picking /usr/local). -if (DEFINED ENV{ENABLE_PAIMON_CPP}) - set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) -endif() -if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) - set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) -endif() - set(CMAKE_SKIP_RPATH TRUE) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) @@ -597,10 +587,6 @@ set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ) -if (ENABLE_PAIMON_CPP) - message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode") -endif() - if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) @@ -660,19 +646,6 @@ if (BUILD_BENCHMARK) ) endif() -set(PAIMON_FACTORY_REGISTRY_LIBS) -if (ENABLE_PAIMON_CPP) - # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via - # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). - # No dual-stack selection needed — single Arrow for everything. - - # paimon_parquet_file_format depends on Arrow Dataset symbols. - # Force-link it with --whole-archive so its factory registration runs. - set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format) - list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS}) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_START_GROUP} @@ -690,29 +663,6 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib) -if (ENABLE_PAIMON_CPP) - if (PAIMON_FACTORY_REGISTRY_LIBS) - if (APPLE) - foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS}) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,-force_load,$) - endforeach() - else() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,--whole-archive - ${PAIMON_FACTORY_REGISTRY_LIBS} - -Wl,--no-whole-archive) - endif() - endif() - - # paimon-cpp internal dependencies (renamed with _paimon suffix) - # These must come after paimon libraries to resolve symbols. - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP}) # Add all external dependencies. They should come after the palo libs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index eed1823515f492..ac78b4cd2d7429 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -187,18 +187,3 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) - -if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64) - add_thirdparty(paimon_parquet_file_format LIB64) - add_thirdparty(paimon_orc_file_format LIB64) - add_thirdparty(paimon_blob_file_format LIB64) - add_thirdparty(paimon_local_file_system LIB64) - add_thirdparty(paimon_file_index LIB64) - add_thirdparty(paimon_global_index LIB64) - - add_thirdparty(roaring_bitmap_paimon LIB64) - add_thirdparty(xxhash_paimon LIB64) - add_thirdparty(fmt_paimon LIB64) - add_thirdparty(tbb_paimon LIB64) -endif() diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 43f329d85c3d5b..8a862be3c7da56 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1636,41 +1636,6 @@ DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, [](int64_t bytes) -> bool { return bytes > 0; }); -// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. -// Each entry uses the format "=", and file_type must be one of: -// local, hdfs, s3, http, broker. -DEFINE_Strings(paimon_file_system_scheme_mappings, - "file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs," - "s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3," - "abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http," - "ofs=broker,gfs=broker"); -DEFINE_Validator(paimon_file_system_scheme_mappings, - ([](const std::vector& mappings) -> bool { - doris::StringCaseUnorderedSet seen_schemes; - static const doris::StringCaseUnorderedSet supported_types = { - "local", "hdfs", "s3", "http", "broker"}; - for (const auto& raw_entry : mappings) { - std::string_view entry = doris::trim(raw_entry); - size_t separator = entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - std::string scheme = std::string(doris::trim(entry.substr(0, separator))); - std::string file_type = - std::string(doris::trim(entry.substr(separator + 1))); - if (scheme.empty() || file_type.empty()) { - return false; - } - if (supported_types.find(file_type) == supported_types.end()) { - return false; - } - if (!seen_schemes.insert(scheme).second) { - return false; - } - } - return true; - })); - DEFINE_mInt32(thrift_client_open_num_tries, "1"); DEFINE_Bool(enable_index_compaction, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 1e50e539d96bed..e9f64c2c4290cd 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1698,9 +1698,6 @@ DECLARE_mInt64(iceberg_sink_max_file_size); // Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); -/** Paimon file system configurations **/ -DECLARE_Strings(paimon_file_system_scheme_mappings); - // Number of open tries, default 1 means only try to open once. // Retry the Open num_retries time waiting 100 milliseconds between retries. DECLARE_mInt32(thrift_client_open_num_tries); diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 5c7beb3e87e268..209d7db1d7527a 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -73,9 +73,7 @@ #include "format/table/iceberg_sys_table_jni_reader.h" #include "format/table/lakesoul_jni_reader.h" #include "format/table/max_compute_jni_reader.h" -#include "format/table/paimon_cpp_reader.h" #include "format/table/paimon_jni_reader.h" -#include "format/table/paimon_predicate_converter.h" #include "format/table/paimon_reader.h" #include "format/table/partition_column_filler.h" #include "format/table/remote_doris_reader.h" @@ -1111,25 +1109,9 @@ Status FileScanner::_get_next_reader() { _cur_reader = std::move(mc_reader); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "paimon") { - if (_state->query_options().__isset.enable_paimon_cpp_reader && - _state->query_options().enable_paimon_cpp_reader) { - auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state, - _profile, range, _params); - cpp_reader->set_push_down_agg_type(_get_push_down_agg_type()); - if (!_is_load && !_push_down_conjuncts.empty()) { - PaimonPredicateConverter predicate_converter(_file_slot_descs, _state); - auto predicate = predicate_converter.build(_push_down_conjuncts); - if (predicate) { - cpp_reader->set_predicate(std::move(predicate)); - } - } - init_status = cpp_reader->init_reader(); - _cur_reader = std::move(cpp_reader); - } else { - _cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile, - range, _params); - init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader(); - } + _cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile, + range, _params); + init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader(); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "hudi") { _cur_reader = HudiJniReader::create_unique(*_params, diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 77cf7a06fee747..ac775fd94bff76 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -145,12 +145,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { } const auto& params = range.table_format_params.paimon_params; if (params.__isset.reader_type) { - if (params.reader_type == TPaimonReaderType::PAIMON_JNI) { - return params.__isset.paimon_split; - } - // V2 cannot pass a logical DataSplit through a raw native child without silently - // dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback. - return false; + return params.reader_type == TPaimonReaderType::PAIMON_JNI && + params.__isset.paimon_split; } if (params.__isset.paimon_split) { // Before reader_type was added, an encoded split unambiguously selected the Java diff --git a/be/src/format/table/paimon_cpp_reader.cpp b/be/src/format/table/paimon_cpp_reader.cpp deleted file mode 100644 index 944c316bff2129..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.cpp +++ /dev/null @@ -1,342 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "arrow/record_batch.h" -#include "arrow/result.h" -#include "core/block/block.h" -#include "core/block/column_with_type_and_name.h" -#include "format/table/paimon_doris_file_system.h" -#include "paimon/defs.h" -#include "paimon/memory/memory_pool.h" -#include "paimon/read_context.h" -#include "paimon/table/source/table_read.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/url_coding.h" - -namespace doris { -#include "common/compile_check_begin.h" - -namespace { -constexpr const char* VALUE_KIND_FIELD = "_VALUE_KIND"; - -} // namespace - -PaimonCppReader::PaimonCppReader(const std::vector& file_slot_descs, - RuntimeState* state, RuntimeProfile* profile, - const TFileRangeDesc& range, - const TFileScanRangeParams* range_params) - : _file_slot_descs(file_slot_descs), - _state(state), - _profile(profile), - _range(range), - _range_params(range_params) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz); - if (range.__isset.table_format_params && - range.table_format_params.__isset.table_level_row_count) { - _remaining_table_level_row_count = range.table_format_params.table_level_row_count; - } else { - _remaining_table_level_row_count = -1; - } -} - -PaimonCppReader::~PaimonCppReader() = default; - -Status PaimonCppReader::init_reader() { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - return Status::OK(); - } - return _init_paimon_reader(); -} - -Status PaimonCppReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - auto rows = std::min(_remaining_table_level_row_count, - (int64_t)_state->query_options().batch_size); - _remaining_table_level_row_count -= rows; - auto mutable_columns_guard = block->mutate_columns_scoped(); - auto& mutate_columns = mutable_columns_guard.mutable_columns(); - for (auto& col : mutate_columns) { - col->resize(rows); - } - *read_rows = rows; - *eof = false; - if (_remaining_table_level_row_count == 0) { - *eof = true; - } - return Status::OK(); - } - - if (!_batch_reader) { - return Status::InternalError("paimon-cpp reader is not initialized"); - } - - if (_col_name_to_block_idx.empty()) { - _col_name_to_block_idx = block->get_name_to_pos_map(); - } - - auto batch_result = _batch_reader->NextBatch(); - if (!batch_result.ok()) { - return Status::InternalError("paimon-cpp read batch failed: {}", - batch_result.status().ToString()); - } - auto batch = std::move(batch_result).value(); - if (paimon::BatchReader::IsEofBatch(batch)) { - *read_rows = 0; - *eof = true; - return Status::OK(); - } - - arrow::Result> import_result = - arrow::ImportRecordBatch(batch.first.get(), batch.second.get()); - if (!import_result.ok()) { - return Status::InternalError("failed to import paimon-cpp arrow batch: {}", - import_result.status().message()); - } - - auto record_batch = std::move(import_result).ValueUnsafe(); - const auto num_rows = static_cast(record_batch->num_rows()); - const auto num_columns = record_batch->num_columns(); - auto columns_guard = block->mutate_columns_scoped(); - auto& columns = columns_guard.mutable_columns(); - for (int c = 0; c < num_columns; ++c) { - const auto& field = record_batch->schema()->field(c); - if (field->name() == VALUE_KIND_FIELD) { - continue; - } - - auto it = _col_name_to_block_idx.find(field->name()); - if (it == _col_name_to_block_idx.end()) { - // Skip columns that are not in the block (e.g., partition columns handled elsewhere) - continue; - } - const auto block_pos = it->second; - try { - RETURN_IF_ERROR(columns_guard.get_datatype_by_position(block_pos) - ->get_serde() - ->read_column_from_arrow(*columns[block_pos], - record_batch->column(c).get(), 0, - num_rows, _ctzz)); - } catch (Exception& e) { - return Status::InternalError("Failed to convert from arrow to block: {}", e.what()); - } - } - - *read_rows = num_rows; - *eof = false; - return Status::OK(); -} - -Status PaimonCppReader::get_columns(std::unordered_map* name_to_type, - std::unordered_set* missing_cols) { - for (const auto& slot : _file_slot_descs) { - name_to_type->emplace(slot->col_name(), slot->type()); - } - return Status::OK(); -} - -Status PaimonCppReader::close() { - if (_batch_reader) { - _batch_reader->Close(); - } - return Status::OK(); -} - -Status PaimonCppReader::_init_paimon_reader() { - register_paimon_doris_file_system(); - RETURN_IF_ERROR(_decode_split(&_split)); - - auto table_path_opt = _resolve_table_path(); - if (!table_path_opt.has_value()) { - return Status::InternalError( - "paimon-cpp missing paimon_table; cannot resolve paimon table root path"); - } - auto options = _build_options(); - auto read_columns = _build_read_columns(); - - // Avoid moving strings across module boundaries to prevent allocator mismatches in ASAN builds. - std::string table_path = table_path_opt.value(); - static std::once_flag options_log_once; - std::call_once(options_log_once, [&]() { - auto has_key = [&](const char* key) { - auto it = options.find(key); - return (it != options.end() && !it->second.empty()) ? "set" : "empty"; - }; - auto value_or = [&](const char* key) { - auto it = options.find(key); - return it != options.end() ? it->second : std::string(""); - }; - LOG(INFO) << "paimon-cpp options summary: table_path=" << table_path - << " AWS_ACCESS_KEY=" << has_key("AWS_ACCESS_KEY") - << " AWS_SECRET_KEY=" << has_key("AWS_SECRET_KEY") - << " AWS_TOKEN=" << has_key("AWS_TOKEN") - << " AWS_ENDPOINT=" << value_or("AWS_ENDPOINT") - << " AWS_REGION=" << value_or("AWS_REGION") - << " use_path_style=" << value_or("use_path_style") - << " fs.oss.endpoint=" << value_or("fs.oss.endpoint") - << " fs.s3a.endpoint=" << value_or("fs.s3a.endpoint"); - }); - paimon::ReadContextBuilder builder(table_path); - if (!read_columns.empty()) { - builder.SetReadSchema(read_columns); - } - if (!options.empty()) { - builder.SetOptions(options); - } - if (_predicate) { - builder.SetPredicate(_predicate); - builder.EnablePredicateFilter(true); - } - - auto context_result = builder.Finish(); - if (!context_result.ok()) { - return Status::InternalError("paimon-cpp build read context failed: {}", - context_result.status().ToString()); - } - auto context = std::move(context_result).value(); - - auto table_read_result = paimon::TableRead::Create(std::move(context)); - if (!table_read_result.ok()) { - return Status::InternalError("paimon-cpp create table read failed: {}", - table_read_result.status().ToString()); - } - auto table_read = std::move(table_read_result).value(); - auto reader_result = table_read->CreateReader(_split); - if (!reader_result.ok()) { - return Status::InternalError("paimon-cpp create reader failed: {}", - reader_result.status().ToString()); - } - _table_read = std::move(table_read); - _batch_reader = std::move(reader_result).value(); - return Status::OK(); -} - -Status PaimonCppReader::_decode_split(std::shared_ptr* split) { - if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.paimon_params || - !_range.table_format_params.paimon_params.__isset.paimon_split) { - return Status::InternalError("paimon-cpp missing paimon_split in scan range"); - } - const auto& encoded_split = _range.table_format_params.paimon_params.paimon_split; - std::string decoded_split; - if (!base64_decode(encoded_split, &decoded_split)) { - return Status::InternalError("paimon-cpp base64 decode paimon_split failed"); - } - auto pool = paimon::GetDefaultPool(); - auto split_result = - paimon::Split::Deserialize(decoded_split.data(), decoded_split.size(), pool); - if (!split_result.ok()) { - return Status::InternalError("paimon-cpp deserialize split failed: {}", - split_result.status().ToString()); - } - *split = std::move(split_result).value(); - return Status::OK(); -} - -std::optional PaimonCppReader::_resolve_table_path() const { - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_table && - !_range.table_format_params.paimon_params.paimon_table.empty()) { - return _range.table_format_params.paimon_params.paimon_table; - } - return std::nullopt; -} - -std::vector PaimonCppReader::_build_read_columns() const { - std::vector columns; - columns.reserve(_file_slot_descs.size()); - for (const auto& slot : _file_slot_descs) { - columns.emplace_back(slot->col_name()); - } - return columns; -} - -std::map PaimonCppReader::_build_options() const { - std::map options; - if (_range_params && _range_params->__isset.paimon_options && - !_range_params->paimon_options.empty()) { - options.insert(_range_params->paimon_options.begin(), _range_params->paimon_options.end()); - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_options) { - options.insert(_range.table_format_params.paimon_params.paimon_options.begin(), - _range.table_format_params.paimon_params.paimon_options.end()); - } - - if (_range_params && _range_params->__isset.properties && !_range_params->properties.empty()) { - for (const auto& kv : _range_params->properties) { - options[kv.first] = kv.second; - } - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.hadoop_conf) { - for (const auto& kv : _range.table_format_params.paimon_params.hadoop_conf) { - options[kv.first] = kv.second; - } - } - - auto copy_if_missing = [&](const char* from_key, const char* to_key) { - if (options.find(to_key) != options.end()) { - return; - } - auto it = options.find(from_key); - if (it != options.end() && !it->second.empty()) { - options[to_key] = it->second; - } - }; - - // Map common OSS/S3 Hadoop configs to Doris S3 property keys. - copy_if_missing("fs.oss.accessKeyId", "AWS_ACCESS_KEY"); - copy_if_missing("fs.oss.accessKeySecret", "AWS_SECRET_KEY"); - copy_if_missing("fs.oss.sessionToken", "AWS_TOKEN"); - copy_if_missing("fs.oss.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.oss.region", "AWS_REGION"); - copy_if_missing("fs.s3a.access.key", "AWS_ACCESS_KEY"); - copy_if_missing("fs.s3a.secret.key", "AWS_SECRET_KEY"); - copy_if_missing("fs.s3a.session.token", "AWS_TOKEN"); - copy_if_missing("fs.s3a.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.s3a.region", "AWS_REGION"); - copy_if_missing("fs.s3a.path.style.access", "use_path_style"); - - // Backfill file.format/manifest.format from split file_format to avoid - // paimon-cpp falling back to default manifest.format=avro. - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.file_format && - !_range.table_format_params.paimon_params.file_format.empty()) { - const auto& split_file_format = _range.table_format_params.paimon_params.file_format; - auto file_format_it = options.find(paimon::Options::FILE_FORMAT); - if (file_format_it == options.end() || file_format_it->second.empty()) { - options[paimon::Options::FILE_FORMAT] = split_file_format; - } - auto manifest_format_it = options.find(paimon::Options::MANIFEST_FORMAT); - if (manifest_format_it == options.end() || manifest_format_it->second.empty()) { - options[paimon::Options::MANIFEST_FORMAT] = split_file_format; - } - } - - options[paimon::Options::FILE_SYSTEM] = "doris"; - return options; -} - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/format/table/paimon_cpp_reader.h b/be/src/format/table/paimon_cpp_reader.h deleted file mode 100644 index d567b1b24bb71a..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.h +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "common/status.h" -#include "format/generic_reader.h" -#include "paimon/reader/batch_reader.h" -#include "paimon/table/source/split.h" -#include "storage/olap_scan_common.h" - -namespace paimon { -class TableRead; -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeProfile; -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { -#include "common/compile_check_begin.h" - -class Block; - -class PaimonCppReader : public GenericReader { - ENABLE_FACTORY_CREATOR(PaimonCppReader); - -public: - PaimonCppReader(const std::vector& file_slot_descs, RuntimeState* state, - RuntimeProfile* profile, const TFileRangeDesc& range, - const TFileScanRangeParams* range_params); - ~PaimonCppReader() override; - - Status init_reader(); - Status get_next_block(Block* block, size_t* read_rows, bool* eof) override; - Status get_columns(std::unordered_map* name_to_type, - std::unordered_set* missing_cols) override; - Status close() override; - void set_predicate(std::shared_ptr predicate) { - _predicate = std::move(predicate); - } - -private: - Status _init_paimon_reader(); - Status _decode_split(std::shared_ptr* split); - // Resolve paimon table root path for schema/manifest lookup. - std::optional _resolve_table_path() const; - std::vector _build_read_columns() const; - std::map _build_options() const; - - const std::vector& _file_slot_descs; - RuntimeState* _state = nullptr; - [[maybe_unused]] RuntimeProfile* _profile = nullptr; - const TFileRangeDesc& _range; - const TFileScanRangeParams* _range_params = nullptr; - - std::shared_ptr _split; - std::unique_ptr _table_read; - std::unique_ptr _batch_reader; - std::shared_ptr _predicate; - - std::unordered_map _col_name_to_block_idx; - int64_t _remaining_table_level_row_count = -1; - cctz::time_zone _ctzz; -}; - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.cpp b/be/src/format/table/paimon_doris_file_system.cpp deleted file mode 100644 index bd303b5e26afb8..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.cpp +++ /dev/null @@ -1,679 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common/config.h" -#include "common/status.h" -#include "io/file_factory.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_system.h" -#include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" -#include "paimon/factories/factory.h" -#include "paimon/fs/file_system.h" -#include "paimon/fs/file_system_factory.h" -#include "paimon/result.h" -#include "paimon/status.h" -#include "util/string_util.h" - -namespace paimon { - -struct ParsedUri { - std::string scheme; - std::string authority; -}; - -ParsedUri parse_uri(const std::string& path) { - ParsedUri parsed; - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos || scheme_pos == 0) { - return parsed; - } - parsed.scheme = doris::to_lower(path.substr(0, scheme_pos)); - size_t authority_start = scheme_pos + delim_len; - if (authority_start >= path.size() || path[authority_start] == '/') { - return parsed; - } - size_t next_slash = path.find('/', authority_start); - if (next_slash == std::string::npos) { - parsed.authority = path.substr(authority_start); - } else { - parsed.authority = path.substr(authority_start, next_slash - authority_start); - } - return parsed; -} - -bool parse_scheme_mapping_target(std::string_view raw_target, doris::TFileType::type* type) { - std::string target = doris::to_lower(std::string(doris::trim(raw_target))); - if (target == "local") { - *type = doris::TFileType::FILE_LOCAL; - return true; - } - if (target == "hdfs") { - *type = doris::TFileType::FILE_HDFS; - return true; - } - if (target == "s3") { - *type = doris::TFileType::FILE_S3; - return true; - } - if (target == "http") { - *type = doris::TFileType::FILE_HTTP; - return true; - } - if (target == "broker") { - *type = doris::TFileType::FILE_BROKER; - return true; - } - return false; -} - -bool parse_scheme_mapping_entry(std::string_view raw_entry, std::string* scheme, - doris::TFileType::type* type) { - size_t separator = raw_entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - *scheme = doris::to_lower(std::string(doris::trim(raw_entry.substr(0, separator)))); - if (scheme->empty()) { - return false; - } - return parse_scheme_mapping_target(raw_entry.substr(separator + 1), type); -} - -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme) { - if (scheme.empty()) { - return doris::TFileType::FILE_HDFS; - } - std::string normalized_scheme = doris::to_lower(scheme); - for (const auto& mapping_entry : doris::config::paimon_file_system_scheme_mappings) { - std::string configured_scheme; - doris::TFileType::type configured_type; - if (!parse_scheme_mapping_entry(mapping_entry, &configured_scheme, &configured_type)) { - continue; - } - if (configured_scheme == normalized_scheme) { - return configured_type; - } - } - return doris::TFileType::FILE_HDFS; -} - -std::string replace_scheme(const std::string& path, const std::string& scheme) { - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return path; - } - return scheme + "://" + path.substr(scheme_pos + delim_len); -} - -std::string normalize_local_path(const std::string& path) { - if (!path.starts_with("file:")) { - return path; - } - constexpr size_t file_prefix_len = 5; - size_t start = file_prefix_len; - if (path.compare(start, 2, "//") == 0 && path.size() - start > 2) { - size_t next_slash = path.find('/', start + 2); - if (next_slash == std::string::npos) { - return ""; - } - start = next_slash; - } - return path.substr(start); -} - -std::string normalize_path_for_type(const std::string& path, const std::string& scheme, - doris::TFileType::type type) { - if (type == doris::TFileType::FILE_LOCAL) { - return normalize_local_path(path); - } - if (type == doris::TFileType::FILE_S3 && scheme != "s3") { - return replace_scheme(path, "s3"); - } - return path; -} - -std::string build_fs_cache_key(doris::TFileType::type type, const ParsedUri& uri, - const std::string& default_fs_name) { - switch (type) { - case doris::TFileType::FILE_LOCAL: - return "local"; - case doris::TFileType::FILE_S3: - return "s3://" + uri.authority; - case doris::TFileType::FILE_HTTP: - return "http://" + uri.authority; - case doris::TFileType::FILE_BROKER: - return "broker"; - case doris::TFileType::FILE_HDFS: - default: - if (!uri.scheme.empty() || !uri.authority.empty()) { - return uri.scheme + "://" + uri.authority; - } - return default_fs_name; - } -} - -paimon::Status to_paimon_status(const doris::Status& status) { - if (status.ok()) { - return paimon::Status::OK(); - } - switch (status.code()) { - case doris::ErrorCode::NOT_FOUND: - case doris::ErrorCode::DIR_NOT_EXIST: - return paimon::Status::NotExist(status.to_string()); - case doris::ErrorCode::ALREADY_EXIST: - case doris::ErrorCode::FILE_ALREADY_EXIST: - return paimon::Status::Exist(status.to_string()); - case doris::ErrorCode::INVALID_ARGUMENT: - case doris::ErrorCode::INVALID_INPUT_SYNTAX: - return paimon::Status::Invalid(status.to_string()); - case doris::ErrorCode::NOT_IMPLEMENTED_ERROR: - return paimon::Status::NotImplemented(status.to_string()); - default: - return paimon::Status::IOError(status.to_string()); - } -} - -std::string join_path(const std::string& base, const std::string& child) { - if (base.empty()) { - return child; - } - if (base.back() == '/') { - return base + child; - } - return base + "/" + child; -} - -std::string parent_path_no_scheme(const std::string& path) { - if (path.empty()) { - return ""; - } - size_t end = path.size(); - while (end > 1 && path[end - 1] == '/') { - --end; - } - size_t pos = path.rfind('/', end - 1); - if (pos == std::string::npos) { - return ""; - } - if (pos == 0) { - return "/"; - } - return path.substr(0, pos); -} - -std::string parent_path(const std::string& path) { - ParsedUri uri = parse_uri(path); - if (uri.scheme.empty()) { - return parent_path_no_scheme(path); - } - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return parent_path_no_scheme(path); - } - size_t start = scheme_pos + delim_len; - size_t slash = path.find('/', start); - if (slash == std::string::npos) { - return ""; - } - std::string path_part = path.substr(slash); - std::string parent_part = parent_path_no_scheme(path_part); - if (parent_part.empty()) { - return ""; - } - std::string prefix = uri.scheme + "://"; - if (!uri.authority.empty()) { - prefix += uri.authority; - } - return prefix + parent_part; -} - -class DorisInputStream : public InputStream { -public: - DorisInputStream(doris::io::FileReaderSPtr reader, std::string path) - : reader_(std::move(reader)), path_(std::move(path)) {} - - Status Seek(int64_t offset, SeekOrigin origin) override { - int64_t target = 0; - if (origin == SeekOrigin::FS_SEEK_SET) { - target = offset; - } else if (origin == SeekOrigin::FS_SEEK_CUR) { - target = position_ + offset; - } else if (origin == SeekOrigin::FS_SEEK_END) { - target = static_cast(reader_->size()) + offset; - } else { - return Status::Invalid("unknown seek origin"); - } - if (target < 0) { - return Status::Invalid("seek position is negative"); - } - position_ = target; - return Status::OK(); - } - - Result GetPos() const override { return position_; } - - Result Read(char* buffer, uint32_t size) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(position_, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - position_ += static_cast(bytes_read); - return static_cast(bytes_read); - } - - Result Read(char* buffer, uint32_t size, uint64_t offset) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(offset, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(bytes_read); - } - - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, - std::function&& callback) override { - Result result = Read(buffer, size, offset); - Status status = Status::OK(); - if (!result.ok()) { - status = result.status(); - } - callback(status); - } - - Result GetUri() const override { return path_; } - - Result Length() const override { return static_cast(reader_->size()); } - - Status Close() override { return to_paimon_status(reader_->close()); } - -private: - doris::io::FileReaderSPtr reader_; - std::string path_; - int64_t position_ = 0; -}; - -class DorisOutputStream : public OutputStream { -public: - DorisOutputStream(doris::io::FileWriterPtr writer, std::string path) - : writer_(std::move(writer)), path_(std::move(path)) {} - - Result Write(const char* buffer, uint32_t size) override { - doris::Status status = writer_->append(doris::Slice(buffer, size)); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(size); - } - - Status Flush() override { return Status::OK(); } - - Result GetPos() const override { - return static_cast(writer_->bytes_appended()); - } - - Result GetUri() const override { return path_; } - - Status Close() override { return to_paimon_status(writer_->close()); } - -private: - doris::io::FileWriterPtr writer_; - std::string path_; -}; - -class DorisBasicFileStatus : public BasicFileStatus { -public: - DorisBasicFileStatus(std::string path, bool is_dir) : path_(std::move(path)), is_dir_(is_dir) {} - - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - -private: - std::string path_; - bool is_dir_; -}; - -class DorisFileStatus : public FileStatus { -public: - DorisFileStatus(std::string path, bool is_dir, uint64_t length, int64_t mtime) - : path_(std::move(path)), is_dir_(is_dir), length_(length), mtime_(mtime) {} - - uint64_t GetLen() const override { return length_; } - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - int64_t GetModificationTime() const override { return mtime_; } - -private: - std::string path_; - bool is_dir_; - uint64_t length_; - int64_t mtime_; -}; - -class DorisFileSystem : public FileSystem { -public: - explicit DorisFileSystem(std::map options) - : options_(std::move(options)) { - auto it = options_.find("fs.defaultFS"); - if (it != options_.end()) { - default_fs_name_ = it->second; - } - } - - Result> Open(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - doris::io::FileReaderSPtr reader; - doris::io::FileReaderOptions reader_options = doris::io::FileReaderOptions::DEFAULT; - doris::Status status = fs->open_file(normalized_path, &reader, &reader_options); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(reader), normalized_path); - } - - Result> Create(const std::string& path, - bool overwrite) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - if (!overwrite) { - bool exists = false; - doris::Status exists_status = fs->exists(normalized_path, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (exists) { - return Status::Exist("file already exists: ", normalized_path); - } - } - std::string parent = parent_path(normalized_path); - if (!parent.empty()) { - doris::Status mkdir_status = fs->create_directory(parent); - if (!mkdir_status.ok()) { - return to_paimon_status(mkdir_status); - } - } - doris::io::FileWriterPtr writer; - doris::Status status = fs->create_file(normalized_path, &writer); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(writer), normalized_path); - } - - Status Mkdirs(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - doris::Status status = resolved.first->create_directory(resolved.second); - return to_paimon_status(status); - } - - Status Rename(const std::string& src, const std::string& dst) const override { - PAIMON_ASSIGN_OR_RAISE(auto src_resolved, resolve_path(src)); - PAIMON_ASSIGN_OR_RAISE(auto dst_resolved, resolve_path(dst)); - doris::Status status = src_resolved.first->rename(src_resolved.second, dst_resolved.second); - return to_paimon_status(status); - } - - Status Delete(const std::string& path, bool recursive = true) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - return Status::OK(); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return to_paimon_status(resolved.first->delete_file(resolved.second)); - } - if (recursive) { - return to_paimon_status(resolved.first->delete_directory(resolved.second)); - } - return to_paimon_status(size_status); - } - - Result> GetFileStatus(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - if (type != doris::TFileType::FILE_S3) { - return Status::NotExist("path not exists: ", resolved.second); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return std::make_unique(resolved.second, false, - static_cast(size), 0); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - - Status ListDir(const std::string& directory, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(directory)); - auto file_status = GetFileStatus(directory); - if (file_status.ok() && !file_status.value()->IsDir()) { - return Status::IOError("path is not a directory: ", directory); - } - std::vector files; - bool exists = false; - doris::Status status = resolved.first->list(resolved.second, false, &files, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file)); - } - return Status::OK(); - } - - Status ListFileStatus(const std::string& path, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto self_status = GetFileStatus(path); - if (!self_status.ok()) { - if (self_status.status().IsNotExist()) { - return Status::OK(); - } - return self_status.status(); - } - if (!self_status.value()->IsDir()) { - status_list->emplace_back(std::move(self_status).value()); - return Status::OK(); - } - std::vector files; - bool exists = false; - doris::Status list_status = resolved.first->list(resolved.second, false, &files, &exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - uint64_t length = file.is_file ? static_cast(file.file_size) : 0; - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file, length, 0)); - } - return Status::OK(); - } - - Result Exists(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status status = resolved.first->exists(resolved.second, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists && type == doris::TFileType::FILE_S3) { - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - return list_exists || !files.empty(); - } - return exists; - } - -private: - Result> resolve_path( - const std::string& path) const { - auto uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - std::string normalized_path = normalize_path_for_type(path, uri.scheme, type); - if (type == doris::TFileType::FILE_LOCAL) { - doris::io::FileSystemSPtr fs = doris::io::global_local_filesystem(); - return std::make_pair(std::move(fs), normalized_path); - } - std::string fs_key = build_fs_cache_key(type, uri, default_fs_name_); - { - std::lock_guard lock(fs_lock_); - auto it = fs_cache_.find(fs_key); - if (it != fs_cache_.end()) { - return std::make_pair(it->second, normalized_path); - } - } - doris::io::FSPropertiesRef fs_properties(type); - const std::map* properties = &options_; - std::map properties_override; - if (type == doris::TFileType::FILE_HTTP && !options_.contains("uri") && - !uri.scheme.empty()) { - properties_override = options_; - properties_override["uri"] = uri.scheme + "://" + uri.authority; - properties = &properties_override; - } - fs_properties.properties = properties; - if (!broker_addresses_.empty()) { - fs_properties.broker_addresses = &broker_addresses_; - } - doris::io::FileDescription file_description = { - .path = normalized_path, .file_size = -1, .mtime = 0, .fs_name = default_fs_name_}; - auto fs_result = doris::FileFactory::create_fs(fs_properties, file_description); - if (!fs_result.has_value()) { - return to_paimon_status(fs_result.error()); - } - doris::io::FileSystemSPtr fs = std::move(fs_result).value(); - { - std::lock_guard lock(fs_lock_); - fs_cache_.emplace(std::move(fs_key), fs); - } - return std::make_pair(std::move(fs), std::move(normalized_path)); - } - - std::map options_; - std::vector broker_addresses_; - std::string default_fs_name_; - mutable std::mutex fs_lock_; - mutable std::unordered_map fs_cache_; -}; - -class DorisFileSystemFactory : public FileSystemFactory { -public: - static const char IDENTIFIER[]; - - const char* Identifier() const override { return IDENTIFIER; } - - Result> Create( - const std::string& path, - const std::map& options) const override { - return std::make_unique(options); - } -}; - -const char DorisFileSystemFactory::IDENTIFIER[] = "doris"; - -REGISTER_PAIMON_FACTORY(DorisFileSystemFactory); - -} // namespace paimon - -namespace doris { - -void register_paimon_doris_file_system() {} - -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.h b/be/src/format/table/paimon_doris_file_system.h deleted file mode 100644 index 561e7aeac30fe4..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include - -namespace paimon { - -// Visible for tests: maps a URI scheme to the Doris file type used by paimon-cpp. -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme); - -} // namespace paimon - -namespace doris { - -// Force-link helper so the paimon-cpp file system factory registration is kept. -void register_paimon_doris_file_system(); - -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.cpp b/be/src/format/table/paimon_predicate_converter.cpp deleted file mode 100644 index fadf1812e3039b..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.cpp +++ /dev/null @@ -1,659 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_predicate_converter.h" - -#include -#include -#include - -#include "core/column/column_const.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/field.h" -#include "core/types.h" -#include "core/value/decimalv2_value.h" -#include "core/value/timestamptz_value.h" -#include "core/value/vdatetime_value.h" -#include "exprs/vcompound_pred.h" -#include "exprs/vdirect_in_predicate.h" -#include "exprs/vectorized_fn_call.h" -#include "exprs/vexpr.h" -#include "exprs/vin_predicate.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "paimon/data/decimal.h" -#include "paimon/data/timestamp.h" -#include "paimon/predicate/predicate_builder.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/timezone_utils.h" - -namespace doris { -#include "common/compile_check_begin.h" - -PaimonPredicateConverter::PaimonPredicateConverter( - const std::vector& file_slot_descs, RuntimeState* state) - : _state(state) { - _field_index_by_name.reserve(file_slot_descs.size()); - for (size_t i = 0; i < file_slot_descs.size(); ++i) { - const auto& name = file_slot_descs[i]->col_name(); - auto normalized = _normalize_name(name); - if (_field_index_by_name.find(normalized) == _field_index_by_name.end()) { - _field_index_by_name.emplace(std::move(normalized), static_cast(i)); - } - } - - if (!TimezoneUtils::find_cctz_time_zone("GMT", _gmt_tz)) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _gmt_tz); - } -} - -std::shared_ptr PaimonPredicateConverter::build( - const VExprContextSPtrs& conjuncts) { - std::vector> predicates; - predicates.reserve(conjuncts.size()); - for (const auto& conjunct : conjuncts) { - if (!conjunct || !conjunct->root()) { - continue; - } - auto root = conjunct->root(); - if (root->is_rf_wrapper()) { - if (auto impl = root->get_impl()) { - root = impl; - } - } - auto predicate = _convert_expr(root); - if (predicate) { - predicates.emplace_back(std::move(predicate)); - } - } - - if (predicates.empty()) { - return nullptr; - } - if (predicates.size() == 1) { - return predicates.front(); - } - auto and_result = paimon::PredicateBuilder::And(predicates); - if (!and_result.ok()) { - return nullptr; - } - return std::move(and_result).value(); -} - -std::shared_ptr PaimonPredicateConverter::_convert_expr(const VExprSPtr& expr) { - if (!expr) { - return nullptr; - } - - auto uncast = VExpr::expr_without_cast(expr); - - if (auto* direct_in = dynamic_cast(uncast.get())) { - VExprSPtr in_expr; - if (direct_in->get_slot_in_expr(in_expr)) { - return _convert_in(in_expr); - } - return nullptr; - } - - if (dynamic_cast(uncast.get()) != nullptr) { - return _convert_in(uncast); - } - - switch (uncast->op()) { - case TExprOpcode::COMPOUND_AND: - case TExprOpcode::COMPOUND_OR: - return _convert_compound(uncast); - case TExprOpcode::COMPOUND_NOT: - return nullptr; - case TExprOpcode::EQ: - case TExprOpcode::EQ_FOR_NULL: - case TExprOpcode::NE: - case TExprOpcode::GE: - case TExprOpcode::GT: - case TExprOpcode::LE: - case TExprOpcode::LT: - return _convert_binary(uncast); - default: - break; - } - - if (auto* fn = dynamic_cast(uncast.get())) { - auto fn_name = _normalize_name(fn->function_name()); - if (fn_name == "is_null_pred" || fn_name == "is_not_null_pred") { - return _convert_is_null(uncast, fn_name); - } - if (fn_name == "like") { - return _convert_like_prefix(uncast); - } - } - - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_compound( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto left = _convert_expr(expr->get_child(0)); - if (!left) { - return nullptr; - } - auto right = _convert_expr(expr->get_child(1)); - if (!right) { - return nullptr; - } - - if (expr->op() == TExprOpcode::COMPOUND_AND) { - auto and_result = paimon::PredicateBuilder::And({left, right}); - return and_result.ok() ? std::move(and_result).value() : nullptr; - } - if (expr->op() == TExprOpcode::COMPOUND_OR) { - auto or_result = paimon::PredicateBuilder::Or({left, right}); - return or_result.ok() ? std::move(or_result).value() : nullptr; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_in(const VExprSPtr& expr) { - auto* in_pred = dynamic_cast(expr.get()); - if (!in_pred || expr->get_num_children() < 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - std::vector literals; - literals.reserve(expr->get_num_children() - 1); - for (uint16_t i = 1; i < expr->get_num_children(); ++i) { - auto literal = _convert_literal(expr->get_child(i), *field_meta->slot_desc, - field_meta->field_type); - if (!literal) { - return nullptr; - } - literals.emplace_back(std::move(*literal)); - } - - if (literals.empty()) { - return nullptr; - } - if (in_pred->is_not_in()) { - return paimon::PredicateBuilder::NotIn(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); - } - return paimon::PredicateBuilder::In(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); -} - -std::shared_ptr PaimonPredicateConverter::_convert_binary( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - if (expr->op() == TExprOpcode::EQ_FOR_NULL) { - return paimon::PredicateBuilder::IsNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - - auto literal = - _convert_literal(expr->get_child(1), *field_meta->slot_desc, field_meta->field_type); - if (!literal) { - return nullptr; - } - - switch (expr->op()) { - case TExprOpcode::EQ: - return paimon::PredicateBuilder::Equal(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::NE: - return paimon::PredicateBuilder::NotEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GE: - return paimon::PredicateBuilder::GreaterOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GT: - return paimon::PredicateBuilder::GreaterThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LE: - return paimon::PredicateBuilder::LessOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LT: - return paimon::PredicateBuilder::LessThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - default: - break; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_is_null( - const VExprSPtr& expr, const std::string& fn_name) { - if (!expr || expr->get_num_children() != 1) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - if (fn_name == "is_not_null_pred") { - return paimon::PredicateBuilder::IsNotNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - return paimon::PredicateBuilder::IsNull(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type); -} - -std::shared_ptr PaimonPredicateConverter::_convert_like_prefix( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta || field_meta->field_type != paimon::FieldType::STRING) { - return nullptr; - } - - auto pattern_opt = _extract_string_literal(expr->get_child(1)); - if (!pattern_opt) { - return nullptr; - } - const std::string& pattern = *pattern_opt; - if (!pattern.empty() && pattern.front() == '%') { - return nullptr; - } - if (pattern.empty() || pattern.back() != '%') { - return nullptr; - } - - std::string prefix = pattern.substr(0, pattern.size() - 1); - paimon::Literal lower_literal(paimon::FieldType::STRING, prefix.data(), prefix.size()); - auto lower_pred = paimon::PredicateBuilder::GreaterOrEqual( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type, - lower_literal); - - auto upper_prefix = _next_prefix(prefix); - if (!upper_prefix) { - return lower_pred; - } - - paimon::Literal upper_literal(paimon::FieldType::STRING, upper_prefix->data(), - upper_prefix->size()); - auto upper_pred = - paimon::PredicateBuilder::LessThan(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, upper_literal); - auto and_result = paimon::PredicateBuilder::And({lower_pred, upper_pred}); - return and_result.ok() ? std::move(and_result).value() : nullptr; -} - -std::optional PaimonPredicateConverter::_resolve_field( - const VExprSPtr& expr) const { - if (!_state || !expr) { - return std::nullopt; - } - auto slot_expr = VExpr::expr_without_cast(expr); - auto* slot_ref = dynamic_cast(slot_expr.get()); - if (!slot_ref) { - return std::nullopt; - } - auto* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id()); - if (!slot_desc) { - return std::nullopt; - } - auto normalized = _normalize_name(slot_desc->col_name()); - auto it = _field_index_by_name.find(normalized); - if (it == _field_index_by_name.end()) { - return std::nullopt; - } - auto slot_type = slot_desc->type(); - auto field_type = - _to_paimon_field_type(slot_type->get_primitive_type(), slot_type->get_precision()); - if (!field_type) { - return std::nullopt; - } - return FieldMeta {it->second, *field_type, slot_desc}; -} - -std::optional PaimonPredicateConverter::_convert_literal( - const VExprSPtr& expr, const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - PrimitiveType slot_primitive = slot_desc.type()->get_primitive_type(); - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - - Field field; - col->get(0, field); - - switch (slot_primitive) { - case TYPE_BOOLEAN: { - if (literal_primitive != TYPE_BOOLEAN) { - return std::nullopt; - } - return paimon::Literal(static_cast(field.get())); - } - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: { - if (!_is_integer_type(literal_primitive)) { - return std::nullopt; - } - int64_t value = 0; - switch (literal_primitive) { - case TYPE_TINYINT: - value = field.get(); - break; - case TYPE_SMALLINT: - value = field.get(); - break; - case TYPE_INT: - value = field.get(); - break; - case TYPE_BIGINT: - value = field.get(); - break; - default: - return std::nullopt; - } - if (slot_primitive == TYPE_TINYINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_SMALLINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_INT) { - return paimon::Literal(static_cast(value)); - } - return paimon::Literal(static_cast(value)); - } - case TYPE_DOUBLE: { - if (literal_primitive != TYPE_DOUBLE && literal_primitive != TYPE_FLOAT) { - return std::nullopt; - } - double value = 0; - if (literal_primitive == TYPE_FLOAT) { - value = static_cast(field.get()); - } else { - value = field.get(); - } - return paimon::Literal(value); - } - case TYPE_DATE: - case TYPE_DATEV2: { - if (!_is_date_type(literal_primitive)) { - return std::nullopt; - } - int64_t seconds = 0; - if (literal_primitive == TYPE_DATE) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } else if (literal_primitive == TYPE_DATEV2) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } - int32_t days = _seconds_to_days(seconds); - return paimon::Literal(paimon::FieldType::DATE, days); - } - case TYPE_DATETIME: - case TYPE_DATETIMEV2: { - if (!_is_datetime_type(literal_primitive)) { - return std::nullopt; - } - if (literal_primitive == TYPE_DATETIME) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - int64_t seconds = 0; - dt.unix_timestamp(&seconds, _gmt_tz); - return paimon::Literal(paimon::Timestamp::FromEpochMillis(seconds * 1000)); - } - std::pair ts; - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&ts, _gmt_tz); - int64_t millis = ts.first * 1000 + ts.second / 1000; - return paimon::Literal(paimon::Timestamp::FromEpochMillis(millis)); - } - case TYPE_VARCHAR: - case TYPE_STRING: { - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - const auto& value = field.get(); - return paimon::Literal(field_type, value.data(), value.size()); - } - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: { - if (!_is_decimal_type(literal_primitive)) { - return std::nullopt; - } - int32_t precision = static_cast(literal_type->get_precision()); - int32_t scale = static_cast(literal_type->get_scale()); - if (precision <= 0 || precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - - paimon::Decimal::int128_t value = 0; - switch (literal_primitive) { - case TYPE_DECIMALV2: { - const auto& dec = field.get(); - value = dec.value(); - break; - } - case TYPE_DECIMAL32: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL64: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL128I: { - const auto& dec = field.get(); - value = dec.value; - break; - } - default: - return std::nullopt; - } - return paimon::Literal(paimon::Decimal(precision, scale, value)); - } - default: - break; - } - return std::nullopt; -} - -std::optional PaimonPredicateConverter::_extract_string_literal( - const VExprSPtr& expr) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - Field field; - col->get(0, field); - const auto& value = field.get(); - return value; -} - -std::string PaimonPredicateConverter::_normalize_name(std::string_view name) { - std::string out(name); - std::transform(out.begin(), out.end(), out.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return out; -} - -std::optional PaimonPredicateConverter::_next_prefix(const std::string& prefix) { - if (prefix.empty()) { - return std::nullopt; - } - std::string upper = prefix; - for (int i = static_cast(upper.size()) - 1; i >= 0; --i) { - auto c = static_cast(upper[i]); - if (c != 0xFF) { - upper[i] = static_cast(c + 1); - upper.resize(i + 1); - return upper; - } - } - return std::nullopt; -} - -int32_t PaimonPredicateConverter::_seconds_to_days(int64_t seconds) { - static constexpr int64_t kSecondsPerDay = 24 * 60 * 60; - int64_t days = seconds / kSecondsPerDay; - if (seconds < 0 && seconds % kSecondsPerDay != 0) { - --days; - } - return static_cast(days); -} - -bool PaimonPredicateConverter::_is_integer_type(PrimitiveType type) { - switch (type) { - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_string_type(PrimitiveType type) { - return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING; -} - -bool PaimonPredicateConverter::_is_decimal_type(PrimitiveType type) { - switch (type) { - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_date_type(PrimitiveType type) { - return type == TYPE_DATE || type == TYPE_DATEV2; -} - -bool PaimonPredicateConverter::_is_datetime_type(PrimitiveType type) { - return type == TYPE_DATETIME || type == TYPE_DATETIMEV2; -} - -std::optional PaimonPredicateConverter::_to_paimon_field_type( - PrimitiveType type, uint32_t precision) { - switch (type) { - case TYPE_BOOLEAN: - return paimon::FieldType::BOOLEAN; - case TYPE_TINYINT: - return paimon::FieldType::TINYINT; - case TYPE_SMALLINT: - return paimon::FieldType::SMALLINT; - case TYPE_INT: - return paimon::FieldType::INT; - case TYPE_BIGINT: - return paimon::FieldType::BIGINT; - case TYPE_DOUBLE: - return paimon::FieldType::DOUBLE; - case TYPE_VARCHAR: - case TYPE_STRING: - return paimon::FieldType::STRING; - case TYPE_DATE: - case TYPE_DATEV2: - return paimon::FieldType::DATE; - case TYPE_DATETIME: - case TYPE_DATETIMEV2: - return paimon::FieldType::TIMESTAMP; - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - if (precision > 0 && precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - return paimon::FieldType::DECIMAL; - case TYPE_FLOAT: - case TYPE_CHAR: - default: - return std::nullopt; - } -} - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.h b/be/src/format/table/paimon_predicate_converter.h deleted file mode 100644 index 424ad66113b3ff..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.h +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "core/data_type/define_primitive_type.h" -#include "exprs/vexpr_fwd.h" -#include "paimon/defs.h" -#include "paimon/predicate/literal.h" - -namespace paimon { -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { -#include "common/compile_check_begin.h" - -class PaimonPredicateConverter { -public: - PaimonPredicateConverter(const std::vector& file_slot_descs, - RuntimeState* state); - - std::shared_ptr build(const VExprContextSPtrs& conjuncts); - -private: - struct FieldMeta { - int32_t index = -1; - paimon::FieldType field_type = paimon::FieldType::UNKNOWN; - const SlotDescriptor* slot_desc = nullptr; - }; - - std::shared_ptr _convert_expr(const VExprSPtr& expr); - std::shared_ptr _convert_compound(const VExprSPtr& expr); - std::shared_ptr _convert_in(const VExprSPtr& expr); - std::shared_ptr _convert_binary(const VExprSPtr& expr); - std::shared_ptr _convert_is_null(const VExprSPtr& expr, - const std::string& fn_name); - std::shared_ptr _convert_like_prefix(const VExprSPtr& expr); - - std::optional _resolve_field(const VExprSPtr& expr) const; - std::optional _convert_literal(const VExprSPtr& expr, - const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const; - std::optional _extract_string_literal(const VExprSPtr& expr) const; - - static std::string _normalize_name(std::string_view name); - static std::optional _next_prefix(const std::string& prefix); - static int32_t _seconds_to_days(int64_t seconds); - static bool _is_integer_type(PrimitiveType type); - static bool _is_string_type(PrimitiveType type); - static bool _is_decimal_type(PrimitiveType type); - static bool _is_date_type(PrimitiveType type); - static bool _is_datetime_type(PrimitiveType type); - static std::optional _to_paimon_field_type(PrimitiveType type, - uint32_t precision); - - std::unordered_map _field_index_by_name; - RuntimeState* _state = nullptr; - cctz::time_zone _gmt_tz; -}; - -#include "common/compile_check_end.h" -} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 90ddbedb07c70c..58a3b02a99c5dc 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -91,15 +91,6 @@ TFileRangeDesc hudi_range_with_delta_logs() { return range; } -TFileRangeDesc paimon_cpp_jni_range() { - auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); - TPaimonFileDesc paimon_params; - paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_CPP); - paimon_params.__set_file_format("parquet"); - range.table_format_params.__set_paimon_params(std::move(paimon_params)); - return range; -} - TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); TPaimonFileDesc paimon_params; @@ -879,19 +870,10 @@ TEST(FileScannerV2Test, GeneratedChildrenKeepOneGlobalRowIdSourceMapping) { TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { TQueryOptions query_options; query_options.__set_enable_file_scanner_v2(true); - query_options.__set_enable_paimon_cpp_reader(true); TFileScanRangeParams params; params.__set_format_type(TFileFormatType::FORMAT_JNI); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - const auto cpp_range = paimon_cpp_jni_range(); - EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_range)); - const auto cpp_status = FileScannerV2::TEST_validate_scan_range(params, cpp_range); - EXPECT_TRUE(cpp_status.is()); - - // Older FE plans without reader_type used Java whenever the C++ option was disabled. - query_options.__set_enable_paimon_cpp_reader(false); - EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } diff --git a/be/test/format/table/paimon_cpp_reader_test.cpp b/be/test/format/table/paimon_cpp_reader_test.cpp deleted file mode 100644 index 11f337d32ba653..00000000000000 --- a/be/test/format/table/paimon_cpp_reader_test.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include - -#include -#include - -#include "core/block/block.h" -#include "runtime/runtime_profile.h" -#include "runtime/runtime_state.h" - -namespace doris { - -class PaimonCppReaderTest : public testing::Test { -protected: - void SetUp() override { - _query_options.__set_batch_size(3); - _runtime_state = std::make_unique(_query_globals); - _runtime_state->set_query_options(_query_options); - } - - TFileRangeDesc _build_range_with_table_level_row_count(int64_t row_count) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.table_level_row_count = true; - range.table_format_params.table_level_row_count = row_count; - return range; - } - - TQueryOptions _query_options; - TQueryGlobals _query_globals; - std::unique_ptr _runtime_state; - RuntimeProfile _profile {"paimon_cpp_reader_test"}; - std::vector _file_slot_descs; -}; - -TEST_F(PaimonCppReaderTest, CountPushDownUsesTableLevelRowCount) { - auto range = _build_range_with_table_level_row_count(5); - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - reader.set_push_down_agg_type(TPushAggOp::type::COUNT); - - auto init_status = reader.init_reader(); - ASSERT_TRUE(init_status.ok()) << init_status; - - Block block; - size_t read_rows = 0; - bool eof = false; - - auto first_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(first_status.ok()) << first_status; - EXPECT_EQ(3, read_rows); - EXPECT_FALSE(eof); - - auto second_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(second_status.ok()) << second_status; - EXPECT_EQ(2, read_rows); - EXPECT_TRUE(eof); - - auto third_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(third_status.ok()) << third_status; - EXPECT_EQ(0, read_rows); - EXPECT_TRUE(eof); -} - -TEST_F(PaimonCppReaderTest, InitReaderFailsWithoutPaimonSplit) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.paimon_params = true; - range.table_format_params.paimon_params.__isset.paimon_table = true; - range.table_format_params.paimon_params.paimon_table = "s3://bucket/db.tbl"; - - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - auto status = reader.init_reader(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("missing paimon_split"), std::string::npos); -} - -} // namespace doris diff --git a/be/test/format/table/paimon_doris_file_system_test.cpp b/be/test/format/table/paimon_doris_file_system_test.cpp deleted file mode 100644 index a2032dff8188c9..00000000000000 --- a/be/test/format/table/paimon_doris_file_system_test.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include - -#include "common/config.h" - -namespace doris { - -class PaimonDorisFileSystemTest : public testing::Test { -protected: - void SetUp() override { saved_mappings_ = config::paimon_file_system_scheme_mappings; } - - void TearDown() override { config::paimon_file_system_scheme_mappings = saved_mappings_; } - - std::vector saved_mappings_; -}; - -TEST_F(PaimonDorisFileSystemTest, UsesDefaultSchemeMappings) { - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("jfs")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("s3a")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("https")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("ofs")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("unknown")); -} - -TEST_F(PaimonDorisFileSystemTest, AllowsOverridingSchemeMappingsFromConfig) { - config::paimon_file_system_scheme_mappings = {"file=local", "jfs = s3", "gs = hdfs", - "custom-http = http", "custom-broker = broker"}; - - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("JFS")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("custom-http")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("custom-broker")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("still-unknown")); -} - -} // namespace doris diff --git a/build.sh b/build.sh index 76fd3599dac2d7..199cf09729e31d 100755 --- a/build.sh +++ b/build.sh @@ -446,59 +446,16 @@ else LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a' fi -# The final-library sentinel only proves that some third-party build completed. It cannot -# distinguish an older prebuilt whose Arrow/Paimon closure predates the selected sources. -# shellcheck source=thirdparty/arrow-paimon-vars.sh -. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh" -NEED_ARROW_PAIMON_THIRDPARTY=false -if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 || - "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" || - "${BUILD_INDEX_TOOL}" == "ON" ]]; then - NEED_ARROW_PAIMON_THIRDPARTY=true -fi - -rebuild_thirdparty_libraries() { - local remove_installed="$1" - shift - local build_script="${DORIS_THIRDPARTY}/build-thirdparty.sh" - local build_args=(-j "${PARALLEL}") - local selected_thirdparty_root - local checkout_thirdparty_root - - if [[ ! -f "${build_script}" ]]; then - echo "Cannot rebuild thirdparty libraries: ${build_script} is missing." >&2 - echo "DORIS_THIRDPARTY=${DORIS_THIRDPARTY} is an install-only or incomplete prefix. Use a matching compilation image/prebuilt, or unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree." >&2 - exit 1 - fi - selected_thirdparty_root="$(cd "${DORIS_THIRDPARTY}" && pwd -P)" - checkout_thirdparty_root="$(cd "${DORIS_HOME}/thirdparty" && pwd -P)" - if [[ "${selected_thirdparty_root}" != "${checkout_thirdparty_root}" ]]; then - echo "Cannot rebuild thirdparty libraries with an external source tree: ${selected_thirdparty_root}." >&2 - echo "Unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree, then use the resulting version-matched installation." >&2 - exit 1 - fi - build_script="${checkout_thirdparty_root}/build-thirdparty.sh" - if [[ "${remove_installed}" == "true" ]]; then - # Some libraries, such as lz4, fail when an earlier installation remains. - rm -rf "${DORIS_THIRDPARTY}/installed" - fi - if [[ "${CLEAN}" -eq 1 ]]; then - build_args+=(--clean) - fi - bash "${build_script}" "${build_args[@]}" "$@" - if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 - exit 1 - fi -} - if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." - rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && - ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" + # need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists + rm -rf "${DORIS_THIRDPARTY}/installed" + + if [[ "${CLEAN}" -eq 0 ]]; then + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" + else + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean + fi fi update_submodule() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a793c59da149d4..c786ae2489875c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -54,7 +54,6 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; -import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.partition.Partition; @@ -65,7 +64,6 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.Table; -import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.tag.Tag; import org.apache.paimon.types.ArrayType; @@ -718,23 +716,6 @@ private byte[] toByteArray() { private static final class SerializationSizeLimitException extends IOException { } - /** - * Serialize DataSplit using Paimon's native binary format. - * This format is compatible with paimon-cpp reader. - * Uses standard Base64 encoding (not URL-safe) for BE compatibility. - */ - public static String encodeDataSplitToString(DataSplit split) { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos); - split.serialize(out); - byte[] bytes = baos.toByteArray(); - return Base64.getEncoder().encodeToString(bytes); - } catch (IOException e) { - throw new RuntimeException("Failed to serialize DataSplit using Paimon native format", e); - } - } - public static Map getPartitionInfoMap(Table table, BinaryRow partitionValues, String timeZone) { Map partitionInfoMap = new HashMap<>(); List partitionKeys = table.partitionKeys(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 2eb3c92ed50f63..9e96179b9b7498 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -412,8 +412,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) String fileFormat = getFileFormat(paimonSplit.getPathString()); if (split != null) { rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); - // A logical DataSplit may span multiple files, so keep it intact for the JNI reader - // until the C++ path has a split-aware V2 adapter. + // A logical DataSplit may span multiple files, so keep it intact for the JNI reader. fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI); fileDesc.setPaimonSplit(PaimonUtil.encodeObjectToString(split)); rangeDesc.setSelfSplitWeight(paimonSplit.getSelfSplitWeight()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6425c3fc35c526..a69a286babd6dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -807,8 +807,6 @@ public String toString() { public static final String FORCE_JNI_SCANNER = "force_jni_scanner"; - public static final String ENABLE_PAIMON_CPP_READER = "enable_paimon_cpp_reader"; - public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table"; @@ -1447,6 +1445,7 @@ public enum IgnoreSplitType { NONE, IGNORE_JNI, IGNORE_NATIVE, + // Deprecated compatibility value. It behaves like NONE because no C++ splits are emitted. IGNORE_PAIMON_CPP } @@ -3051,11 +3050,6 @@ public Map getForceEagerAggHintMap() { description = {"强制使用 jni 方式读取外表", "Force the use of jni mode to read external table"}) private boolean forceJniScanner = false; - @VariableMgr.VarAttr(name = ENABLE_PAIMON_CPP_READER, - fuzzy = true, - description = {"Paimon 非原生文件读取使用 paimon-cpp", "Use paimon-cpp for non-native Paimon reads"}) - private boolean enablePaimonCppReader = false; - @VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) @@ -3971,8 +3965,6 @@ private void setFuzzyForCatalog(Random random) { // jni this.forceJniScanner = random.nextBoolean(); - this.enablePaimonCppReader = random.nextBoolean(); - // statistics this.fetchHiveRowCountSync = random.nextBoolean(); @@ -5713,7 +5705,6 @@ public TQueryOptions toThrift() { tResult.setEnableParquetFilePageCache(enableParquetFilePageCache); tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax); tResult.setEnableExprZonemapFilter(enableExprZonemapFilter); - tResult.setEnablePaimonCppReader(enablePaimonCppReader); tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds); tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess); @@ -6510,10 +6501,6 @@ public boolean isForceJniScanner() { return forceJniScanner; } - public boolean isEnablePaimonCppReader() { - return enablePaimonCppReader; - } - public String getIgnoreSplitType() { return ignoreSplitType; } @@ -6537,10 +6524,6 @@ public void setForceJniScanner(boolean force) { forceJniScanner = force; } - public void setEnablePaimonCppReader(boolean enable) { - enablePaimonCppReader = enable; - } - public boolean isEnableCountPushDownForExternalTable() { return enableCountPushDownForExternalTable; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 55d6b489a49744..fbde6fecc8a5b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -1727,12 +1727,8 @@ public void testNativeSplitCarriesPartitionMetadataWithoutRuntimeFilterPruning() } @Test - public void testSetPaimonParamsUsesJniWhenCppOptionEnabled() throws Exception { - // Keep this as real session state because the JNI-only path need not read the option; - // strict mocks should not make the test depend on whether that implementation detail is consulted. - SessionVariable cppEnabledSession = new SessionVariable(); - cppEnabledSession.setEnablePaimonCppReader(true); - PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), cppEnabledSession); + public void testSetPaimonParamsUsesJniForLogicalSplit() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), new SessionVariable()); PaimonSource source = Mockito.mock(PaimonSource.class); Table paimonTable = mockPaimonTableWithPartitionKeys(Collections.emptyList()); Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index d98c4b3961f308..6c298939158e92 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -435,7 +435,7 @@ struct TQueryOptions { 195: optional bool enable_left_semi_direct_return_opt; 200: optional bool enable_adjust_conjunct_order_by_cost; - // Use paimon-cpp to read Paimon splits on BE + // Deprecated: the paimon-cpp reader has been removed. Retained for wire compatibility. 201: optional bool enable_paimon_cpp_reader = false; // Whether all fragments of this query are assigned to a single backend. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 15701eb34d75b0..90833189d18d4a 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -354,6 +354,7 @@ struct TPaimonDeletionFileDesc { enum TPaimonReaderType { PAIMON_NATIVE = 0, PAIMON_JNI = 1, + // Deprecated wire value kept during rolling upgrades. New plans never emit it. PAIMON_CPP = 2, } diff --git a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md index 5a551b6d3d8e34..4d2f3ae0529c4f 100644 --- a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md @@ -70,7 +70,7 @@ Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contrac | Paimon | Append table, partitioned table, primitive and nested types | Covered | `test_paimon_catalog`, `test_paimon_partition_table`, `test_paimon_full_schema_change` | | Paimon | Primary-key deduplicate, partial-update, aggregation and first-row | Covered | `test_paimon_merge_engine_matrix` | | Paimon | Fixed bucket, dynamic bucket and cross-partition update | Covered | `test_paimon_merge_engine_matrix`, `test_paimon_partition_pk_delete_refs` | -| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_cpp_reader` | +| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_reader_parity` | | Paimon | Snapshot/timestamp/tag/branch and incremental modes | Covered | `paimon_time_travel`, `paimon_incr_read`, `test_paimon_schema_time_travel_matrix` | | Paimon | Schema evolution, partition-key restrictions and historical schema binding | Covered | `test_paimon_schema_time_travel_matrix`, `test_paimon_partition_mutation_atomicity` | | Paimon | Deletion vectors, upsert/delete visibility and data/system tables | Covered | `test_paimon_deletion_vector`, `paimon_data_system_table`, `paimon_system_table` | diff --git a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy index 5244a210c015e2..3feb4de2c56648 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy @@ -138,7 +138,6 @@ suite("paimon_data_system_table", "p0,external,doris,external_docker,external_do } sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" // Paimon data system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection @@ -183,7 +182,6 @@ suite("paimon_data_system_table", "p0,external,doris,external_docker,external_do qt_jni_native_binlog_rows """select rowkind, id[1], id[2], name[1], name[2] from ${nativeTableName}\$binlog order by id[1]""" qt_jni_native_audit_log_rows """select rowkind, id, name from ${nativeTableName}\$audit_log order by id""" } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" } } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy index 85007d147ba2d0..f8c599804f13f8 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy @@ -321,14 +321,12 @@ suite("test_paimon_partition_pk_delete_refs", where part in ('p1', 'p2') order by part, id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current DV") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_dv"(currentReaderQuery) "qt_${actionSuffix}_jni_historical"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current DV") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_dv"(currentReaderQuery) @@ -336,7 +334,6 @@ suite("test_paimon_partition_pk_delete_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy index 979fbb7784c798..e2b7ffeae99cd4 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy @@ -341,14 +341,12 @@ suite("test_paimon_partition_schema_filter_refs", where part in ('p1', 'p2') order by id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_complex"(currentReaderQuery) "qt_${actionSuffix}_jni_historical_complex"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_complex"(currentReaderQuery) @@ -356,7 +354,6 @@ suite("test_paimon_partition_schema_filter_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy similarity index 81% rename from regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy rename to regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy index 64ef323add6119..4172c8e805b075 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -suite("test_paimon_cpp_reader", "p0,external") { +suite("test_paimon_reader_parity", "p0,external") { String enabled = context.config.otherConfigs.get("enablePaimonTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disabled paimon test") return } - String catalogName = "test_paimon_cpp_reader" + String catalogName = "test_paimon_reader_parity" String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -35,9 +35,6 @@ suite("test_paimon_cpp_reader", "p0,external") { );""" sql """switch ${catalogName}""" sql """use db1""" - // Do not force JNI; keep default selection behavior. - sql """set force_jni_scanner=false""" - def testQueries = [ """select c1 from complex_all order by c1""", """select c1 from complex_all where c1 >= 2 order by c1""", @@ -51,19 +48,17 @@ suite("test_paimon_cpp_reader", "p0,external") { """select * from deletion_vector_parquet""" ] - // Default path is JNI when enable_paimon_cpp_reader=false. - sql """set enable_paimon_cpp_reader=false""" + sql """set force_jni_scanner=true""" def jniResults = testQueries.collect { query -> sql(query) } - sql """set enable_paimon_cpp_reader=true""" - def cppResults = testQueries.collect { query -> sql(query) } + sql """set force_jni_scanner=false""" + def nativeResults = testQueries.collect { query -> sql(query) } - assertTrue(cppResults[0].size() > 0) + assertTrue(nativeResults[0].size() > 0) for (int i = 0; i < testQueries.size(); i++) { - assertEquals(jniResults[i].toString(), cppResults[i].toString()) + assertEquals(jniResults[i].toString(), nativeResults[i].toString()) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index 1a41eee698ff1f..274bc1c67ab069 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -367,9 +367,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { sql """refresh catalog ${catalogName}""" // row_tracking exposes Paimon-generated hidden columns, so its incremental scan must - // bypass the C++ reader even when the session otherwise enables native Paimon scans. + // use Paimon-side JNI semantics. sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" String rowTrackingExplain = sql(""" explain verbose select id, name, _ROW_ID, _SEQUENCE_NUMBER @@ -386,8 +385,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { @incr('startSnapshotId'=0, 'endSnapshotId'=1) order by id """ - sql """set enable_paimon_cpp_reader=false""" - // Scenario TC01: validate latest schema/data, explicit new binding, predicate and aggregate. assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], sql("""select id, victim from ${topTable} order by id""")) @@ -571,9 +568,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { // Scenario T14: incremental reads crossing a rename checkpoint bind the end schema. List> incrementalJni - List> incrementalCpp - sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=false""" + List> incrementalAutomatic + sql """set force_jni_scanner=true""" incrementalJni = sql(""" select id, full_name, score from ${pkTable}@incr( @@ -582,8 +578,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - sql """set enable_paimon_cpp_reader=true""" - incrementalCpp = sql(""" + sql """set force_jni_scanner=false""" + incrementalAutomatic = sql(""" select id, full_name, score from ${pkTable}@incr( 'startSnapshotId'='${pkCp0}', @@ -591,7 +587,7 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - assertEquals(incrementalJni, incrementalCpp) + assertEquals(incrementalJni, incrementalAutomatic) // Scenario TC03/S16: partition pruning and renamed payloads bind to their own snapshots. assertEquals([[1, "p1", "old"], [2, "p2", "old"]], @@ -691,7 +687,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { """)) sql """switch ${catalogName}""" sql """use ${dbName}""" - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" List> forcedJniRows = sql(""" select id, old_name, victim, metric @@ -699,21 +694,19 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { order by id """) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" - List> cppRows = sql(""" + List> automaticRows = sql(""" select id, old_name, victim, metric from ${topTable} for version as of ${topCp0} order by id """) - assertEquals(forcedJniRows, cppRows) - // Schema-selecting OPTIONS must bypass paimon-cpp, whose table handle always uses the - // latest schema, even when native Paimon scans are enabled for the session. - List> cppOptionsRows = sql(""" + assertEquals(forcedJniRows, automaticRows) + // Schema-selecting OPTIONS retain the requested historical schema under automatic routing. + List> automaticOptionsRows = sql(""" select id, old_name, victim, metric from ${topTable}@options('scan.snapshot-id'='${topCp0}') order by id """) - assertEquals(forcedJniRows, cppOptionsRows) + assertEquals(forcedJniRows, automaticOptionsRows) // Scenario T10/T11: retained tags survive expiration; missing refs never fall back to latest. spark_paimon """ @@ -741,7 +734,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { exception "tag" } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop database if exists internal.${viewDb} force""" sql """drop catalog if exists ${catalogName}""" diff --git a/run-be-ut.sh b/run-be-ut.sh index ae93cd565c009d..9e091ca0c29e2a 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -47,29 +47,6 @@ fi export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include" export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" . "${DORIS_HOME}/env.sh" -# shellcheck source=thirdparty/arrow-paimon-vars.sh -. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh" - -prepare_build_image_arrow_paimon_prebuilt() { - local selected_thirdparty_root - local checkout_thirdparty_root - - selected_thirdparty_root="$(cd "${DORIS_THIRDPARTY}" && pwd -P)" - checkout_thirdparty_root="$(cd "${DORIS_HOME}/thirdparty" && pwd -P)" - if [[ "${selected_thirdparty_root}" == "${checkout_thirdparty_root}" ]]; then - return 0 - fi - - # The official Linux x86_64 build image carries an install-only thirdparty - # tree. Refresh it from the shared automation asset when the image predates - # the Arrow/Paimon closure selected by this checkout. - if [[ "${selected_thirdparty_root}" != "/var/local/thirdparty" || - "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then - return 0 - fi - ensure_arrow_paimon_prebuilt_from_url "${selected_thirdparty_root}" \ - "${ARROW_PAIMON_SHARED_PREBUILT_LINUX_X86_64_URL}" -} trim_whitespace() { local value="$1" @@ -250,8 +227,6 @@ echo "Get params: " echo "Build Backend UT" -prepare_build_image_arrow_paimon_prebuilt - update_submodule() { local submodule_path=$1 local submodule_name=$2 diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh deleted file mode 100644 index ab139125035d7a..00000000000000 --- a/thirdparty/arrow-paimon-vars.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/bin/bash -# shellcheck disable=2034 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can -# distinguish this stack from unrelated thirdparty changes. - -# arrow -ARROW_VERSION="24.0.0" -ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" -ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" - -# Arrow bundled dependencies -BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" -BROTLI_NAME="brotli-1.0.9.tar.gz" -BROTLI_SOURCE="brotli-1.0.9" -BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" - -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" -XSIMD_NAME="14.0.0.tar.gz" -XSIMD_SOURCE=xsimd-14.0.0 -XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" - -# paimon-cpp -PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" -PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" - -ARROW_PAIMON_SHARED_PREBUILT_LINUX_X86_64_URL="${ARROW_PAIMON_SHARED_PREBUILT_LINUX_X86_64_URL:-https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz}" - -# Bump the corresponding schema version whenever output-affecting build options or -# helper behavior in build_arrow() or build_paimon_cpp() changes. The fingerprints -# below intentionally describe only this component stack so master and release -# branches can reuse the same shared prebuilt when their semantic inputs match. -ARROW_BUILD_SCHEMA_VERSION="1" -PAIMON_BUILD_SCHEMA_VERSION="1" - -# The current shared automation prebuilt was published from master with the former -# whole-script fingerprint. Keep these exact markers during the schema transition; -# version and complete artifact validation are still mandatory before reuse. -ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" -# Dependency ordering does not change the installed Paimon artifacts, so existing -# complete prebuilts remain compatible with the corrected clean-build graph. -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c" -ARROW_LEGACY_BUILD_FINGERPRINTS=( - 9d03645dd1cded5184a8126f5c7f4a6eb9b92b53 -) -PAIMON_LEGACY_BUILD_FINGERPRINTS=( - dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28 -) - -# Arrow consumes xsimd and Brotli as bundled source archives, but neither is a -# build target in the focused Arrow/Paimon recovery path. -ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) -ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) -ARROW_PAIMON_DOWNLOAD_PACKAGES=() - -prepare_arrow_paimon_download_packages() { - ARROW_PAIMON_DOWNLOAD_PACKAGES=("$@") - - local package - local source_package - local arrow_requested=false - local source_requested - for package in "$@"; do - if [[ "${package}" == "arrow" ]]; then - arrow_requested=true - break - fi - done - if [[ "${arrow_requested}" != "true" ]]; then - return - fi - - for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do - source_requested=false - for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do - if [[ "${package}" == "${source_package}" ]]; then - source_requested=true - break - fi - done - if [[ "${source_requested}" != "true" ]]; then - ARROW_PAIMON_DOWNLOAD_PACKAGES+=("${source_package}") - fi - done -} - -# Print stable path-and-content records for fingerprint inputs. Including the path -# makes patch selection and ordering part of the contract, not only file contents. -arrow_paimon_fingerprint_files() { - local file - local blob - while IFS= read -r file; do - blob="$(git hash-object "${file}")" || return 1 - printf 'file=%s\n' "${file}" - printf 'blob=%s\n' "${blob}" - done < <(printf '%s\n' "$@" | LC_ALL=C sort) -} - -# Identify only the source, patch, and explicit build-schema inputs selected for -# Arrow. Arrow and Paimon publish separate installed markers so a package-only -# build cannot certify a component that it did not rebuild. -arrow_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}" - printf 'ARROW_VERSION=%s\n' "${ARROW_VERSION}" - printf 'ARROW_NAME=%s\n' "${ARROW_NAME}" - printf 'ARROW_SOURCE=%s\n' "${ARROW_SOURCE}" - printf 'ARROW_MD5SUM=%s\n' "${ARROW_MD5SUM}" - printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}" - printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}" - printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}" - printf 'XSIMD_NAME=%s\n' "${XSIMD_NAME}" - printf 'XSIMD_SOURCE=%s\n' "${XSIMD_SOURCE}" - printf 'XSIMD_MD5SUM=%s\n' "${XSIMD_MD5SUM}" - arrow_paimon_fingerprint_files \ - patches/apache-arrow-"${ARROW_VERSION}"-*.patch - } | git hash-object --stdin - ) -} - -paimon_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - arrow_build_fingerprint - printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}" - printf 'PAIMON_CPP_NAME=%s\n' "${PAIMON_CPP_NAME}" - printf 'PAIMON_CPP_SOURCE=%s\n' "${PAIMON_CPP_SOURCE}" - printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}" - arrow_paimon_fingerprint_files \ - paimon-cpp-cache.cmake \ - patches/paimon-cpp-*.patch - } | git hash-object --stdin - ) -} - -# Source patch markers use a combined value because either component's inputs -# may change the external-Arrow contract applied to both source trees. -arrow_paimon_build_fingerprint() { - { - arrow_build_fingerprint - paimon_build_fingerprint - } | git hash-object --stdin -} - -arrow_paimon_fingerprint_matches() { - local installed_fingerprint="$1" - local expected_fingerprint="$2" - local legacy_compatible_expected_fingerprint="$3" - shift 3 - local compatible_fingerprint - - if [[ "${installed_fingerprint}" == "${expected_fingerprint}" ]]; then - return 0 - fi - if [[ "${expected_fingerprint}" != "${legacy_compatible_expected_fingerprint}" ]]; then - return 1 - fi - for compatible_fingerprint in "$@"; do - if [[ "${installed_fingerprint}" == "${compatible_fingerprint}" ]]; then - return 0 - fi - done - return 1 -} - -ARROW_REQUIRED_LIBRARIES=( - libbrotlicommon.a - libbrotlidec.a - libbrotlienc.a - libarrow.a - libarrow_compute.a - libarrow_flight.a - libarrow_flight_sql.a - libarrow_dataset.a - libarrow_acero.a - libarrow_bundled_dependencies.a - libparquet.a -) - -PAIMON_REQUIRED_LIBRARIES=( - libpaimon.a - libpaimon_parquet_file_format.a - libpaimon_orc_file_format.a - libpaimon_blob_file_format.a - libpaimon_local_file_system.a - libpaimon_file_index.a - libpaimon_global_index.a - libroaring_bitmap_paimon.a - libxxhash_paimon.a - libfmt_paimon.a - libtbb_paimon.a -) - -ARROW_PAIMON_REQUIRED_LIBRARIES=( - "${ARROW_REQUIRED_LIBRARIES[@]}" - "${PAIMON_REQUIRED_LIBRARIES[@]}" -) - -arrow_artifacts_valid() { - local install_dir="$1" - local installed_arrow_version - local library - - if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then - echo "Missing installed Arrow version header" >&2 - return 1 - fi - installed_arrow_version="$( - awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { - gsub(/"/, "", $3); print $3; exit - }' "${install_dir}/include/arrow/util/config.h" - )" - if [[ "${installed_arrow_version}" != "${ARROW_VERSION}" ]]; then - echo "Installed Arrow version ${installed_arrow_version} does not match ${ARROW_VERSION}" >&2 - return 1 - fi - - for library in "${ARROW_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Arrow library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -paimon_artifacts_valid() { - local install_dir="$1" - local library - - for library in "${PAIMON_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Paimon library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -arrow_prebuilt_valid() { - local install_dir="$1" - local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${arrow_fingerprint_mark}" ]]; then - echo "Missing Arrow build fingerprint: ${arrow_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(arrow_build_fingerprint)" - installed_fingerprint="$(<"${arrow_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${ARROW_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Arrow build fingerprint does not match selected inputs" >&2 - return 1 - fi - arrow_artifacts_valid "${install_dir}" -} - -paimon_prebuilt_valid() { - local install_dir="$1" - local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${paimon_fingerprint_mark}" ]]; then - echo "Missing Paimon build fingerprint: ${paimon_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(paimon_build_fingerprint)" - installed_fingerprint="$(<"${paimon_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${PAIMON_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Paimon build fingerprint does not match selected inputs" >&2 - return 1 - fi - paimon_artifacts_valid "${install_dir}" -} - -arrow_paimon_prebuilt_valid() { - local install_dir="$1" - arrow_prebuilt_valid "${install_dir}" && paimon_prebuilt_valid "${install_dir}" -} - -# Validate an extracted candidate before replacing the selected thirdparty -# installation. This keeps an older build image usable when a download is -# incomplete or contains an incompatible Arrow/Paimon closure. -install_arrow_paimon_prebuilt_archive() { - local archive="$1" - local thirdparty_root="$2" - local staging_dir - local had_previous_installed=false - - staging_dir="$(mktemp -d "${thirdparty_root}/.arrow-paimon-install.XXXXXX")" - mkdir -p "${staging_dir}/candidate" - if ! tar -xf "${archive}" -C "${staging_dir}/candidate"; then - rm -rf "${staging_dir}" - return 1 - fi - if ! arrow_paimon_prebuilt_valid "${staging_dir}/candidate/installed"; then - rm -rf "${staging_dir}" - return 1 - fi - - if [[ -e "${thirdparty_root}/installed" || -L "${thirdparty_root}/installed" ]]; then - if ! mv "${thirdparty_root}/installed" "${staging_dir}/previous-installed"; then - rm -rf "${staging_dir}" - return 1 - fi - had_previous_installed=true - fi - if ! mv "${staging_dir}/candidate/installed" "${thirdparty_root}/installed"; then - if [[ "${had_previous_installed}" == "true" ]]; then - mv "${staging_dir}/previous-installed" "${thirdparty_root}/installed" - fi - rm -rf "${staging_dir}" - return 1 - fi - rm -rf "${staging_dir}" -} - -ensure_arrow_paimon_prebuilt_from_url() { - local thirdparty_root="$1" - local prebuilt_url="$2" - local download_dir - local archive - - if arrow_paimon_prebuilt_valid "${thirdparty_root}/installed"; then - return 0 - fi - - echo "Refreshing Arrow/Paimon thirdparty libraries from ${prebuilt_url}" - download_dir="$(mktemp -d "${thirdparty_root}/.arrow-paimon-download.XXXXXX")" - archive="${download_dir}/doris-thirdparty-prebuilt.tar.xz" - if ! curl --fail --location --retry 3 --show-error \ - --output "${archive}" "${prebuilt_url}"; then - rm -rf "${download_dir}" - return 1 - fi - if ! install_arrow_paimon_prebuilt_archive "${archive}" "${thirdparty_root}"; then - rm -rf "${download_dir}" - return 1 - fi - rm -rf "${download_dir}" - arrow_paimon_prebuilt_valid "${thirdparty_root}/installed" -} - -invalidate_arrow_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/arrow-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_arrow_prebuilt_marker() { - local install_dir="$1" - arrow_artifacts_valid "${install_dir}" - arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" -} - -invalidate_paimon_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/paimon-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_paimon_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid "${install_dir}" - paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" -} - -require_arrow_prebuilt_for_paimon() { - local install_dir="$1" - if ! arrow_prebuilt_valid "${install_dir}"; then - echo "Paimon requires Arrow to be built from the currently selected inputs first" >&2 - return 1 - fi -} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 35437afcdf18b7..09221d02d4c382 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -155,8 +155,7 @@ if [[ "${CLEAN}" -eq 1 ]] && [[ -d "${TP_SOURCE_DIR}" ]]; then fi # Download thirdparties. -prepare_arrow_paimon_download_packages "${packages[@]}" -bash "${TP_DIR}/download-thirdparty.sh" "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}" +bash "${TP_DIR}/download-thirdparty.sh" "${packages[@]}" export LD_LIBRARY_PATH="${TP_DIR}/installed/lib:${LD_LIBRARY_PATH}" @@ -400,13 +399,18 @@ build_thrift() { check_if_source_exist "${THRIFT_SOURCE}" cd "${TP_SOURCE_DIR}/${THRIFT_SOURCE}" + # FE UT can rebuild the release-branch Thrift in a build image that already + # contains a newer Thrift. Prefer this source tree's headers so an in-place + # downgrade does not compile old sources against the installed new headers. + local thrift_source_include="${TP_SOURCE_DIR}/${THRIFT_SOURCE}/lib/cpp/src" + if [[ "${KERNEL}" != 'Darwin' ]]; then cflags="-I${TP_INCLUDE_DIR}" - cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" + cxxflags="-I${thrift_source_include} -I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" ldflags="-L${TP_LIB_DIR} --static" else cflags="-I${TP_INCLUDE_DIR} -Wno-implicit-function-declaration -Wno-inconsistent-missing-override" - cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" + cxxflags="-I${thrift_source_include} -I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" ldflags="-L${TP_LIB_DIR}" fi @@ -418,6 +422,18 @@ build_thrift() { --without-haskell --without-go --without-haxe --without-d --without-python -without-java --without-dotnetcore -without-rs --with-cpp \ --with-libevent="${TP_INSTALL_DIR}" --with-boost="${TP_INSTALL_DIR}" --with-openssl="${TP_INSTALL_DIR}" + # Thrift's generated Makefiles put dependency include paths before CXXFLAGS. + # Overlay this version's headers into the target prefix before compilation; + # make install will publish the same headers after the build succeeds. + local thrift_header + local relative_header + while IFS= read -r thrift_header; do + relative_header="${thrift_header#${thrift_source_include}/}" + mkdir -p "${TP_INCLUDE_DIR}/$(dirname "${relative_header}")" + cp "${thrift_header}" "${TP_INCLUDE_DIR}/${relative_header}" + done < <(find "${thrift_source_include}/thrift" -type f \ + \( -name '*.h' -o -name '*.tcc' \) -print) + if [[ -f compiler/cpp/thrifty.hh ]]; then mv compiler/cpp/thrifty.hh compiler/cpp/thrifty.h fi @@ -1065,7 +1081,6 @@ build_grpc() { # arrow build_arrow() { check_if_source_exist "${ARROW_SOURCE}" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" mkdir -p release @@ -1153,7 +1168,6 @@ build_arrow() { strip_lib libarrow_dataset.a strip_lib libarrow_acero.a - publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } # abseil @@ -2043,94 +2057,6 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# paimon-cpp -build_paimon_cpp() { - check_if_source_exist "${PAIMON_CPP_SOURCE}" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" - - # Darwin doesn't build GNU libunwind in this script, so don't force -lunwind there. - local paimon_linker_flags="-L${TP_LIB_DIR} -lbrotlienc -lbrotlidec -lbrotlicommon -llzma" - if [[ "${KERNEL}" != 'Darwin' ]]; then - paimon_linker_flags="${paimon_linker_flags} -lunwind" - fi - - CXXFLAGS="-Wno-nontrivial-memcall" \ - "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ - -G "${GENERATOR}" \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ - -DPAIMON_BUILD_SHARED=OFF \ - -DPAIMON_BUILD_STATIC=ON \ - -DPAIMON_BUILD_TESTS=OFF \ - -DPAIMON_ENABLE_ORC=ON \ - -DPAIMON_ENABLE_AVRO=OFF \ - -DPAIMON_ENABLE_LANCE=OFF \ - -DPAIMON_ENABLE_JINDO=OFF \ - -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_LUCENE=OFF \ - -DCMAKE_EXE_LINKER_FLAGS="${paimon_linker_flags}" \ - -DCMAKE_SHARED_LINKER_FLAGS="${paimon_linker_flags}" \ - .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install - - # Install paimon-cpp internal dependencies with renamed versions - # These libraries are built but not installed by default - echo "Installing paimon-cpp internal dependencies..." - - # Arrow deps: When PAIMON_USE_EXTERNAL_ARROW=ON (Plan B), paimon-cpp - # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps - # directory is not needed. When building its own Arrow (legacy), copy - # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" - if [ -d "arrow_ep-install/lib" ]; then - mkdir -p "${paimon_deps_dir}" - for paimon_arrow_dep in \ - libarrow.a \ - libarrow_compute.a \ - libarrow_filesystem.a \ - libarrow_dataset.a \ - libarrow_acero.a \ - libparquet.a; do - if [ -f "arrow_ep-install/lib/${paimon_arrow_dep}" ]; then - cp -v "arrow_ep-install/lib/${paimon_arrow_dep}" "${paimon_deps_dir}/${paimon_arrow_dep}" - fi - done - else - echo " arrow_ep-install not found (PAIMON_USE_EXTERNAL_ARROW=ON?) – skipping paimon_deps Arrow copy" - fi - - # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap - if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" - fi - - # Install xxhash, renamed to avoid conflict with Doris's xxhash - if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" - fi - - # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) - if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" - fi - - # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) - if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" - fi - - echo "Paimon-cpp internal dependencies installed successfully" - publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" -} - # lance-c build_lance_c() { check_if_source_exist "${LANCE_C_SOURCE}" @@ -2268,7 +2194,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then brotli icu pugixml - paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" @@ -2367,7 +2292,6 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; - paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; lzma) src_var="LZMA_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index bb9c5928809170..6b2f6d1970d8cd 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -74,6 +74,15 @@ while [[ $# -gt 0 ]]; do ) done if [[ "${SPEC_LIB}" != "" ]]; then + # Arrow builds xsimd and Brotli from their source archives. + if [[ " ${SPEC_ARCHIVES[*]} " == *' ARROW '* ]]; then + for arrow_companion in XSIMD BROTLI; do + if [[ " ${SPEC_ARCHIVES[*]} " != *" ${arrow_companion} "* ]]; then + SPEC_ARCHIVES+=("${arrow_companion}") + fi + done + fi + TP_ARCHIVES=("${SPEC_ARCHIVES[@]}") echo "Download and build specified libs only: ${TP_ARCHIVES[*]}" fi @@ -334,20 +343,6 @@ echo "===== Patching thirdparty archives..." # This is to avoid duplicated patch. ################################################################################### PATCHED_MARK="patched_mark" -ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" -ARROW_PAIMON_BUILD_FINGERPRINT="" -if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || - " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" -fi - -reset_arrow_paimon_source() { - local archive_name="$1" - local source_name="$2" - echo "Resetting ${source_name} because its patch state is incomplete or stale" - rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" -} # glog patch if [[ " ${TP_ARCHIVES[*]} " =~ " GLOG " ]]; then @@ -453,16 +448,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then cd - fi if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-24.0.0" ]]; then - arrow_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_SOURCE}/${PATCHED_MARK}" && - -f "${arrow_fingerprint_mark}" ]] || - [[ "$(<"${arrow_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${ARROW_NAME}" "${ARROW_SOURCE}" - cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" - # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size, - # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-paimon.patch" - + cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then # Introducing the parameter that forces writing INT96 timestamps for # compatibility with the Doris Parquet writer. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-force-write-int96-timestamps.patch" @@ -470,10 +457,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then # Add Parquet LZO page decompression support used by file scanner v2. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-lzo.patch" touch "${PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - fi + cd - fi echo "Finished patching ${ARROW_SOURCE}" fi @@ -749,31 +734,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# patch paimon-cpp -if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" - PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK="patched_mark_arrow_24_compute" - paimon_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" && - -f "${paimon_fingerprint_mark}" ]] || - [[ "$(<"${paimon_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${PAIMON_CPP_NAME}" "${PAIMON_CPP_SOURCE}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" - touch "${PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - - fi - echo "Finished patching ${PAIMON_CPP_SOURCE}" -fi - # Patch lance-c with the scan execution statistics API from upstream PR #64. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.7" ]]; then diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake deleted file mode 100644 index 6ab6049c59b0d5..00000000000000 --- a/thirdparty/paimon-cpp-cache.cmake +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# CMake Initial Cache for paimon-cpp -# Configures paimon-cpp to reuse selected Doris third-party libraries -# Usage: cmake -C paimon-cpp-cache.cmake ... - -# Get the Doris thirdparty installation directory from environment -set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") - -if(NOT DORIS_THIRDPARTY_DIR) - message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") -endif() - -message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") - -# Set CMAKE_PREFIX_PATH to help find_package locate our libraries -set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") - -# Library and include paths -set(DORIS_LIB_DIR "${DORIS_THIRDPARTY_DIR}/lib" CACHE PATH "Doris library directory") -set(DORIS_INCLUDE_DIR "${DORIS_THIRDPARTY_DIR}/include" CACHE PATH "Doris include directory") - -# ============================================================================ -# ZLIB - Reuse from Doris (version 1.3.1) -# ============================================================================ -set(ZLIB_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZLIB root directory") -set(ZLIB_LIBRARY "${DORIS_LIB_DIR}/libz.a" CACHE FILEPATH "ZLIB library") -set(ZLIB_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZLIB include directory") - -# ============================================================================ -# ZSTD - Reuse from Doris (version 1.5.7) -# ============================================================================ -set(ZSTD_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZSTD root directory") -set(ZSTD_LIBRARY "${DORIS_LIB_DIR}/libzstd.a" CACHE FILEPATH "ZSTD library") -set(ZSTD_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZSTD include directory") - -# ============================================================================ -# LZ4 - Reuse from Doris (version 1.9.4) -# ============================================================================ -set(LZ4_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "LZ4 root directory") -set(LZ4_LIBRARY "${DORIS_LIB_DIR}/liblz4.a" CACHE FILEPATH "LZ4 library") -set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") - -# ============================================================================ -# glog - NOT reused from Doris -# paimon-cpp's build_glog() unconditionally calls externalproject_add() to -# build glog 0.7.1. Any GLOG_ROOT/GLOG_LIBRARY/GLOG_INCLUDE_DIR set here -# would be overwritten by that macro, so we skip them entirely. -# ============================================================================ - -# ============================================================================ -# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp -# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). -# ============================================================================ -set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") - -set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") - -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") - -# Protobuf, Thrift - still built separately by paimon-cpp - -# ============================================================================ -# Snappy - Reuse from Doris -# ============================================================================ -set(Snappy_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory") -set(SNAPPY_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory (legacy)") -set(SNAPPY_LIBRARY "${DORIS_LIB_DIR}/libsnappy.a" CACHE FILEPATH "Snappy library") -set(SNAPPY_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Snappy include directory") - -# ============================================================================ -# Build configuration -# ============================================================================ -set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build with -fPIC") -set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type") - -# Symbol visibility control to prevent conflicts with Doris -# paimon-cpp builds Arrow/ORC/etc with hidden symbols to avoid conflicts -set(CMAKE_CXX_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C++ symbols by default") -set(CMAKE_C_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C symbols by default") -set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inline function symbols") - -# Verify that required libraries exist -if(NOT EXISTS "${ZLIB_LIBRARY}") - message(FATAL_ERROR "ZLIB library not found: ${ZLIB_LIBRARY}") -endif() -if(NOT EXISTS "${ZSTD_LIBRARY}") - message(FATAL_ERROR "ZSTD library not found: ${ZSTD_LIBRARY}") -endif() -if(NOT EXISTS "${LZ4_LIBRARY}") - message(FATAL_ERROR "LZ4 library not found: ${LZ4_LIBRARY}") -endif() -if(NOT EXISTS "${SNAPPY_LIBRARY}") - message(FATAL_ERROR "Snappy library not found: ${SNAPPY_LIBRARY}") -endif() - -message(STATUS "========================================") -message(STATUS "Paimon-cpp Library Reuse Configuration") -message(STATUS "========================================") -message(STATUS "Reusing from Doris:") -message(STATUS " ✓ ZLIB, ZSTD, LZ4, Snappy") -if(PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " ✓ Arrow, Parquet, Arrow Dataset, Arrow Acero (Plan B)") -else() - message(STATUS " ✗ Arrow (building separately, symbol visibility=hidden)") -endif() -message(STATUS "") -message(STATUS "Building separately:") -if(NOT PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " - Arrow, Protobuf, Thrift, ORC") -else() - message(STATUS " - Protobuf, Thrift, ORC") -endif() -message(STATUS " - glog, RapidJSON, TBB") -message(STATUS "========================================") diff --git a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch index 1847a50694f014..85ec1f8021cd9f 100644 --- a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch @@ -1,11 +1,10 @@ --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,8 +215,9 @@ +@@ -215,7 +215,9 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && -- target_unit == ::arrow::TimeUnit::NANO) { +- if (arrow_properties.support_deprecated_int96_timestamps()) { + if (arrow_properties.force_write_int96_timestamps() || + (arrow_properties.support_deprecated_int96_timestamps() && + target_unit == ::arrow::TimeUnit::NANO)) { diff --git a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch deleted file mode 100644 index 583f85b4400bbf..00000000000000 --- a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch +++ /dev/null @@ -1,226 +0,0 @@ ---- a/cpp/src/parquet/arrow/schema.cc -+++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,7 +215,8 @@ - - // The user is explicitly asking for Impala int96 encoding, there is no - // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && -+ target_unit == ::arrow::TimeUnit::NANO) { - *physical_type = ParquetType::INT96; - return Status::OK(); - } ---- a/cpp/src/parquet/arrow/reader.cc -+++ b/cpp/src/parquet/arrow/reader.cc -@@ -1041,25 +1041,32 @@ - ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); - } - -- int64_t num_rows = 0; -+ std::vector num_rows; - for (int row_group : row_groups) { -- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); -+ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); - } - - using ::arrow::RecordBatchIterator; -+ size_t row_group_idx = 0; - - // NB: This lambda will be invoked outside the scope of this call to - // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. - // `this` is a non-owning pointer so we are relying on the parent FileReader outliving - // this RecordBatchReader. - ::arrow::Iterator batches = ::arrow::MakeFunctionIterator( -- [readers, batch_schema, num_rows, -+ [readers, batch_schema, num_rows, row_group_idx, - this]() mutable -> ::arrow::Result { - ::arrow::ChunkedArrayVector columns(readers.size()); - - // don't reserve more rows than necessary -- int64_t batch_size = std::min(properties().batch_size(), num_rows); -- num_rows -= batch_size; -+ int64_t batch_size = 0; -+ if (!num_rows.empty()) { -+ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); -+ num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && row_group_idx + 1 < num_rows.size()) { -+ ++row_group_idx; -+ } -+ } - - RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( - reader_properties_.use_threads(), static_cast(readers.size()), ---- a/cpp/src/parquet/arrow/writer.cc -+++ b/cpp/src/parquet/arrow/writer.cc -@@ -341,6 +341,14 @@ - return Status::OK(); - } - -+ int64_t GetBufferedSize() override { -+ if (row_group_writer_ == nullptr) { -+ return 0; -+ } -+ return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); -+ } -+ - Status Close() override { - if (!closed_) { - // Make idempotent -@@ -444,10 +452,14 @@ - - // Max number of rows allowed in a row group. - const int64_t max_row_group_length = this->properties().max_row_group_length(); -+ const int64_t max_row_group_size = this->properties().max_row_group_size(); - - // Initialize a new buffered row group writer if necessary. - if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || -- row_group_writer_->num_rows() >= max_row_group_length) { -+ row_group_writer_->num_rows() >= max_row_group_length || -+ row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= -+ max_row_group_size) { - RETURN_NOT_OK(NewBufferedRowGroup()); - } - ---- a/cpp/src/parquet/arrow/writer.h -+++ b/cpp/src/parquet/arrow/writer.h -@@ -124,6 +124,9 @@ - /// option in this case. - virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; - -+ /// \brief Return the buffered size in bytes. -+ virtual int64_t GetBufferedSize() = 0; -+ - /// \brief Write the footer and close the file. - virtual ::arrow::Status Close() = 0; - virtual ~FileWriter(); ---- a/cpp/src/parquet/properties.h -+++ b/cpp/src/parquet/properties.h -@@ -160,6 +160,7 @@ - static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; - static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; - static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; -+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; - static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; - static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; - static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -343,6 +344,7 @@ - dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), - write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), - max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), -+ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), - pagesize_(kDefaultDataPageSize), - max_rows_per_page_(kDefaultMaxRowsPerPage), - version_(ParquetVersion::PARQUET_2_6), -@@ -359,6 +361,7 @@ - dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), - write_batch_size_(properties.write_batch_size()), - max_row_group_length_(properties.max_row_group_length()), -+ max_row_group_size_(properties.max_row_group_size()), - pagesize_(properties.data_pagesize()), - max_rows_per_page_(properties.max_rows_per_page()), - version_(properties.version()), -@@ -468,6 +471,13 @@ - return this; - } - -+ /// Specify the max bytes size to put in a single row group. -+ /// Default 128MB. -+ Builder* max_row_group_size(int64_t max_row_group_size) { -+ max_row_group_size_ = max_row_group_size; -+ return this; -+ } -+ - /// Specify the data page size. - /// Default 1MB. - Builder* data_pagesize(int64_t pg_size) { -@@ -867,11 +877,12 @@ - - return std::shared_ptr(new WriterProperties( - pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, -- size_statistics_level_, std::move(file_encryption_properties_), -- default_column_properties_, column_properties, data_page_version_, -- store_decimal_as_integer_, std::move(sorting_columns_), -- content_defined_chunking_enabled_, content_defined_chunking_options_)); -+ max_row_group_size_, pagesize_, max_rows_per_page_, version_, created_by_, -+ page_checksum_enabled_, size_statistics_level_, -+ std::move(file_encryption_properties_), default_column_properties_, -+ column_properties, data_page_version_, store_decimal_as_integer_, -+ std::move(sorting_columns_), content_defined_chunking_enabled_, -+ content_defined_chunking_options_)); - } - - private: -@@ -881,6 +892,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetVersion::type version_; -@@ -917,6 +929,8 @@ - - inline int64_t max_row_group_length() const { return max_row_group_length_; } - -+ inline int64_t max_row_group_size() const { return max_row_group_size_; } -+ - inline int64_t data_pagesize() const { return pagesize_; } - - inline int64_t max_rows_per_page() const { return max_rows_per_page_; } -@@ -1046,9 +1060,10 @@ - private: - explicit WriterProperties( - MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, -- ParquetVersion::type version, const std::string& created_by, -- bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, -+ int64_t max_rows_per_page, ParquetVersion::type version, -+ const std::string& created_by, bool page_write_checksum_enabled, -+ SizeStatisticsLevel size_statistics_level, - std::shared_ptr file_encryption_properties, - const ColumnProperties& default_column_properties, - const std::unordered_map& column_properties, -@@ -1059,6 +1074,7 @@ - dictionary_pagesize_limit_(dictionary_pagesize_limit), - write_batch_size_(write_batch_size), - max_row_group_length_(max_row_group_length), -+ max_row_group_size_(max_row_group_size), - pagesize_(pagesize), - max_rows_per_page_(max_rows_per_page), - parquet_data_page_version_(data_page_version), -@@ -1078,6 +1094,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetDataPageVersion parquet_data_page_version_; ---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1884,7 +1884,20 @@ - REQUIRED_VERSION - 0.11.0) - -- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -+ if(NOT Thrift_VERSION) -+ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -+ set(Thrift_VERSION "${thrift_PC_VERSION}") -+ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -+ set(Thrift_VERSION "${ThriftAlt_VERSION}") -+ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -+ set(Thrift_VERSION "${THRIFT_VERSION}") -+ endif() -+ endif() -+ if(NOT Thrift_VERSION) -+ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -+ endif() -+ -+ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") - list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) - list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) - list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch deleted file mode 100644 index d0e723a34d98ef..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/include/paimon/status.h b/include/paimon/status.h ---- a/include/paimon/status.h -+++ b/include/paimon/status.h -@@ -458,9 +458,4 @@ - } \ - } while (false) - --// This is an internal-use macro and should not be used in public headers. --#ifndef RETURN_NOT_OK --#define RETURN_NOT_OK(s) PAIMON_RETURN_NOT_OK(s) --#endif -- - } // namespace paimon -diff --git a/src/paimon/format/parquet/parquet_input_stream_impl.cpp b/src/paimon/format/parquet/parquet_input_stream_impl.cpp ---- a/src/paimon/format/parquet/parquet_input_stream_impl.cpp -+++ b/src/paimon/format/parquet/parquet_input_stream_impl.cpp -@@ -59,7 +59,7 @@ arrow::Result> ParquetInputStreamImpl::Read(int64 - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, Read(nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -@@ -78,7 +78,7 @@ arrow::Result> ParquetInputStreamImpl::ReadAt(int - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, ReadAt(position, nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp ---- a/src/paimon/format/parquet/file_reader_wrapper.cpp -+++ b/src/paimon/format/parquet/file_reader_wrapper.cpp -@@ -81,8 +81,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); - } -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- target_row_group_indices, target_column_indices_, &batch_reader_)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader_, file_reader_->GetRecordBatchReader(target_row_group_indices, -+ target_column_indices_)); - return Status::OK(); - } - } -@@ -152,9 +153,11 @@ Status FileReaderWrapper::PrepareForReading( - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - std::unique_ptr batch_reader; -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -- column_indices, &batch_reader)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader, -+ file_reader_->GetRecordBatchReader( -+ std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -+ column_indices)); - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch b/thirdparty/patches/paimon-cpp-arrow-24-compute.patch deleted file mode 100644 index 1b581b8f93ee92..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -38,6 +38,8 @@ - set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH - "Include directory for external Arrow/Parquet headers") - set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "" CACHE FILEPATH -+ "Path to external libarrow_compute.a") - set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") - set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") - set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -@@ -1008,6 +1010,7 @@ - - foreach(_paimon_external_lib - IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_COMPUTE_LIB - PAIMON_EXTERNAL_ARROW_DATASET_LIB - PAIMON_EXTERNAL_ARROW_ACERO_LIB - PAIMON_EXTERNAL_PARQUET_LIB -@@ -1028,6 +1031,13 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -+ add_library(arrow_compute STATIC IMPORTED) -+ set_target_properties(arrow_compute -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_COMPUTE_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ - add_library(arrow_dataset STATIC IMPORTED) - set_target_properties(arrow_dataset - PROPERTIES IMPORTED_LOCATION -@@ -1055,7 +1065,9 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -- target_link_libraries(arrow_acero INTERFACE arrow) -+ target_link_libraries(arrow_compute INTERFACE arrow) -+ -+ target_link_libraries(arrow_acero INTERFACE arrow_compute) - - target_link_libraries(arrow_dataset INTERFACE arrow_acero) diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch deleted file mode 100644 index 213cc279f52e5c..00000000000000 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ /dev/null @@ -1,362 +0,0 @@ -diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake -index 74654a4..4065297 100644 ---- a/cmake_modules/BuildUtils.cmake -+++ b/cmake_modules/BuildUtils.cmake -@@ -55,12 +55,18 @@ function(add_paimon_lib LIB_NAME) - # Necessary to make static linking into other shared libraries work properly - set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) - if(ARG_DEPENDENCIES) -- # Avoid add_dependencies on non-existent targets (e.g. when building static only). -+ # In static-only builds, some dependency names are still declared as -+ # *_shared. Map them to *_static when the shared target is unavailable. - set(_paimon_objlib_deps) - foreach(_paimon_dep IN LISTS ARG_DEPENDENCIES) -- if(TARGET ${_paimon_dep}) -- list(APPEND _paimon_objlib_deps ${_paimon_dep}) -+ set(_paimon_mapped_dep "${_paimon_dep}") -+ if(NOT TARGET ${_paimon_mapped_dep} AND _paimon_dep MATCHES "_shared$") -+ string(REGEX REPLACE "_shared$" "_static" _paimon_mapped_dep "${_paimon_dep}") - endif() -+ if(TARGET ${_paimon_mapped_dep}) -+ list(APPEND _paimon_objlib_deps ${_paimon_mapped_dep}) -+ endif() -+ unset(_paimon_mapped_dep) - endforeach() - if(_paimon_objlib_deps) - add_dependencies(${LIB_NAME}_objlib ${_paimon_objlib_deps}) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 57f6c50..d8e3cd0 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -38,6 +38,8 @@ string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) - --set(CMAKE_CXX_STANDARD 17) -+if(NOT DEFINED CMAKE_CXX_STANDARD) -+ set(CMAKE_CXX_STANDARD 17) -+endif() - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -909,9 +909,6 @@ macro(build_orc) - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" -- "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" - "-DLZ4_HOME=${ORC_LZ4_ROOT}" - "-DZSTD_HOME=${ORC_ZSTD_ROOT}" -@@ -923,6 +920,13 @@ macro(build_orc) - -DBUILD_TOOLS=OFF - -DBUILD_CPP_ENABLE_METRICS=ON) - -+ if(ORC_RPATH) -+ list(APPEND ORC_CMAKE_ARGS -+ "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}") -+ endif() -+ - set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/orc.diff") - externalproject_add(orc_ep - URL ${ORC_SOURCE_URL} - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1024,6 +1024,14 @@ macro(build_arrow) - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" - -DARROW_DEPENDENCY_USE_SHARED=OFF -+ # Avoid forcing CONDA dependency mode when CONDA_PREFIX is present. -+ # AUTO keeps the normal "find system first, fallback to bundled" -+ # behavior and prevents accidental pickup of conda's thrift/zstd. -+ -DARROW_DEPENDENCY_SOURCE=AUTO -+ # Isolate from user/system CMake package registries to improve -+ # reproducibility in CI and local mixed environments. -+ -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF -+ -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF - -DARROW_BUILD_SHARED=OFF - -DARROW_BUILD_STATIC=ON - -DARROW_BUILD_TESTS=OFF - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -761,6 +761,9 @@ macro(build_protobuf) - get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}" - DIRECTORY) -+ set(THIRDPARTY_ZLIB_STATIC_LIB -+ "${THIRDPARTY_ZLIB_ROOT}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" -+ ) - - # Strip lto flags (which may be added by dh_auto_configure) - # See https://github.com/protocolbuffers/protobuf/issues/7092 -@@ -778,6 +781,10 @@ macro(build_protobuf) - "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}" - "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}" -+ "-DZLIB_INCLUDE_DIR=${THIRDPARTY_ZLIB_INCLUDE_DIR}" -+ "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_DEBUG=${THIRDPARTY_ZLIB_STATIC_LIB}" - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) - set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS}) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -34,6 +34,16 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") - -+option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF) -+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH -+ "Include directory for external Arrow/Parquet headers") -+set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") -+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") -+set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH -+ "Path to external libarrow_bundled_dependencies.a") -+ - macro(set_urls URLS) - set(${URLS} ${ARGN}) - endmacro() - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -961,5 +961,95 @@ macro(build_orc) - endmacro() - - macro(build_arrow) -- message(STATUS "Building Arrow from source") -+ if(PAIMON_USE_EXTERNAL_ARROW) -+ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include") -+ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") -+ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow" -+ "${ARROW_INCLUDE_DIR}/arrow") -+ endif() -+ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ "${ARROW_INCLUDE_DIR}/parquet") -+ endif() -+ -+ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR) -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON" -+ ) -+ endif() -+ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}") -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}" -+ ) -+ endif() -+ -+ foreach(_paimon_external_lib -+ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_DATASET_LIB -+ PAIMON_EXTERNAL_ARROW_ACERO_LIB -+ PAIMON_EXTERNAL_PARQUET_LIB -+ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB) -+ if(NOT ${_paimon_external_lib}) -+ message(FATAL_ERROR -+ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON") -+ endif() -+ if(NOT EXISTS "${${_paimon_external_lib}}") -+ message(FATAL_ERROR -+ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}") -+ endif() -+ endforeach() -+ -+ add_library(arrow STATIC IMPORTED) -+ set_target_properties(arrow -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_dataset STATIC IMPORTED) -+ set_target_properties(arrow_dataset -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_acero STATIC IMPORTED) -+ set_target_properties(arrow_acero -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(parquet STATIC IMPORTED) -+ set_target_properties(parquet -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_bundled_dependencies STATIC IMPORTED) -+ set_target_properties(arrow_bundled_dependencies -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ target_link_libraries(arrow_acero INTERFACE arrow) -+ -+ target_link_libraries(arrow_dataset INTERFACE arrow_acero) -+ -+ target_link_libraries(arrow -+ INTERFACE zstd -+ snappy -+ lz4 -+ zlib -+ arrow_bundled_dependencies) -+ -+ target_link_libraries(parquet -+ INTERFACE zstd snappy lz4 zlib arrow_bundled_dependencies -+ arrow_dataset) -+ else() -+ message(STATUS "Building Arrow from source") - - get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1121,6 +1121,7 @@ macro(build_arrow) - zlib - arrow_bundled_dependencies - arrow_dataset) -+ endif() - - endmacro(build_arrow) - -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -326,10 +326,10 @@ if(PAIMON_ENABLE_LUMINA) - include_directories("${CMAKE_SOURCE_DIR}/third_party/lumina/include") - endif() - -+include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) - include_directories(SYSTEM ${TBB_INCLUDE_DIR}) - --include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - add_compile_definitions("GLOG_USE_GLOG_EXPORT") - - set(THREADS_PREFER_PTHREAD_FLAG ON) - -diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp ---- a/src/paimon/common/logging/logging.cpp -+++ b/src/paimon/common/logging/logging.cpp -@@ -83,7 +83,7 @@ std::unique_ptr Logger::GetLogger(const std::string& path) { - } - std::unique_lock ulock(getRegistryLock()); - if (!google::IsGoogleLoggingInitialized()) { -- google::InitGoogleLogging(program_invocation_name); -+ google::InitGoogleLogging("paimon-cpp"); - } - return std::make_unique(); - } - -diff --git a/src/paimon/common/memory/memory_pool.cpp b/src/paimon/common/memory/memory_pool.cpp ---- a/src/paimon/common/memory/memory_pool.cpp -+++ b/src/paimon/common/memory/memory_pool.cpp -@@ -55,7 +55,7 @@ void* MemoryPoolImpl::Malloc(uint64_t size, uint64_t alignment) { - return memptr; - } - --void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, size_t alignment) { -+void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment) { - if (alignment == 0) { - void* memptr = ::realloc(p, new_size); - total_allocated_size.fetch_add(new_size - old_size); - -diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp ---- a/src/paimon/format/blob/blob_format_writer.cpp -+++ b/src/paimon/format/blob/blob_format_writer.cpp -@@ -138,7 +138,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - uint64_t total_read_length = 0; -- uint32_t read_len = std::min(file_length, tmp_buffer_->size()); -+ uint32_t read_len = -+ static_cast(std::min(file_length, tmp_buffer_->size())); - while (read_len > 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); - if (static_cast(actual_read_len) != read_len) { -@@ -149,7 +150,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); - total_read_length += actual_read_len; -- read_len = std::min(file_length - total_read_length, tmp_buffer_->size()); -+ read_len = static_cast( -+ std::min(file_length - total_read_length, tmp_buffer_->size())); - } - - // write bin length - ---- a/cmake_modules/arrow.diff -+++ b/cmake_modules/arrow.diff -@@ -196,3 +196,29 @@ - int64_t pagesize_; - ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+index 9df922afa2..5c8b3d4d07 100644 -+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) -+ REQUIRED_VERSION -+ 0.11.0) -+ -+- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -++ if(NOT Thrift_VERSION) -++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -++ set(Thrift_VERSION "${thrift_PC_VERSION}") -++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -++ set(Thrift_VERSION "${ThriftAlt_VERSION}") -++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -++ set(Thrift_VERSION "${THRIFT_VERSION}") -++ endif() -++ endif() -++ if(NOT Thrift_VERSION) -++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -++ endif() -++ -++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") -+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) -+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) -+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) - -diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt -index 76dd03b..9317324 100644 ---- a/src/paimon/CMakeLists.txt -+++ b/src/paimon/CMakeLists.txt -@@ -281,6 +281,11 @@ add_paimon_lib(paimon - ${PAIMON_COMMON_SRCS} - ${PAIMON_CORE_SRCS} - DEPENDENCIES -+ # Codec headers come from ExternalProject install steps, so object -+ # compilation must not race those steps in clean parallel builds. -+ zstd -+ snappy -+ lz4 - arrow - tbb - glog diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh deleted file mode 100755 index 9873218a40a403..00000000000000 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ /dev/null @@ -1,492 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -create_fingerprint_fixture() { - local destination="$1" - - mkdir -p "${destination}/patches" - cp "${ROOT}/arrow-paimon-vars.sh" "${destination}/arrow-paimon-vars.sh" - cp "${ROOT}/paimon-cpp-cache.cmake" "${destination}/paimon-cpp-cache.cmake" - cp "${ROOT}"/patches/apache-arrow-24.0.0-*.patch "${destination}/patches/" - cp "${ROOT}"/patches/paimon-cpp-*.patch "${destination}/patches/" -} - -fingerprint_from_fixture() { - local fixture="$1" - local component="$2" - local result_variable="$3" - local fingerprint - - fingerprint="$( - set -e - # shellcheck source=/dev/null - . "${fixture}/arrow-paimon-vars.sh" - "${component}_build_fingerprint" - )" - printf -v "${result_variable}" '%s' "${fingerprint}" -} - -exercise_semantic_fingerprints() { - local first_fixture="${tmpdir}/fingerprint-first" - local second_fixture="${tmpdir}/fingerprint-second" - local first_arrow - local first_paimon - local second_arrow - local second_paimon - local changed_arrow - local changed_paimon - - create_fingerprint_fixture "${first_fixture}" - create_fingerprint_fixture "${second_fixture}" - - printf '%s\n' first-env >"${first_fixture}/env.sh" - printf '%s\n' second-env >"${second_fixture}/env.sh" - printf '%s\n' first-vars >"${first_fixture}/vars.sh" - printf '%s\n' second-vars >"${second_fixture}/vars.sh" - printf '%s\n' first-download >"${first_fixture}/download-thirdparty.sh" - printf '%s\n' second-download >"${second_fixture}/download-thirdparty.sh" - printf '%s\n' first-build >"${first_fixture}/build-thirdparty.sh" - printf '%s\n' second-build >"${second_fixture}/build-thirdparty.sh" - printf '%s\n' '# release-branch-only comment' >>"${second_fixture}/arrow-paimon-vars.sh" - - fingerprint_from_fixture "${first_fixture}" arrow first_arrow - fingerprint_from_fixture "${first_fixture}" paimon first_paimon - [[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Arrow legacy marker migration target is stale" - [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Paimon legacy marker migration target is stale" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon second_paimon - [[ "${first_arrow}" == "${second_arrow}" ]] || - fail "unrelated branch scripts changed the Arrow fingerprint" - [[ "${first_paimon}" == "${second_paimon}" ]] || - fail "unrelated branch scripts changed the Paimon fingerprint" - - printf '%s\n' semantic-change \ - >>"${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - fingerprint_from_fixture "${second_fixture}" arrow changed_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow patch change did not change the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "an Arrow patch change did not change the Paimon fingerprint" - - cp "${first_fixture}/patches/apache-arrow-24.0.0-lzo.patch" \ - "${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - printf '%s\n' semantic-change >>"${second_fixture}/paimon-cpp-cache.cmake" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${second_arrow}" == "${first_arrow}" ]] || - fail "a Paimon-only cache change changed the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "a Paimon cache change did not change the Paimon fingerprint" - - changed_arrow="$( - set -e - # shellcheck source=/dev/null - . "${first_fixture}/arrow-paimon-vars.sh" - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_build_fingerprint - )" - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow build-schema change did not change its fingerprint" -} - -# shellcheck source=../arrow-paimon-vars.sh -. "${ROOT}/arrow-paimon-vars.sh" -exercise_semantic_fingerprints - -harness="${tmpdir}/harness" -mkdir -p "${harness}/src" "${harness}/patches" "${harness}/installed" -cp "${ROOT}/download-thirdparty.sh" "${harness}/download-thirdparty.sh" - -create_patch() { - local patch_file="$1" - local source_file="$2" - { - printf '%s\n' "--- a/${source_file}" - printf '%s\n' "+++ b/${source_file}" - printf '%s\n' '@@ -1 +1 @@' - printf '%s\n' '-original' - printf '%s\n' '+patched' - } >"${patch_file}" -} - -create_archive() { - local source_name="$1" - local archive_name="$2" - local prefix="$3" - local index - - mkdir -p "${harness}/src/${source_name}" - for index in 1 2 3; do - printf '%s\n' original >"${harness}/src/${source_name}/${prefix}-${index}.txt" - done - tar -czf "${harness}/src/${archive_name}" -C "${harness}/src" "${source_name}" - rm -rf "${harness}/src/${source_name}" -} - -arrow_source="arrow-apache-arrow-24.0.0" -arrow_archive="apache-arrow-24.0.0.tar.gz" -paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" -paimon_archive="paimon-cpp-0a4f4e2.tar.gz" - -create_archive "${arrow_source}" "${arrow_archive}" arrow -create_archive "${paimon_source}" "${paimon_archive}" paimon - -arrow_patches=( - apache-arrow-24.0.0-paimon.patch - apache-arrow-24.0.0-force-write-int96-timestamps.patch - apache-arrow-24.0.0-lzo.patch -) -paimon_patches=( - paimon-cpp-buildutils-static-deps.patch - paimon-cpp-arrow-24-compatibility.patch - paimon-cpp-arrow-24-compute.patch -) - -for index in 0 1 2; do - create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" - create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" -done - -arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" -paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" -{ - printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" - printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" - printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' - printf 'ARROW_NAME="%s"\n' "${arrow_archive}" - printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" - printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" - printf '%s\n' 'ARROW_DOWNLOAD="unused"' - printf 'PAIMON_CPP_NAME="%s"\n' "${paimon_archive}" - printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" - printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" - printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' - printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' -} >"${harness}/vars.sh" - -exercise_interrupted_patch_set() { - local package="$1" - local source_name="$2" - local archive_name="$3" - local prefix="$4" - shift 4 - local patches=("$@") - local applied - local index - - for applied in 1 2 3; do - rm -rf "${harness}/src/${source_name}" - tar -xzf "${harness}/src/${archive_name}" -C "${harness}/src" - for ((index = 0; index < applied; ++index)); do - ( - cd "${harness}/src/${source_name}" - patch -s -p1 <"${harness}/patches/${patches[${index}]}" - ) - done - - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - - for index in 1 2 3; do - [[ "$(<"${harness}/src/${source_name}/${prefix}-${index}.txt")" == "patched" ]] || - fail "${package} did not recover after interruption boundary ${applied}" - done - [[ "$(<"${harness}/src/${source_name}/patched_mark_arrow_paimon_fingerprint")" == "test-fingerprint" ]] || - fail "${package} fingerprint marker is missing" - - touch "${harness}/src/${source_name}/idempotence-sentinel" - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - [[ -f "${harness}/src/${source_name}/idempotence-sentinel" ]] || - fail "${package} reset a completely patched source tree" - done -} - -exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow \ - "${arrow_patches[@]}" -exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ - "${paimon_patches[@]}" - -exercise_generic_recovery_dispatch() { - local generic="${tmpdir}/generic-recovery" - local thirdparty_dir="${generic}/thirdparty" - local external_thirdparty_dir="${generic}/external-thirdparty" - local args_file="${generic}/build-args.txt" - local output_file="${generic}/build-output.txt" - local external_builder_called="${generic}/external-builder-called" - local fake_mvn="${generic}/fake-mvn" - local non_native_status=79 - local non_native_target - local status - local flag - local parallel - local clean - local package1 - local package2 - local extra - - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ - "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ - "${generic}/be/output" - cp "${ROOT}/../build.sh" "${generic}/build.sh" - cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" - printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" - # shellcheck disable=SC2016 - printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" - chmod +x "${fake_mvn}" - { - printf '%s\n' '#!/usr/bin/env bash' - printf 'exit %q\n' "${non_native_status}" - } >"${generic}/generated-source.sh" - chmod +x "${generic}/generated-source.sh" - { - printf '%s\n' 'DORIS_BUILD_PROFILE=0' - printf '%s\n' 'TARGET_SYSTEM=Linux' - printf 'MVN_CMD=%q\n' "${fake_mvn}" - } >"${generic}/env.sh" - { - printf '%s\n' '#!/usr/bin/env bash' - # RECOVERY_ARGS_FILE is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'printf "%s\n" "$*" >"${RECOVERY_ARGS_FILE:?}"' - # shellcheck disable=SC2016 - printf '%s\n' 'exit "${RECOVERY_EXIT_STATUS:-73}"' - } >"${thirdparty_dir}/build-thirdparty.sh" - - DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --clean >"${output_file}" 2>&1 || - fail "bare clean failed before reaching its clean-only exit" - [[ ! -e "${args_file}" ]] || fail "bare clean invoked the thirdparty builder" - [[ ! -d "${generic}/be/build_Release" && ! -d "${generic}/be/output" ]] || - fail "bare clean did not remove BE build outputs" - - for non_native_target in --fe --hive-udf; do - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" "${non_native_target}" >"${output_file}" 2>&1; then - fail "${non_native_target} did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "${non_native_target} failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "${non_native_target} invoked the native Arrow/Paimon builder" - done - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then - fail "--fe --clean did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "--fe --clean failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "--fe --clean invoked the native Arrow/Paimon builder" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic stale-prebuilt recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be --clean >"${output_file}" 2>&1; then - fail "generic clean recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic clean recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - RECOVERY_EXIT_STATUS=0 bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted artifacts that failed current-checkout validation" - fi - grep -Fq "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs" \ - "${output_file}" || fail "generic recovery did not validate artifacts after its builder returned" - - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an invalid install-only thirdparty prefix" - fi - grep -Fq "is an install-only or incomplete prefix" "${output_file}" || - fail "generic recovery did not explain how to refresh an install-only prebuilt" - - { - printf '%s\n' '#!/usr/bin/env bash' - # EXTERNAL_BUILDER_CALLED is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'touch "${EXTERNAL_BUILDER_CALLED:?}"' - } >"${external_thirdparty_dir}/build-thirdparty.sh" - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - EXTERNAL_BUILDER_CALLED="${external_builder_called}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an external thirdparty source tree" - fi - grep -Fq "Cannot rebuild thirdparty libraries with an external source tree" "${output_file}" || - fail "generic recovery did not reject an external thirdparty source tree" - [[ ! -e "${external_builder_called}" ]] || - fail "generic recovery invoked an external thirdparty builder" -} - -exercise_generic_recovery_dispatch - -# A Paimon-only build may publish only its own fingerprint. It must not make a -# stale Arrow installation pass the shared prebuilt validation. -prebuilt="${tmpdir}/prebuilt" -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" -printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" -for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" -done - -prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" -[[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || - fail "focused recovery dispatches a bundled source package as a build target" -[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || - fail "focused recovery does not download the complete Arrow source closure" - -# A legacy prebuilt may have the old combined marker but no component markers. -# Generic build.sh consumers must reject it before importing Arrow Compute. -arrow_paimon_build_fingerprint >"${prebuilt}/arrow-paimon-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "legacy combined marker certified an unversioned component closure" -fi - -printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/arrow-build-fingerprint.txt" -printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/paimon-build-fingerprint.txt" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "the complete shared prebuilt was rejected during fingerprint migration" -if ( - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Arrow marker survived a semantic fingerprint change" -fi -if ( - PAIMON_BUILD_SCHEMA_VERSION="${PAIMON_BUILD_SCHEMA_VERSION}-changed" - paimon_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Paimon marker survived a semantic fingerprint change" -fi - -prebuilt_archive_root="${tmpdir}/prebuilt-archive-root" -prebuilt_archive="${tmpdir}/prebuilt.tar.xz" -mkdir -p "${prebuilt_archive_root}/installed" -cp -a "${prebuilt}/." "${prebuilt_archive_root}/installed/" -tar -C "${prebuilt_archive_root}" -cJf "${prebuilt_archive}" installed - -external_thirdparty="${tmpdir}/external-thirdparty" -mkdir -p "${external_thirdparty}/installed" -touch "${external_thirdparty}/installed/old-image-artifact" -ensure_arrow_paimon_prebuilt_from_url "${external_thirdparty}" \ - "file://${prebuilt_archive}" >/dev/null 2>&1 || - fail "BE UT could not refresh an outdated build-image prebuilt" -arrow_paimon_prebuilt_valid "${external_thirdparty}/installed" || - fail "BE UT installed an invalid build-image prebuilt" -[[ ! -e "${external_thirdparty}/installed/old-image-artifact" ]] || - fail "BE UT kept files from the outdated build-image prebuilt" -ensure_arrow_paimon_prebuilt_from_url "${external_thirdparty}" \ - "file://${tmpdir}/missing-prebuilt.tar.xz" >/dev/null 2>&1 || - fail "BE UT tried to download over a valid build-image prebuilt" - -invalid_archive_root="${tmpdir}/invalid-prebuilt-archive-root" -invalid_archive="${tmpdir}/invalid-prebuilt.tar.xz" -mkdir -p "${invalid_archive_root}/installed" -cp -a "${prebuilt}/." "${invalid_archive_root}/installed/" -rm "${invalid_archive_root}/installed/lib64/libarrow_compute.a" -tar -C "${invalid_archive_root}" -cJf "${invalid_archive}" installed - -invalid_external_thirdparty="${tmpdir}/invalid-external-thirdparty" -mkdir -p "${invalid_external_thirdparty}/installed" -touch "${invalid_external_thirdparty}/installed/preserved-after-invalid-download" -if ensure_arrow_paimon_prebuilt_from_url "${invalid_external_thirdparty}" \ - "file://${invalid_archive}" >/dev/null 2>&1; then - fail "BE UT accepted a downloaded prebuilt without Arrow Compute" -fi -[[ -e "${invalid_external_thirdparty}/installed/preserved-after-invalid-download" ]] || - fail "BE UT replaced the build-image prebuilt before validating its download" - -publish_arrow_prebuilt_marker "${prebuilt}" -publish_paimon_prebuilt_marker "${prebuilt}" -rm "${prebuilt}/lib64/libarrow_compute.a" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "prebuilt validation accepted a missing Arrow Compute archive" -fi -touch "${prebuilt}/lib64/libarrow_compute.a" - -printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon-only marker update certified a stale Arrow build" -fi -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon build accepted a stale installed Arrow" -fi - -publish_arrow_prebuilt_marker "${prebuilt}" -require_arrow_prebuilt_for_paimon "${prebuilt}" || - fail "Paimon build rejected the selected installed Arrow" -arrow_paimon_prebuilt_valid "${prebuilt}" || fail "matching component markers were rejected" - -invalidate_paimon_prebuilt_marker "${prebuilt}" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Paimon rebuild left its old marker valid" -fi -publish_paimon_prebuilt_marker "${prebuilt}" - -invalidate_arrow_prebuilt_marker "${prebuilt}" -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Arrow rebuild left its old marker valid" -fi -publish_arrow_prebuilt_marker "${prebuilt}" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "republished component markers were rejected" - -echo "PASS" diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh deleted file mode 100755 index 4d3b92458b8c26..00000000000000 --- a/thirdparty/test/paimon-codec-dependency-test.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -fixture="${tmpdir}/paimon-codec-dependency" -focused_patch="${fixture}/codec-dependency.patch" -mkdir -p "${fixture}/src/paimon" -{ - printf '%s\n' 'add_paimon_lib(paimon' - printf '%s\n' ' SOURCES' - printf '%s\n' ' ${PAIMON_COMMON_SRCS}' - printf '%s\n' ' ${PAIMON_CORE_SRCS}' - printf '%s\n' ' DEPENDENCIES' - printf '%s\n' ' arrow' - printf '%s\n' ' tbb' - printf '%s\n' ' glog' - printf '%s\n' ' fmt' - printf '%s\n' ' roaring_bitmap' - printf '%s\n' ' xxhash' - printf '%s\n' ' Threads::Threads' - printf '%s\n' ' RapidJSON' - printf '%s\n' ' STATIC_LINK_LIBS' - printf '%s\n' ' arrow)' -} >"${fixture}/src/paimon/CMakeLists.txt" - -awk ' - /^diff --git / { - if (found) { - exit - } - if ($0 == "diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt") { - found = 1 - } - } - found { print } -' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}" -[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is missing" -( - cd "${fixture}" - patch -s -p1 <"${focused_patch}" -) || fail "the Paimon codec dependency patch did not apply" - -dependencies="$(sed -n '/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \ - "${fixture}/src/paimon/CMakeLists.txt")" -grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the ZSTD headers" -grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the Snappy headers" -grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the LZ4 headers" - -echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 219bdef38b03a8..3bd4cd9e2fa136 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -50,10 +50,6 @@ export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" # export REPOSITORY_URL= DORIS_THIRDPARTY_REPOSITORY_URL="${DORIS_THIRDPARTY_REPOSITORY_URL:-https://doris-thirdparty-repo.bj.bcebos.com/thirdparty}" -DORIS_THIRDPARTY_VARS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=arrow-paimon-vars.sh -. "${DORIS_THIRDPARTY_VARS_DIR}/arrow-paimon-vars.sh" - ##################################################### # Download url, filename and unpaced filename # of all thirdparties @@ -145,7 +141,6 @@ LZO2_SOURCE=lzo-2.10 LZO2_MD5SUM="39d3f3f9c55c87b1e5d6888e1420f4b5" # rapidjson -# Updated to match paimon-cpp version (commit 232389d4f1012dddec4ef84861face2d2ba85709) RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" RAPIDJSON_NAME=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz RAPIDJSON_SOURCE=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709 @@ -239,6 +234,12 @@ ZSTD_NAME=zstd-1.5.7.tar.gz ZSTD_SOURCE=zstd-1.5.7 ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48" +# brotli +BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" +BROTLI_NAME="brotli-1.0.9.tar.gz" +BROTLI_SOURCE="brotli-1.0.9" +BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" + # flatbuffers FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz" FLATBUFFERS_NAME=flatbuffers-23.5.26.tar.gz @@ -258,6 +259,12 @@ GRPC_NAME="grpc-v1.54.3.tar.gz" GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" +# arrow +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz" +ARROW_NAME="apache-arrow-24.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-24.0.0" +ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" + # Abseil ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/releases/download/20250512.1/abseil-cpp-20250512.1.tar.gz" ABSEIL_NAME="abseil-cpp-20250512.1.tar.gz" @@ -418,6 +425,13 @@ BENCHMARK_NAME=benchmark-v1.8.0.tar.gz BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" +# xsimd +# for arrow-24.0.0, if arrow upgrade, this version may also need to be changed +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" +XSIMD_NAME="14.0.0.tar.gz" +XSIMD_SOURCE=xsimd-14.0.0 +XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" + # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" SIMDJSON_NAME=simdjson-3.11.6.tar.gz @@ -641,7 +655,6 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' - 'PAIMON_CPP' 'LANCE_C' )