diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index d970796dd21cb7..201e8d0a6404c8 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -75,9 +75,8 @@ jobs: ) brew install "${cellars[@]}" || true - # Paimon's bundled dependencies still require pre-CMake-4 policy - # compatibility. Match the supported version used by the full - # thirdparty jobs instead of relying on Homebrew's latest CMake. + # Match the supported version used by the full thirdparty jobs instead + # of relying on Homebrew's latest CMake. brew unlink cmake || true wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz tar -xzf cmake-3.25.3-macos-universal.tar.gz @@ -101,22 +100,6 @@ jobs: fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz - # Rebuild the Arrow/Paimon stack when the shared prebuilt predates the - # selected Arrow version/component closure. - # shellcheck source=thirdparty/arrow-paimon-vars.sh - . ./arrow-paimon-vars.sh - arrow_paimon_prebuilt_is_valid=false - if arrow_paimon_prebuilt_valid installed; then - arrow_paimon_prebuilt_is_valid=true - fi - if [[ "${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ - -o doris-thirdparty-source.tgz - tar -zxvf doris-thirdparty-source.tgz - ./download-thirdparty.sh arrow paimon_cpp xsimd brotli - export MACOSX_DEPLOYMENT_TARGET=12.0 - ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp - fi popd # macos-15 runners are Apple Silicon (arm64), so the JDK env var is diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 03244548080032..6baff66f3b7aed 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -32,7 +32,7 @@ jobs: if: github.event_name == 'pull_request' outputs: thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }} - arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }} + focused_test_changes: ${{ steps.filter.outputs.focused_test_changes }} steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 @@ -48,41 +48,15 @@ jobs: thirdparty_changes: - 'thirdparty/**' - 'env.sh' - arrow_paimon_lifecycle_changes: - - 'build.sh' - - 'env.sh' - - 'thirdparty/arrow-paimon-vars.sh' - - 'thirdparty/vars.sh' - - 'thirdparty/build-thirdparty.sh' - - 'thirdparty/download-thirdparty.sh' - - 'thirdparty/paimon-cpp-cache.cmake' - - 'thirdparty/patches/apache-arrow-*.patch' - - 'thirdparty/patches/paimon-cpp-*.patch' + focused_test_changes: - 'regression-test/pipeline/external/conf/fe.conf' - - 'thirdparty/test/arrow-paimon-lifecycle-test.sh' - 'thirdparty/test/adbc-jni-config-test.sh' - - 'thirdparty/test/paimon-codec-dependency-test.sh' - '.github/workflows/build-thirdparty.yml' - arrow_paimon_lifecycle_test: - name: Arrow/Paimon Lifecycle Test - needs: changes - if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout ${{ github.ref }} - uses: actions/checkout@v4 - - - name: Test focused thirdparty lifecycle - run: | - thirdparty/test/arrow-paimon-lifecycle-test.sh - thirdparty/test/adbc-jni-config-test.sh - thirdparty/test/paimon-codec-dependency-test.sh - script_test: name: Thirdparty Script Test needs: changes - if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} + if: ${{ needs.changes.outputs.thirdparty_changes == 'true' || needs.changes.outputs.focused_test_changes == 'true' }} runs-on: ubuntu-22.04 steps: - name: Checkout ${{ github.ref }} @@ -104,6 +78,10 @@ jobs: run: | thirdparty/test/azure-vcpkg-retry-test.sh + - name: Test thirdparty configuration + run: | + thirdparty/test/adbc-jni-config-test.sh + build_linux: name: Build Third Party Libraries (Linux) needs: changes diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 0dc2196dc81534..6b7df1b43fc3ab 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -199,8 +199,6 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF) message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") -option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) -set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -218,14 +216,6 @@ if (ENABLE_VARIANT_NESTED_GROUP AND "${VARIANT_NESTED_GROUP_MODULE_DIR}" STREQUA message(FATAL_ERROR "ENABLE_VARIANT_NESTED_GROUP requires VARIANT_NESTED_GROUP_MODULE_DIR") endif() -# Allow env to override when reconfiguring (avoid picking /usr/local). -if (DEFINED ENV{ENABLE_PAIMON_CPP}) - set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) -endif() -if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) - set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) -endif() - set(CMAKE_SKIP_RPATH TRUE) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) @@ -696,10 +686,6 @@ set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ) -if (ENABLE_PAIMON_CPP) - message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode") -endif() - if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) @@ -759,19 +745,6 @@ if (BUILD_BENCHMARK) ) endif() -set(PAIMON_FACTORY_REGISTRY_LIBS) -if (ENABLE_PAIMON_CPP) - # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via - # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). - # No dual-stack selection needed — single Arrow for everything. - - # paimon_parquet_file_format depends on Arrow Dataset symbols. - # Force-link it with --whole-archive so its factory registration runs. - set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format) - list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS}) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_START_GROUP} @@ -789,29 +762,6 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib) -if (ENABLE_PAIMON_CPP) - if (PAIMON_FACTORY_REGISTRY_LIBS) - if (APPLE) - foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS}) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,-force_load,$) - endforeach() - else() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,--whole-archive - ${PAIMON_FACTORY_REGISTRY_LIBS} - -Wl,--no-whole-archive) - endif() - endif() - - # paimon-cpp internal dependencies (renamed with _paimon suffix) - # These must come after paimon libraries to resolve symbols. - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP}) # Add all external dependencies. They should come after the palo libs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index f3bd9867ee0fdd..fc7a38bf7b790e 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -189,18 +189,3 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) - -if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64) - add_thirdparty(paimon_parquet_file_format LIB64) - add_thirdparty(paimon_orc_file_format LIB64) - add_thirdparty(paimon_blob_file_format LIB64) - add_thirdparty(paimon_local_file_system LIB64) - add_thirdparty(paimon_file_index LIB64) - add_thirdparty(paimon_global_index LIB64) - - add_thirdparty(roaring_bitmap_paimon LIB64) - add_thirdparty(xxhash_paimon LIB64) - add_thirdparty(fmt_paimon LIB64) - add_thirdparty(tbb_paimon LIB64) -endif() diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index b4d7ef6116525f..faec885e0ef2e1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1721,40 +1721,10 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB /** Iceberg sink configurations **/ DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB -// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. -// Each entry uses the format "=", and file_type must be one of: -// local, hdfs, s3, http, broker. -DEFINE_Strings(paimon_file_system_scheme_mappings, - "file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs," - "s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3," - "abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http," - "ofs=broker,gfs=broker"); -DEFINE_Validator(paimon_file_system_scheme_mappings, - ([](const std::vector& mappings) -> bool { - doris::StringCaseUnorderedSet seen_schemes; - static const doris::StringCaseUnorderedSet supported_types = { - "local", "hdfs", "s3", "http", "broker"}; - for (const auto& raw_entry : mappings) { - std::string_view entry = doris::trim(raw_entry); - size_t separator = entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - std::string scheme = std::string(doris::trim(entry.substr(0, separator))); - std::string file_type = - std::string(doris::trim(entry.substr(separator + 1))); - if (scheme.empty() || file_type.empty()) { - return false; - } - if (supported_types.find(file_type) == supported_types.end()) { - return false; - } - if (!seen_schemes.insert(scheme).second) { - return false; - } - } - return true; - })); +/** Paimon sink configurations **/ +DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB +DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, + [](int64_t bytes) -> bool { return bytes > 0; }); DEFINE_mInt32(thrift_client_open_num_tries, "1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index c99be92dc23181..2deadaa9f61f1e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1787,8 +1787,9 @@ DECLARE_mInt64(hive_sink_max_file_size); /** Iceberg sink configurations **/ DECLARE_mInt64(iceberg_sink_max_file_size); -/** Paimon file system configurations **/ -DECLARE_Strings(paimon_file_system_scheme_mappings); +/** Paimon sink configurations **/ +// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. +DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); // Number of open tries, default 1 means only try to open once. // Retry the Open num_retries time waiting 100 milliseconds between retries. diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index d15b2c2b5f7c81..b367a799459d3e 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -76,9 +76,7 @@ #include "format/table/iceberg_sys_table_jni_reader.h" #include "format/table/jdbc_jni_reader.h" #include "format/table/max_compute_jni_reader.h" -#include "format/table/paimon_cpp_reader.h" #include "format/table/paimon_jni_reader.h" -#include "format/table/paimon_predicate_converter.h" #include "format/table/paimon_reader.h" #include "format/table/partition_column_filler.h" #include "format/table/remote_doris_reader.h" @@ -1085,50 +1083,11 @@ Status FileScanner::_get_next_reader() { _cur_reader = std::move(mc_reader); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "paimon") { - const auto& paimon_params = range.table_format_params.paimon_params; - bool use_paimon_cpp_reader = false; - if (paimon_params.__isset.reader_type) { - switch (paimon_params.reader_type) { - case TPaimonReaderType::PAIMON_CPP: - use_paimon_cpp_reader = true; - break; - case TPaimonReaderType::PAIMON_JNI: - break; - case TPaimonReaderType::PAIMON_NATIVE: - return Status::InternalError( - "invalid PAIMON_NATIVE reader_type for paimon FORMAT_JNI split, " - "possibly caused by FE/BE protocol mismatch"); - default: - return Status::InternalError( - "unknown paimon reader_type for paimon FORMAT_JNI split, possibly " - "caused by FE/BE protocol mismatch"); - } - } else { - // TODO: Remove this fallback after all FE versions set TPaimonReaderType. - use_paimon_cpp_reader = - _state->query_options().__isset.enable_paimon_cpp_reader && - _state->query_options().enable_paimon_cpp_reader; - } - if (use_paimon_cpp_reader) { - auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state, - _profile, range, _params); - if (!_is_load && !_push_down_conjuncts.empty()) { - PaimonPredicateConverter predicate_converter(_file_slot_descs, _state); - auto predicate = predicate_converter.build(_push_down_conjuncts); - if (predicate) { - cpp_reader->set_predicate(std::move(predicate)); - } - } - init_status = - static_cast(cpp_reader.get())->init_reader(&jni_ctx); - _cur_reader = std::move(cpp_reader); - } else { - auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, - _profile, range, _params); - init_status = - static_cast(paimon_reader.get())->init_reader(&jni_ctx); - _cur_reader = std::move(paimon_reader); - } + auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, + _profile, range, _params); + init_status = + static_cast(paimon_reader.get())->init_reader(&jni_ctx); + _cur_reader = std::move(paimon_reader); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "hudi") { auto hudi_reader = HudiJniReader::create_unique( diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index af574ae216f065..52d71d5f8004ed 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -142,12 +142,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { } const auto& params = range.table_format_params.paimon_params; if (params.__isset.reader_type) { - if (params.reader_type == TPaimonReaderType::PAIMON_JNI) { - return params.__isset.paimon_split; - } - // V2 cannot pass a logical DataSplit through a raw native child without silently - // dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback. - return false; + return params.reader_type == TPaimonReaderType::PAIMON_JNI && + params.__isset.paimon_split; } if (params.__isset.paimon_split) { // Before reader_type was added, an encoded split unambiguously selected the Java diff --git a/be/src/format/generic_reader.h b/be/src/format/generic_reader.h index cc21ce82c2805e..10bad9e463a732 100644 --- a/be/src/format/generic_reader.h +++ b/be/src/format/generic_reader.h @@ -178,7 +178,7 @@ class GenericReader : public ProfileCollector { /// cannot be accidentally overridden. Status init_reader(ReaderInitContext* ctx) { // Apply push_down_agg_type early so _open_file_reader and _do_init_reader - // can use it (e.g., PaimonCppReader skips full init on COUNT pushdown). + // can use it before format-specific initialization. // on_after_init_reader may reset this (e.g., Iceberg with equality deletes). set_push_down_agg_type(ctx->push_down_agg_type); diff --git a/be/src/format/table/paimon_cpp_reader.cpp b/be/src/format/table/paimon_cpp_reader.cpp deleted file mode 100644 index d4164238ec371f..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.cpp +++ /dev/null @@ -1,417 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "arrow/record_batch.h" -#include "arrow/result.h" -#include "core/block/block.h" -#include "core/block/column_with_type_and_name.h" -#include "format/table/paimon_doris_file_system.h" -#include "format/table/partition_column_filler.h" -#include "paimon/defs.h" -#include "paimon/memory/memory_pool.h" -#include "paimon/read_context.h" -#include "paimon/table/source/table_read.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/url_coding.h" - -namespace doris { - -namespace { -constexpr const char* VALUE_KIND_FIELD = "_VALUE_KIND"; - -} // namespace - -PaimonCppReader::PaimonCppReader(const std::vector& file_slot_descs, - RuntimeState* state, RuntimeProfile* profile, - const TFileRangeDesc& range, - const TFileScanRangeParams* range_params) - : _file_slot_descs(file_slot_descs), - _state(state), - _profile(profile), - _range(range), - _range_params(range_params) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz); - if (range.__isset.table_format_params && - range.table_format_params.__isset.table_level_row_count) { - _remaining_table_level_row_count = range.table_format_params.table_level_row_count; - } else { - _remaining_table_level_row_count = -1; - } -} - -PaimonCppReader::~PaimonCppReader() = default; - -Status PaimonCppReader::on_before_init_reader(ReaderInitContext* ctx) { - _column_descs = ctx->column_descs; - _partition_values.clear(); - _partition_value_is_null.clear(); - if (ctx->range == nullptr || ctx->tuple_descriptor == nullptr || - !ctx->range->__isset.columns_from_path_keys) { - return Status::OK(); - } - - DORIS_CHECK(ctx->range->__isset.columns_from_path); - DORIS_CHECK(ctx->range->columns_from_path.size() == ctx->range->columns_from_path_keys.size()); - const bool has_null_flags = ctx->range->__isset.columns_from_path_is_null; - if (has_null_flags) { - DORIS_CHECK(ctx->range->columns_from_path_is_null.size() == - ctx->range->columns_from_path_keys.size()); - } - - std::unordered_map name_to_slot; - for (auto* slot : ctx->tuple_descriptor->slots()) { - name_to_slot.emplace(slot->col_name(), slot); - } - for (size_t i = 0; i < ctx->range->columns_from_path_keys.size(); ++i) { - const auto& key = ctx->range->columns_from_path_keys[i]; - auto slot_it = name_to_slot.find(key); - if (slot_it == name_to_slot.end()) { - continue; - } - _partition_values.emplace( - key, std::make_tuple(ctx->range->columns_from_path[i], slot_it->second)); - _partition_value_is_null.emplace( - key, has_null_flags ? ctx->range->columns_from_path_is_null[i] : false); - } - return Status::OK(); -} - -Status PaimonCppReader::on_after_read_block(Block* block, size_t* read_rows) { - if (_column_descs == nullptr || _partition_values.empty() || *read_rows == 0 || - _push_down_agg_type == TPushAggOp::type::COUNT) { - return Status::OK(); - } - return _fill_partition_columns(block, *read_rows); -} - -Status PaimonCppReader::init_reader() { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - return Status::OK(); - } - return _init_paimon_reader(); -} - -Status PaimonCppReader::_do_get_next_block(Block* block, size_t* read_rows, bool* eof) { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - auto rows = std::min(_remaining_table_level_row_count, - (int64_t)_state->query_options().batch_size); - _remaining_table_level_row_count -= rows; - auto mutable_columns_guard = block->mutate_columns_scoped(); - auto& mutate_columns = mutable_columns_guard.mutable_columns(); - for (auto& col : mutate_columns) { - col->resize(rows); - } - *read_rows = rows; - *eof = false; - if (_remaining_table_level_row_count == 0) { - *eof = true; - } - return Status::OK(); - } - - if (!_batch_reader) { - return Status::InternalError("paimon-cpp reader is not initialized"); - } - - if (_col_name_to_block_idx.empty()) { - _col_name_to_block_idx = block->get_name_to_pos_map(); - } - - auto batch_result = _batch_reader->NextBatch(); - if (!batch_result.ok()) { - return Status::InternalError("paimon-cpp read batch failed: {}", - batch_result.status().ToString()); - } - auto batch = std::move(batch_result).value(); - if (paimon::BatchReader::IsEofBatch(batch)) { - *read_rows = 0; - *eof = true; - return Status::OK(); - } - - arrow::Result> import_result = - arrow::ImportRecordBatch(batch.first.get(), batch.second.get()); - if (!import_result.ok()) { - return Status::InternalError("failed to import paimon-cpp arrow batch: {}", - import_result.status().message()); - } - - auto record_batch = std::move(import_result).ValueUnsafe(); - const auto num_rows = static_cast(record_batch->num_rows()); - const auto num_columns = record_batch->num_columns(); - auto columns_guard = block->mutate_columns_scoped(); - auto& columns = columns_guard.mutable_columns(); - for (int c = 0; c < num_columns; ++c) { - const auto& field = record_batch->schema()->field(c); - if (field->name() == VALUE_KIND_FIELD) { - continue; - } - - auto it = _col_name_to_block_idx.find(field->name()); - if (it == _col_name_to_block_idx.end()) { - // Skip columns that are not in the block (e.g., partition columns handled elsewhere) - continue; - } - const auto block_pos = it->second; - try { - RETURN_IF_ERROR(columns_guard.get_datatype_by_position(block_pos) - ->get_serde() - ->read_column_from_arrow(*columns[block_pos], - record_batch->column(c).get(), 0, - num_rows, _ctzz)); - } catch (Exception& e) { - return Status::InternalError("Failed to convert from arrow to block: {}", e.what()); - } - } - - *read_rows = num_rows; - *eof = false; - return Status::OK(); -} - -Status PaimonCppReader::_get_columns_impl( - std::unordered_map* name_to_type) { - for (const auto& slot : _file_slot_descs) { - name_to_type->emplace(slot->col_name(), slot->type()); - } - return Status::OK(); -} - -Status PaimonCppReader::_fill_partition_columns(Block* block, size_t num_rows) { - if (_col_name_to_block_idx.empty()) { - _col_name_to_block_idx = block->get_name_to_pos_map(); - } - - for (const auto& desc : *_column_descs) { - if (desc.category != ColumnCategory::PARTITION_KEY) { - continue; - } - auto value_it = _partition_values.find(desc.name); - if (value_it == _partition_values.end()) { - continue; - } - auto col_it = _col_name_to_block_idx.find(desc.name); - if (col_it == _col_name_to_block_idx.end()) { - return Status::InternalError("Missing partition column {} in block {}", desc.name, - block->dump_structure()); - } - - auto& column_with_type_and_name = block->get_by_position(col_it->second); - auto mutable_column = std::move(*column_with_type_and_name.column).mutate(); - const auto& [value, slot_desc] = value_it->second; - auto null_it = _partition_value_is_null.find(desc.name); - DORIS_CHECK(null_it != _partition_value_is_null.end()); - RETURN_IF_ERROR(fill_partition_column_from_path_value(*mutable_column, *slot_desc, value, - num_rows, null_it->second)); - column_with_type_and_name.column = std::move(mutable_column); - } - return Status::OK(); -} - -Status PaimonCppReader::close() { - if (_batch_reader) { - _batch_reader->Close(); - } - return Status::OK(); -} - -Status PaimonCppReader::_init_paimon_reader() { - register_paimon_doris_file_system(); - RETURN_IF_ERROR(_decode_split(&_split)); - - auto table_path_opt = _resolve_table_path(); - if (!table_path_opt.has_value()) { - return Status::InternalError( - "paimon-cpp missing paimon_table; cannot resolve paimon table root path"); - } - auto options = _build_options(); - auto read_columns = _build_read_columns(); - - // Avoid moving strings across module boundaries to prevent allocator mismatches in ASAN builds. - std::string table_path = table_path_opt.value(); - static std::once_flag options_log_once; - std::call_once(options_log_once, [&]() { - auto has_key = [&](const char* key) { - auto it = options.find(key); - return (it != options.end() && !it->second.empty()) ? "set" : "empty"; - }; - auto value_or = [&](const char* key) { - auto it = options.find(key); - return it != options.end() ? it->second : std::string(""); - }; - LOG(INFO) << "paimon-cpp options summary: table_path=" << table_path - << " AWS_ACCESS_KEY=" << has_key("AWS_ACCESS_KEY") - << " AWS_SECRET_KEY=" << has_key("AWS_SECRET_KEY") - << " AWS_TOKEN=" << has_key("AWS_TOKEN") - << " AWS_ENDPOINT=" << value_or("AWS_ENDPOINT") - << " AWS_REGION=" << value_or("AWS_REGION") - << " use_path_style=" << value_or("use_path_style") - << " fs.oss.endpoint=" << value_or("fs.oss.endpoint") - << " fs.s3a.endpoint=" << value_or("fs.s3a.endpoint"); - }); - paimon::ReadContextBuilder builder(table_path); - if (!read_columns.empty()) { - builder.SetReadSchema(read_columns); - } - if (!options.empty()) { - builder.SetOptions(options); - } - if (_predicate) { - builder.SetPredicate(_predicate); - builder.EnablePredicateFilter(true); - } - - auto context_result = builder.Finish(); - if (!context_result.ok()) { - return Status::InternalError("paimon-cpp build read context failed: {}", - context_result.status().ToString()); - } - auto context = std::move(context_result).value(); - - auto table_read_result = paimon::TableRead::Create(std::move(context)); - if (!table_read_result.ok()) { - return Status::InternalError("paimon-cpp create table read failed: {}", - table_read_result.status().ToString()); - } - auto table_read = std::move(table_read_result).value(); - auto reader_result = table_read->CreateReader(_split); - if (!reader_result.ok()) { - return Status::InternalError("paimon-cpp create reader failed: {}", - reader_result.status().ToString()); - } - _table_read = std::move(table_read); - _batch_reader = std::move(reader_result).value(); - return Status::OK(); -} - -Status PaimonCppReader::_decode_split(std::shared_ptr* split) { - if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.paimon_params || - !_range.table_format_params.paimon_params.__isset.paimon_split) { - return Status::InternalError("paimon-cpp missing paimon_split in scan range"); - } - const auto& encoded_split = _range.table_format_params.paimon_params.paimon_split; - std::string decoded_split; - if (!base64_decode(encoded_split, &decoded_split)) { - return Status::InternalError("paimon-cpp base64 decode paimon_split failed"); - } - auto pool = paimon::GetDefaultPool(); - auto split_result = - paimon::Split::Deserialize(decoded_split.data(), decoded_split.size(), pool); - if (!split_result.ok()) { - return Status::InternalError("paimon-cpp deserialize split failed: {}", - split_result.status().ToString()); - } - *split = std::move(split_result).value(); - return Status::OK(); -} - -std::optional PaimonCppReader::_resolve_table_path() const { - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_table && - !_range.table_format_params.paimon_params.paimon_table.empty()) { - return _range.table_format_params.paimon_params.paimon_table; - } - return std::nullopt; -} - -std::vector PaimonCppReader::_build_read_columns() const { - std::vector columns; - columns.reserve(_file_slot_descs.size()); - for (const auto& slot : _file_slot_descs) { - columns.emplace_back(slot->col_name()); - } - return columns; -} - -std::map PaimonCppReader::_build_options() const { - std::map options; - if (_range_params && _range_params->__isset.paimon_options && - !_range_params->paimon_options.empty()) { - options.insert(_range_params->paimon_options.begin(), _range_params->paimon_options.end()); - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_options) { - options.insert(_range.table_format_params.paimon_params.paimon_options.begin(), - _range.table_format_params.paimon_params.paimon_options.end()); - } - - if (_range_params && _range_params->__isset.properties && !_range_params->properties.empty()) { - for (const auto& kv : _range_params->properties) { - options[kv.first] = kv.second; - } - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.hadoop_conf) { - for (const auto& kv : _range.table_format_params.paimon_params.hadoop_conf) { - options[kv.first] = kv.second; - } - } - - auto copy_if_missing = [&](const char* from_key, const char* to_key) { - if (options.find(to_key) != options.end()) { - return; - } - auto it = options.find(from_key); - if (it != options.end() && !it->second.empty()) { - options[to_key] = it->second; - } - }; - - // Map common OSS/S3 Hadoop configs to Doris S3 property keys. - copy_if_missing("fs.oss.accessKeyId", "AWS_ACCESS_KEY"); - copy_if_missing("fs.oss.accessKeySecret", "AWS_SECRET_KEY"); - copy_if_missing("fs.oss.sessionToken", "AWS_TOKEN"); - copy_if_missing("fs.oss.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.oss.region", "AWS_REGION"); - copy_if_missing("fs.s3a.access.key", "AWS_ACCESS_KEY"); - copy_if_missing("fs.s3a.secret.key", "AWS_SECRET_KEY"); - copy_if_missing("fs.s3a.session.token", "AWS_TOKEN"); - copy_if_missing("fs.s3a.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.s3a.region", "AWS_REGION"); - copy_if_missing("fs.s3a.path.style.access", "use_path_style"); - - // Backfill file.format/manifest.format from split file_format to avoid - // paimon-cpp falling back to default manifest.format=avro. - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.file_format && - !_range.table_format_params.paimon_params.file_format.empty()) { - const auto& split_file_format = _range.table_format_params.paimon_params.file_format; - auto file_format_it = options.find(paimon::Options::FILE_FORMAT); - if (file_format_it == options.end() || file_format_it->second.empty()) { - options[paimon::Options::FILE_FORMAT] = split_file_format; - } - auto manifest_format_it = options.find(paimon::Options::MANIFEST_FORMAT); - if (manifest_format_it == options.end() || manifest_format_it->second.empty()) { - options[paimon::Options::MANIFEST_FORMAT] = split_file_format; - } - } - - options[paimon::Options::FILE_SYSTEM] = "doris"; - return options; -} - -} // namespace doris diff --git a/be/src/format/table/paimon_cpp_reader.h b/be/src/format/table/paimon_cpp_reader.h deleted file mode 100644 index f695234614104c..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.h +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "common/status.h" -#include "format/generic_reader.h" -#include "paimon/reader/batch_reader.h" -#include "paimon/table/source/split.h" -#include "storage/olap_scan_common.h" - -namespace paimon { -class TableRead; -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeProfile; -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { - -class Block; - -class PaimonCppReader : public GenericReader { - ENABLE_FACTORY_CREATOR(PaimonCppReader); - -public: - PaimonCppReader(const std::vector& file_slot_descs, RuntimeState* state, - RuntimeProfile* profile, const TFileRangeDesc& range, - const TFileScanRangeParams* range_params); - ~PaimonCppReader() override; - - Status init_reader(); - Status _do_get_next_block(Block* block, size_t* read_rows, bool* eof) override; - Status _get_columns_impl(std::unordered_map* name_to_type) override; - Status close() override; - void set_predicate(std::shared_ptr predicate) { - _predicate = std::move(predicate); - } - -protected: - Status on_before_init_reader(ReaderInitContext* ctx) override; - Status on_after_read_block(Block* block, size_t* read_rows) override; - Status _do_init_reader(ReaderInitContext* /*ctx*/) override { return init_reader(); } - -private: - Status _fill_partition_columns(Block* block, size_t num_rows); - Status _init_paimon_reader(); - Status _decode_split(std::shared_ptr* split); - // Resolve paimon table root path for schema/manifest lookup. - std::optional _resolve_table_path() const; - std::vector _build_read_columns() const; - std::map _build_options() const; - - const std::vector& _file_slot_descs; - RuntimeState* _state = nullptr; - [[maybe_unused]] RuntimeProfile* _profile = nullptr; - const TFileRangeDesc& _range; - const TFileScanRangeParams* _range_params = nullptr; - - std::shared_ptr _split; - std::unique_ptr _table_read; - std::unique_ptr _batch_reader; - std::shared_ptr _predicate; - - std::unordered_map> - _partition_values; - std::unordered_map _partition_value_is_null; - std::unordered_map _col_name_to_block_idx; - int64_t _remaining_table_level_row_count = -1; - cctz::time_zone _ctzz; -}; - -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.cpp b/be/src/format/table/paimon_doris_file_system.cpp deleted file mode 100644 index bd303b5e26afb8..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.cpp +++ /dev/null @@ -1,679 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common/config.h" -#include "common/status.h" -#include "io/file_factory.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_system.h" -#include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" -#include "paimon/factories/factory.h" -#include "paimon/fs/file_system.h" -#include "paimon/fs/file_system_factory.h" -#include "paimon/result.h" -#include "paimon/status.h" -#include "util/string_util.h" - -namespace paimon { - -struct ParsedUri { - std::string scheme; - std::string authority; -}; - -ParsedUri parse_uri(const std::string& path) { - ParsedUri parsed; - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos || scheme_pos == 0) { - return parsed; - } - parsed.scheme = doris::to_lower(path.substr(0, scheme_pos)); - size_t authority_start = scheme_pos + delim_len; - if (authority_start >= path.size() || path[authority_start] == '/') { - return parsed; - } - size_t next_slash = path.find('/', authority_start); - if (next_slash == std::string::npos) { - parsed.authority = path.substr(authority_start); - } else { - parsed.authority = path.substr(authority_start, next_slash - authority_start); - } - return parsed; -} - -bool parse_scheme_mapping_target(std::string_view raw_target, doris::TFileType::type* type) { - std::string target = doris::to_lower(std::string(doris::trim(raw_target))); - if (target == "local") { - *type = doris::TFileType::FILE_LOCAL; - return true; - } - if (target == "hdfs") { - *type = doris::TFileType::FILE_HDFS; - return true; - } - if (target == "s3") { - *type = doris::TFileType::FILE_S3; - return true; - } - if (target == "http") { - *type = doris::TFileType::FILE_HTTP; - return true; - } - if (target == "broker") { - *type = doris::TFileType::FILE_BROKER; - return true; - } - return false; -} - -bool parse_scheme_mapping_entry(std::string_view raw_entry, std::string* scheme, - doris::TFileType::type* type) { - size_t separator = raw_entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - *scheme = doris::to_lower(std::string(doris::trim(raw_entry.substr(0, separator)))); - if (scheme->empty()) { - return false; - } - return parse_scheme_mapping_target(raw_entry.substr(separator + 1), type); -} - -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme) { - if (scheme.empty()) { - return doris::TFileType::FILE_HDFS; - } - std::string normalized_scheme = doris::to_lower(scheme); - for (const auto& mapping_entry : doris::config::paimon_file_system_scheme_mappings) { - std::string configured_scheme; - doris::TFileType::type configured_type; - if (!parse_scheme_mapping_entry(mapping_entry, &configured_scheme, &configured_type)) { - continue; - } - if (configured_scheme == normalized_scheme) { - return configured_type; - } - } - return doris::TFileType::FILE_HDFS; -} - -std::string replace_scheme(const std::string& path, const std::string& scheme) { - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return path; - } - return scheme + "://" + path.substr(scheme_pos + delim_len); -} - -std::string normalize_local_path(const std::string& path) { - if (!path.starts_with("file:")) { - return path; - } - constexpr size_t file_prefix_len = 5; - size_t start = file_prefix_len; - if (path.compare(start, 2, "//") == 0 && path.size() - start > 2) { - size_t next_slash = path.find('/', start + 2); - if (next_slash == std::string::npos) { - return ""; - } - start = next_slash; - } - return path.substr(start); -} - -std::string normalize_path_for_type(const std::string& path, const std::string& scheme, - doris::TFileType::type type) { - if (type == doris::TFileType::FILE_LOCAL) { - return normalize_local_path(path); - } - if (type == doris::TFileType::FILE_S3 && scheme != "s3") { - return replace_scheme(path, "s3"); - } - return path; -} - -std::string build_fs_cache_key(doris::TFileType::type type, const ParsedUri& uri, - const std::string& default_fs_name) { - switch (type) { - case doris::TFileType::FILE_LOCAL: - return "local"; - case doris::TFileType::FILE_S3: - return "s3://" + uri.authority; - case doris::TFileType::FILE_HTTP: - return "http://" + uri.authority; - case doris::TFileType::FILE_BROKER: - return "broker"; - case doris::TFileType::FILE_HDFS: - default: - if (!uri.scheme.empty() || !uri.authority.empty()) { - return uri.scheme + "://" + uri.authority; - } - return default_fs_name; - } -} - -paimon::Status to_paimon_status(const doris::Status& status) { - if (status.ok()) { - return paimon::Status::OK(); - } - switch (status.code()) { - case doris::ErrorCode::NOT_FOUND: - case doris::ErrorCode::DIR_NOT_EXIST: - return paimon::Status::NotExist(status.to_string()); - case doris::ErrorCode::ALREADY_EXIST: - case doris::ErrorCode::FILE_ALREADY_EXIST: - return paimon::Status::Exist(status.to_string()); - case doris::ErrorCode::INVALID_ARGUMENT: - case doris::ErrorCode::INVALID_INPUT_SYNTAX: - return paimon::Status::Invalid(status.to_string()); - case doris::ErrorCode::NOT_IMPLEMENTED_ERROR: - return paimon::Status::NotImplemented(status.to_string()); - default: - return paimon::Status::IOError(status.to_string()); - } -} - -std::string join_path(const std::string& base, const std::string& child) { - if (base.empty()) { - return child; - } - if (base.back() == '/') { - return base + child; - } - return base + "/" + child; -} - -std::string parent_path_no_scheme(const std::string& path) { - if (path.empty()) { - return ""; - } - size_t end = path.size(); - while (end > 1 && path[end - 1] == '/') { - --end; - } - size_t pos = path.rfind('/', end - 1); - if (pos == std::string::npos) { - return ""; - } - if (pos == 0) { - return "/"; - } - return path.substr(0, pos); -} - -std::string parent_path(const std::string& path) { - ParsedUri uri = parse_uri(path); - if (uri.scheme.empty()) { - return parent_path_no_scheme(path); - } - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return parent_path_no_scheme(path); - } - size_t start = scheme_pos + delim_len; - size_t slash = path.find('/', start); - if (slash == std::string::npos) { - return ""; - } - std::string path_part = path.substr(slash); - std::string parent_part = parent_path_no_scheme(path_part); - if (parent_part.empty()) { - return ""; - } - std::string prefix = uri.scheme + "://"; - if (!uri.authority.empty()) { - prefix += uri.authority; - } - return prefix + parent_part; -} - -class DorisInputStream : public InputStream { -public: - DorisInputStream(doris::io::FileReaderSPtr reader, std::string path) - : reader_(std::move(reader)), path_(std::move(path)) {} - - Status Seek(int64_t offset, SeekOrigin origin) override { - int64_t target = 0; - if (origin == SeekOrigin::FS_SEEK_SET) { - target = offset; - } else if (origin == SeekOrigin::FS_SEEK_CUR) { - target = position_ + offset; - } else if (origin == SeekOrigin::FS_SEEK_END) { - target = static_cast(reader_->size()) + offset; - } else { - return Status::Invalid("unknown seek origin"); - } - if (target < 0) { - return Status::Invalid("seek position is negative"); - } - position_ = target; - return Status::OK(); - } - - Result GetPos() const override { return position_; } - - Result Read(char* buffer, uint32_t size) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(position_, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - position_ += static_cast(bytes_read); - return static_cast(bytes_read); - } - - Result Read(char* buffer, uint32_t size, uint64_t offset) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(offset, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(bytes_read); - } - - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, - std::function&& callback) override { - Result result = Read(buffer, size, offset); - Status status = Status::OK(); - if (!result.ok()) { - status = result.status(); - } - callback(status); - } - - Result GetUri() const override { return path_; } - - Result Length() const override { return static_cast(reader_->size()); } - - Status Close() override { return to_paimon_status(reader_->close()); } - -private: - doris::io::FileReaderSPtr reader_; - std::string path_; - int64_t position_ = 0; -}; - -class DorisOutputStream : public OutputStream { -public: - DorisOutputStream(doris::io::FileWriterPtr writer, std::string path) - : writer_(std::move(writer)), path_(std::move(path)) {} - - Result Write(const char* buffer, uint32_t size) override { - doris::Status status = writer_->append(doris::Slice(buffer, size)); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(size); - } - - Status Flush() override { return Status::OK(); } - - Result GetPos() const override { - return static_cast(writer_->bytes_appended()); - } - - Result GetUri() const override { return path_; } - - Status Close() override { return to_paimon_status(writer_->close()); } - -private: - doris::io::FileWriterPtr writer_; - std::string path_; -}; - -class DorisBasicFileStatus : public BasicFileStatus { -public: - DorisBasicFileStatus(std::string path, bool is_dir) : path_(std::move(path)), is_dir_(is_dir) {} - - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - -private: - std::string path_; - bool is_dir_; -}; - -class DorisFileStatus : public FileStatus { -public: - DorisFileStatus(std::string path, bool is_dir, uint64_t length, int64_t mtime) - : path_(std::move(path)), is_dir_(is_dir), length_(length), mtime_(mtime) {} - - uint64_t GetLen() const override { return length_; } - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - int64_t GetModificationTime() const override { return mtime_; } - -private: - std::string path_; - bool is_dir_; - uint64_t length_; - int64_t mtime_; -}; - -class DorisFileSystem : public FileSystem { -public: - explicit DorisFileSystem(std::map options) - : options_(std::move(options)) { - auto it = options_.find("fs.defaultFS"); - if (it != options_.end()) { - default_fs_name_ = it->second; - } - } - - Result> Open(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - doris::io::FileReaderSPtr reader; - doris::io::FileReaderOptions reader_options = doris::io::FileReaderOptions::DEFAULT; - doris::Status status = fs->open_file(normalized_path, &reader, &reader_options); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(reader), normalized_path); - } - - Result> Create(const std::string& path, - bool overwrite) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - if (!overwrite) { - bool exists = false; - doris::Status exists_status = fs->exists(normalized_path, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (exists) { - return Status::Exist("file already exists: ", normalized_path); - } - } - std::string parent = parent_path(normalized_path); - if (!parent.empty()) { - doris::Status mkdir_status = fs->create_directory(parent); - if (!mkdir_status.ok()) { - return to_paimon_status(mkdir_status); - } - } - doris::io::FileWriterPtr writer; - doris::Status status = fs->create_file(normalized_path, &writer); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(writer), normalized_path); - } - - Status Mkdirs(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - doris::Status status = resolved.first->create_directory(resolved.second); - return to_paimon_status(status); - } - - Status Rename(const std::string& src, const std::string& dst) const override { - PAIMON_ASSIGN_OR_RAISE(auto src_resolved, resolve_path(src)); - PAIMON_ASSIGN_OR_RAISE(auto dst_resolved, resolve_path(dst)); - doris::Status status = src_resolved.first->rename(src_resolved.second, dst_resolved.second); - return to_paimon_status(status); - } - - Status Delete(const std::string& path, bool recursive = true) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - return Status::OK(); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return to_paimon_status(resolved.first->delete_file(resolved.second)); - } - if (recursive) { - return to_paimon_status(resolved.first->delete_directory(resolved.second)); - } - return to_paimon_status(size_status); - } - - Result> GetFileStatus(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - if (type != doris::TFileType::FILE_S3) { - return Status::NotExist("path not exists: ", resolved.second); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return std::make_unique(resolved.second, false, - static_cast(size), 0); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - - Status ListDir(const std::string& directory, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(directory)); - auto file_status = GetFileStatus(directory); - if (file_status.ok() && !file_status.value()->IsDir()) { - return Status::IOError("path is not a directory: ", directory); - } - std::vector files; - bool exists = false; - doris::Status status = resolved.first->list(resolved.second, false, &files, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file)); - } - return Status::OK(); - } - - Status ListFileStatus(const std::string& path, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto self_status = GetFileStatus(path); - if (!self_status.ok()) { - if (self_status.status().IsNotExist()) { - return Status::OK(); - } - return self_status.status(); - } - if (!self_status.value()->IsDir()) { - status_list->emplace_back(std::move(self_status).value()); - return Status::OK(); - } - std::vector files; - bool exists = false; - doris::Status list_status = resolved.first->list(resolved.second, false, &files, &exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - uint64_t length = file.is_file ? static_cast(file.file_size) : 0; - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file, length, 0)); - } - return Status::OK(); - } - - Result Exists(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status status = resolved.first->exists(resolved.second, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists && type == doris::TFileType::FILE_S3) { - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - return list_exists || !files.empty(); - } - return exists; - } - -private: - Result> resolve_path( - const std::string& path) const { - auto uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - std::string normalized_path = normalize_path_for_type(path, uri.scheme, type); - if (type == doris::TFileType::FILE_LOCAL) { - doris::io::FileSystemSPtr fs = doris::io::global_local_filesystem(); - return std::make_pair(std::move(fs), normalized_path); - } - std::string fs_key = build_fs_cache_key(type, uri, default_fs_name_); - { - std::lock_guard lock(fs_lock_); - auto it = fs_cache_.find(fs_key); - if (it != fs_cache_.end()) { - return std::make_pair(it->second, normalized_path); - } - } - doris::io::FSPropertiesRef fs_properties(type); - const std::map* properties = &options_; - std::map properties_override; - if (type == doris::TFileType::FILE_HTTP && !options_.contains("uri") && - !uri.scheme.empty()) { - properties_override = options_; - properties_override["uri"] = uri.scheme + "://" + uri.authority; - properties = &properties_override; - } - fs_properties.properties = properties; - if (!broker_addresses_.empty()) { - fs_properties.broker_addresses = &broker_addresses_; - } - doris::io::FileDescription file_description = { - .path = normalized_path, .file_size = -1, .mtime = 0, .fs_name = default_fs_name_}; - auto fs_result = doris::FileFactory::create_fs(fs_properties, file_description); - if (!fs_result.has_value()) { - return to_paimon_status(fs_result.error()); - } - doris::io::FileSystemSPtr fs = std::move(fs_result).value(); - { - std::lock_guard lock(fs_lock_); - fs_cache_.emplace(std::move(fs_key), fs); - } - return std::make_pair(std::move(fs), std::move(normalized_path)); - } - - std::map options_; - std::vector broker_addresses_; - std::string default_fs_name_; - mutable std::mutex fs_lock_; - mutable std::unordered_map fs_cache_; -}; - -class DorisFileSystemFactory : public FileSystemFactory { -public: - static const char IDENTIFIER[]; - - const char* Identifier() const override { return IDENTIFIER; } - - Result> Create( - const std::string& path, - const std::map& options) const override { - return std::make_unique(options); - } -}; - -const char DorisFileSystemFactory::IDENTIFIER[] = "doris"; - -REGISTER_PAIMON_FACTORY(DorisFileSystemFactory); - -} // namespace paimon - -namespace doris { - -void register_paimon_doris_file_system() {} - -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.h b/be/src/format/table/paimon_doris_file_system.h deleted file mode 100644 index 561e7aeac30fe4..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include - -namespace paimon { - -// Visible for tests: maps a URI scheme to the Doris file type used by paimon-cpp. -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme); - -} // namespace paimon - -namespace doris { - -// Force-link helper so the paimon-cpp file system factory registration is kept. -void register_paimon_doris_file_system(); - -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.cpp b/be/src/format/table/paimon_predicate_converter.cpp deleted file mode 100644 index 6fa6d4907e7340..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.cpp +++ /dev/null @@ -1,657 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_predicate_converter.h" - -#include -#include -#include - -#include "core/column/column_const.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/field.h" -#include "core/types.h" -#include "core/value/decimalv2_value.h" -#include "core/value/timestamptz_value.h" -#include "core/value/vdatetime_value.h" -#include "exprs/vcompound_pred.h" -#include "exprs/vdirect_in_predicate.h" -#include "exprs/vectorized_fn_call.h" -#include "exprs/vexpr.h" -#include "exprs/vin_predicate.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "paimon/data/decimal.h" -#include "paimon/data/timestamp.h" -#include "paimon/predicate/predicate_builder.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/timezone_utils.h" - -namespace doris { - -PaimonPredicateConverter::PaimonPredicateConverter( - const std::vector& file_slot_descs, RuntimeState* state) - : _state(state) { - _field_index_by_name.reserve(file_slot_descs.size()); - for (size_t i = 0; i < file_slot_descs.size(); ++i) { - const auto& name = file_slot_descs[i]->col_name(); - auto normalized = _normalize_name(name); - if (_field_index_by_name.find(normalized) == _field_index_by_name.end()) { - _field_index_by_name.emplace(std::move(normalized), static_cast(i)); - } - } - - if (!TimezoneUtils::find_cctz_time_zone("GMT", _gmt_tz)) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _gmt_tz); - } -} - -std::shared_ptr PaimonPredicateConverter::build( - const VExprContextSPtrs& conjuncts) { - std::vector> predicates; - predicates.reserve(conjuncts.size()); - for (const auto& conjunct : conjuncts) { - if (!conjunct || !conjunct->root()) { - continue; - } - auto root = conjunct->root(); - if (root->is_rf_wrapper()) { - if (auto impl = root->get_impl()) { - root = impl; - } - } - auto predicate = _convert_expr(root); - if (predicate) { - predicates.emplace_back(std::move(predicate)); - } - } - - if (predicates.empty()) { - return nullptr; - } - if (predicates.size() == 1) { - return predicates.front(); - } - auto and_result = paimon::PredicateBuilder::And(predicates); - if (!and_result.ok()) { - return nullptr; - } - return std::move(and_result).value(); -} - -std::shared_ptr PaimonPredicateConverter::_convert_expr(const VExprSPtr& expr) { - if (!expr) { - return nullptr; - } - - auto uncast = VExpr::expr_without_cast(expr); - - if (auto* direct_in = dynamic_cast(uncast.get())) { - VExprSPtr in_expr; - if (direct_in->get_slot_in_expr(in_expr)) { - return _convert_in(in_expr); - } - return nullptr; - } - - if (dynamic_cast(uncast.get()) != nullptr) { - return _convert_in(uncast); - } - - switch (uncast->op()) { - case TExprOpcode::COMPOUND_AND: - case TExprOpcode::COMPOUND_OR: - return _convert_compound(uncast); - case TExprOpcode::COMPOUND_NOT: - return nullptr; - case TExprOpcode::EQ: - case TExprOpcode::EQ_FOR_NULL: - case TExprOpcode::NE: - case TExprOpcode::GE: - case TExprOpcode::GT: - case TExprOpcode::LE: - case TExprOpcode::LT: - return _convert_binary(uncast); - default: - break; - } - - if (auto* fn = dynamic_cast(uncast.get())) { - auto fn_name = _normalize_name(fn->function_name()); - if (fn_name == "is_null_pred" || fn_name == "is_not_null_pred") { - return _convert_is_null(uncast, fn_name); - } - if (fn_name == "like") { - return _convert_like_prefix(uncast); - } - } - - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_compound( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto left = _convert_expr(expr->get_child(0)); - if (!left) { - return nullptr; - } - auto right = _convert_expr(expr->get_child(1)); - if (!right) { - return nullptr; - } - - if (expr->op() == TExprOpcode::COMPOUND_AND) { - auto and_result = paimon::PredicateBuilder::And({left, right}); - return and_result.ok() ? std::move(and_result).value() : nullptr; - } - if (expr->op() == TExprOpcode::COMPOUND_OR) { - auto or_result = paimon::PredicateBuilder::Or({left, right}); - return or_result.ok() ? std::move(or_result).value() : nullptr; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_in(const VExprSPtr& expr) { - auto* in_pred = dynamic_cast(expr.get()); - if (!in_pred || expr->get_num_children() < 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - std::vector literals; - literals.reserve(expr->get_num_children() - 1); - for (uint16_t i = 1; i < expr->get_num_children(); ++i) { - auto literal = _convert_literal(expr->get_child(i), *field_meta->slot_desc, - field_meta->field_type); - if (!literal) { - return nullptr; - } - literals.emplace_back(std::move(*literal)); - } - - if (literals.empty()) { - return nullptr; - } - if (in_pred->is_not_in()) { - return paimon::PredicateBuilder::NotIn(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); - } - return paimon::PredicateBuilder::In(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); -} - -std::shared_ptr PaimonPredicateConverter::_convert_binary( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - if (expr->op() == TExprOpcode::EQ_FOR_NULL) { - return paimon::PredicateBuilder::IsNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - - auto literal = - _convert_literal(expr->get_child(1), *field_meta->slot_desc, field_meta->field_type); - if (!literal) { - return nullptr; - } - - switch (expr->op()) { - case TExprOpcode::EQ: - return paimon::PredicateBuilder::Equal(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::NE: - return paimon::PredicateBuilder::NotEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GE: - return paimon::PredicateBuilder::GreaterOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GT: - return paimon::PredicateBuilder::GreaterThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LE: - return paimon::PredicateBuilder::LessOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LT: - return paimon::PredicateBuilder::LessThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - default: - break; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_is_null( - const VExprSPtr& expr, const std::string& fn_name) { - if (!expr || expr->get_num_children() != 1) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - if (fn_name == "is_not_null_pred") { - return paimon::PredicateBuilder::IsNotNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - return paimon::PredicateBuilder::IsNull(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type); -} - -std::shared_ptr PaimonPredicateConverter::_convert_like_prefix( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta || field_meta->field_type != paimon::FieldType::STRING) { - return nullptr; - } - - auto pattern_opt = _extract_string_literal(expr->get_child(1)); - if (!pattern_opt) { - return nullptr; - } - const std::string& pattern = *pattern_opt; - if (!pattern.empty() && pattern.front() == '%') { - return nullptr; - } - if (pattern.empty() || pattern.back() != '%') { - return nullptr; - } - - std::string prefix = pattern.substr(0, pattern.size() - 1); - paimon::Literal lower_literal(paimon::FieldType::STRING, prefix.data(), prefix.size()); - auto lower_pred = paimon::PredicateBuilder::GreaterOrEqual( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type, - lower_literal); - - auto upper_prefix = _next_prefix(prefix); - if (!upper_prefix) { - return lower_pred; - } - - paimon::Literal upper_literal(paimon::FieldType::STRING, upper_prefix->data(), - upper_prefix->size()); - auto upper_pred = - paimon::PredicateBuilder::LessThan(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, upper_literal); - auto and_result = paimon::PredicateBuilder::And({lower_pred, upper_pred}); - return and_result.ok() ? std::move(and_result).value() : nullptr; -} - -std::optional PaimonPredicateConverter::_resolve_field( - const VExprSPtr& expr) const { - if (!_state || !expr) { - return std::nullopt; - } - auto slot_expr = VExpr::expr_without_cast(expr); - auto* slot_ref = dynamic_cast(slot_expr.get()); - if (!slot_ref) { - return std::nullopt; - } - auto* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id()); - if (!slot_desc) { - return std::nullopt; - } - auto normalized = _normalize_name(slot_desc->col_name()); - auto it = _field_index_by_name.find(normalized); - if (it == _field_index_by_name.end()) { - return std::nullopt; - } - auto slot_type = slot_desc->type(); - auto field_type = - _to_paimon_field_type(slot_type->get_primitive_type(), slot_type->get_precision()); - if (!field_type) { - return std::nullopt; - } - return FieldMeta {it->second, *field_type, slot_desc}; -} - -std::optional PaimonPredicateConverter::_convert_literal( - const VExprSPtr& expr, const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - PrimitiveType slot_primitive = slot_desc.type()->get_primitive_type(); - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - - Field field; - col->get(0, field); - - switch (slot_primitive) { - case TYPE_BOOLEAN: { - if (literal_primitive != TYPE_BOOLEAN) { - return std::nullopt; - } - return paimon::Literal(static_cast(field.get())); - } - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: { - if (!_is_integer_type(literal_primitive)) { - return std::nullopt; - } - int64_t value = 0; - switch (literal_primitive) { - case TYPE_TINYINT: - value = field.get(); - break; - case TYPE_SMALLINT: - value = field.get(); - break; - case TYPE_INT: - value = field.get(); - break; - case TYPE_BIGINT: - value = field.get(); - break; - default: - return std::nullopt; - } - if (slot_primitive == TYPE_TINYINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_SMALLINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_INT) { - return paimon::Literal(static_cast(value)); - } - return paimon::Literal(value); - } - case TYPE_DOUBLE: { - if (literal_primitive != TYPE_DOUBLE && literal_primitive != TYPE_FLOAT) { - return std::nullopt; - } - double value = 0; - if (literal_primitive == TYPE_FLOAT) { - value = static_cast(field.get()); - } else { - value = field.get(); - } - return paimon::Literal(value); - } - case TYPE_DATE: - case TYPE_DATEV2: { - if (!_is_date_type(literal_primitive)) { - return std::nullopt; - } - int64_t seconds = 0; - if (literal_primitive == TYPE_DATE) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } else if (literal_primitive == TYPE_DATEV2) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } - int32_t days = _seconds_to_days(seconds); - return paimon::Literal(paimon::FieldType::DATE, days); - } - case TYPE_DATETIME: - case TYPE_DATETIMEV2: { - if (!_is_datetime_type(literal_primitive)) { - return std::nullopt; - } - if (literal_primitive == TYPE_DATETIME) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - int64_t seconds = 0; - dt.unix_timestamp(&seconds, _gmt_tz); - return paimon::Literal(paimon::Timestamp::FromEpochMillis(seconds * 1000)); - } - std::pair ts; - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&ts, _gmt_tz); - int64_t millis = ts.first * 1000 + ts.second / 1000; - return paimon::Literal(paimon::Timestamp::FromEpochMillis(millis)); - } - case TYPE_VARCHAR: - case TYPE_STRING: { - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - const auto& value = field.get(); - return paimon::Literal(field_type, value.data(), value.size()); - } - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: { - if (!_is_decimal_type(literal_primitive)) { - return std::nullopt; - } - int32_t precision = static_cast(literal_type->get_precision()); - int32_t scale = static_cast(literal_type->get_scale()); - if (precision <= 0 || precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - - paimon::Decimal::int128_t value = 0; - switch (literal_primitive) { - case TYPE_DECIMALV2: { - const auto& dec = field.get(); - value = dec.value(); - break; - } - case TYPE_DECIMAL32: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL64: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL128I: { - const auto& dec = field.get(); - value = dec.value; - break; - } - default: - return std::nullopt; - } - return paimon::Literal(paimon::Decimal(precision, scale, value)); - } - default: - break; - } - return std::nullopt; -} - -std::optional PaimonPredicateConverter::_extract_string_literal( - const VExprSPtr& expr) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - Field field; - col->get(0, field); - const auto& value = field.get(); - return value; -} - -std::string PaimonPredicateConverter::_normalize_name(std::string_view name) { - std::string out(name); - std::transform(out.begin(), out.end(), out.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return out; -} - -std::optional PaimonPredicateConverter::_next_prefix(const std::string& prefix) { - if (prefix.empty()) { - return std::nullopt; - } - std::string upper = prefix; - for (int i = static_cast(upper.size()) - 1; i >= 0; --i) { - auto c = static_cast(upper[i]); - if (c != 0xFF) { - upper[i] = static_cast(c + 1); - upper.resize(i + 1); - return upper; - } - } - return std::nullopt; -} - -int32_t PaimonPredicateConverter::_seconds_to_days(int64_t seconds) { - static constexpr int64_t kSecondsPerDay = 24 * 60 * 60; - int64_t days = seconds / kSecondsPerDay; - if (seconds < 0 && seconds % kSecondsPerDay != 0) { - --days; - } - return static_cast(days); -} - -bool PaimonPredicateConverter::_is_integer_type(PrimitiveType type) { - switch (type) { - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_string_type(PrimitiveType type) { - return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING; -} - -bool PaimonPredicateConverter::_is_decimal_type(PrimitiveType type) { - switch (type) { - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_date_type(PrimitiveType type) { - return type == TYPE_DATE || type == TYPE_DATEV2; -} - -bool PaimonPredicateConverter::_is_datetime_type(PrimitiveType type) { - return type == TYPE_DATETIME || type == TYPE_DATETIMEV2; -} - -std::optional PaimonPredicateConverter::_to_paimon_field_type( - PrimitiveType type, uint32_t precision) { - switch (type) { - case TYPE_BOOLEAN: - return paimon::FieldType::BOOLEAN; - case TYPE_TINYINT: - return paimon::FieldType::TINYINT; - case TYPE_SMALLINT: - return paimon::FieldType::SMALLINT; - case TYPE_INT: - return paimon::FieldType::INT; - case TYPE_BIGINT: - return paimon::FieldType::BIGINT; - case TYPE_DOUBLE: - return paimon::FieldType::DOUBLE; - case TYPE_VARCHAR: - case TYPE_STRING: - return paimon::FieldType::STRING; - case TYPE_DATE: - case TYPE_DATEV2: - return paimon::FieldType::DATE; - case TYPE_DATETIME: - case TYPE_DATETIMEV2: - return paimon::FieldType::TIMESTAMP; - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - if (precision > 0 && precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - return paimon::FieldType::DECIMAL; - case TYPE_FLOAT: - case TYPE_CHAR: - default: - return std::nullopt; - } -} - -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.h b/be/src/format/table/paimon_predicate_converter.h deleted file mode 100644 index cb391077cd2e7f..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.h +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "core/data_type/define_primitive_type.h" -#include "exprs/vexpr_fwd.h" -#include "paimon/defs.h" -#include "paimon/predicate/literal.h" - -namespace paimon { -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { - -class PaimonPredicateConverter { -public: - PaimonPredicateConverter(const std::vector& file_slot_descs, - RuntimeState* state); - - std::shared_ptr build(const VExprContextSPtrs& conjuncts); - -private: - struct FieldMeta { - int32_t index = -1; - paimon::FieldType field_type = paimon::FieldType::UNKNOWN; - const SlotDescriptor* slot_desc = nullptr; - }; - - std::shared_ptr _convert_expr(const VExprSPtr& expr); - std::shared_ptr _convert_compound(const VExprSPtr& expr); - std::shared_ptr _convert_in(const VExprSPtr& expr); - std::shared_ptr _convert_binary(const VExprSPtr& expr); - std::shared_ptr _convert_is_null(const VExprSPtr& expr, - const std::string& fn_name); - std::shared_ptr _convert_like_prefix(const VExprSPtr& expr); - - std::optional _resolve_field(const VExprSPtr& expr) const; - std::optional _convert_literal(const VExprSPtr& expr, - const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const; - std::optional _extract_string_literal(const VExprSPtr& expr) const; - - static std::string _normalize_name(std::string_view name); - static std::optional _next_prefix(const std::string& prefix); - static int32_t _seconds_to_days(int64_t seconds); - static bool _is_integer_type(PrimitiveType type); - static bool _is_string_type(PrimitiveType type); - static bool _is_decimal_type(PrimitiveType type); - static bool _is_date_type(PrimitiveType type); - static bool _is_datetime_type(PrimitiveType type); - static std::optional _to_paimon_field_type(PrimitiveType type, - uint32_t precision); - - std::unordered_map _field_index_by_name; - RuntimeState* _state = nullptr; - cctz::time_zone _gmt_tz; -}; - -} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 56b9bc8923133f..8a6a0df7ee0ee6 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -80,15 +80,6 @@ TFileRangeDesc hudi_range_with_delta_logs() { return range; } -TFileRangeDesc paimon_cpp_jni_range() { - auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); - TPaimonFileDesc paimon_params; - paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_CPP); - paimon_params.__set_file_format("parquet"); - range.table_format_params.__set_paimon_params(std::move(paimon_params)); - return range; -} - TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); TPaimonFileDesc paimon_params; @@ -509,19 +500,10 @@ TEST(FileScannerV2Test, LegacyCountExemptionRequiresMetadataCountOnEveryRange) { TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { TQueryOptions query_options; query_options.__set_enable_file_scanner_v2(true); - query_options.__set_enable_paimon_cpp_reader(true); TFileScanRangeParams params; params.__set_format_type(TFileFormatType::FORMAT_JNI); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - const auto cpp_range = paimon_cpp_jni_range(); - EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_range)); - const auto cpp_status = FileScannerV2::TEST_validate_scan_range(params, cpp_range); - EXPECT_TRUE(cpp_status.is()); - - // Older FE plans without reader_type used Java whenever the C++ option was disabled. - query_options.__set_enable_paimon_cpp_reader(false); - EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } diff --git a/be/test/format/table/paimon_cpp_reader_test.cpp b/be/test/format/table/paimon_cpp_reader_test.cpp deleted file mode 100644 index f2602c34fd010e..00000000000000 --- a/be/test/format/table/paimon_cpp_reader_test.cpp +++ /dev/null @@ -1,350 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include -#include - -#include -#include -#include -#include - -#include "core/block/block.h" -#include "exec/common/endian.h" -#include "format/format_common.h" -#include "format/table/deletion_vector_reader.h" -#include "format/table/paimon_reader.h" -#include "io/fs/file_meta_cache.h" -#include "io/io_common.h" -#include "roaring/roaring.hh" -#include "runtime/runtime_profile.h" -#include "runtime/runtime_state.h" -#include "util/timezone_utils.h" - -namespace doris { - -namespace { - -std::vector build_paimon_deletion_vector_buffer(const std::vector& positions) { - roaring::Roaring rows; - for (const auto position : positions) { - rows.add(position); - } - - const size_t bitmap_size = rows.getSizeInBytes(); - const uint32_t total_length = static_cast(4 + bitmap_size); - std::vector buffer(4 + total_length); - BigEndian::Store32(buffer.data(), total_length); - constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; - memcpy(buffer.data() + 4, PAIMON_BITMAP_MAGIC, 4); - rows.write(buffer.data() + 8); - return buffer; -} - -TFileScanRangeParams make_local_paimon_scan_params(TFileFormatType::type format_type) { - TFileScanRangeParams scan_params; - scan_params.__set_file_type(TFileType::FILE_LOCAL); - scan_params.__set_format_type(format_type); - return scan_params; -} - -TFileRangeDesc make_paimon_range_with_deletion_file(const std::string& deletion_file_path) { - TFileRangeDesc range; - range.__set_path("data.parquet"); - range.__set_start_offset(0); - range.__set_size(0); - range.__set_fs_name(""); - range.__isset.table_format_params = true; - range.table_format_params.__set_table_format_type("paimon"); - - TPaimonFileDesc paimon_params; - paimon_params.__set_file_format("parquet"); - TPaimonDeletionFileDesc deletion_file; - deletion_file.__set_path(deletion_file_path); - deletion_file.__set_offset(0); - deletion_file.__set_length(16); - paimon_params.__set_deletion_file(deletion_file); - range.table_format_params.__set_paimon_params(paimon_params); - return range; -} - -} // namespace - -class PaimonCppReaderTest : public testing::Test { -protected: - void SetUp() override { - _query_options.__set_batch_size(3); - _runtime_state = std::make_unique(_query_options, _query_globals); - } - - TFileRangeDesc _build_range_with_table_level_row_count(int64_t row_count) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.table_level_row_count = true; - range.table_format_params.table_level_row_count = row_count; - return range; - } - - TQueryOptions _query_options; - TQueryGlobals _query_globals; - std::unique_ptr _runtime_state; - RuntimeProfile _profile {"paimon_cpp_reader_test"}; - std::vector _file_slot_descs; -}; - -TEST_F(PaimonCppReaderTest, CountPushDownUsesTableLevelRowCount) { - auto range = _build_range_with_table_level_row_count(5); - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - reader.set_push_down_agg_type(TPushAggOp::type::COUNT); - - auto init_status = reader.init_reader(); - ASSERT_TRUE(init_status.ok()) << init_status; - - Block block; - size_t read_rows = 0; - bool eof = false; - - auto first_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(first_status.ok()) << first_status; - EXPECT_EQ(3, read_rows); - EXPECT_FALSE(eof); - - auto second_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(second_status.ok()) << second_status; - EXPECT_EQ(2, read_rows); - EXPECT_TRUE(eof); - - auto third_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(third_status.ok()) << third_status; - EXPECT_EQ(0, read_rows); - EXPECT_TRUE(eof); -} - -TEST_F(PaimonCppReaderTest, InitReaderFailsWithoutPaimonSplit) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.paimon_params = true; - range.table_format_params.paimon_params.__isset.paimon_table = true; - range.table_format_params.paimon_params.paimon_table = "s3://bucket/db.tbl"; - - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - auto status = reader.init_reader(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("missing paimon_split"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, DecodeValidBuffer) { - // Scenario: a valid Paimon DV stays compressed after decoding instead of becoming one int64_t - // per deleted row in the query cache. - const auto buffer = build_paimon_deletion_vector_buffer({0, 3, 5}); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(deletion_vector.cardinality(), 3); - EXPECT_TRUE(deletion_vector.contains(uint64_t {0})); - EXPECT_TRUE(deletion_vector.contains(uint64_t {3})); - EXPECT_TRUE(deletion_vector.contains(uint64_t {5})); - EXPECT_FALSE(deletion_vector.contains(uint64_t {4})); -} - -TEST(PaimonDeletionVectorTest, RejectShortBuffer) { - // Scenario: malformed DV content must fail before reading the length and magic fields. - const std::vector buffer(7, '\0'); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("file size too small"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectNullBuffer) { - DeletionVector deletion_vector; - const auto status = decode_paimon_deletion_vector_buffer(nullptr, 8, &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("blob is null"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectLengthMismatch) { - // Scenario: the big-endian length prefix protects against using a truncated or over-read DV - // slice from a shared deletion-vector file. - auto buffer = build_paimon_deletion_vector_buffer({1}); - BigEndian::Store32(buffer.data(), static_cast(buffer.size())); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("length mismatch"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectMagicMismatch) { - // Scenario: Paimon DV buffers have a fixed magic header, so a cache entry or offset pointing - // to unrelated bytes must be rejected. - auto buffer = build_paimon_deletion_vector_buffer({1}); - buffer[4] = '\0'; - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectCorruptRoaringBitmap) { - // Scenario: a buffer with a valid header but incomplete Roaring payload should surface as a - // data-quality error instead of silently producing partial delete rows. - auto buffer = build_paimon_deletion_vector_buffer({1, 2}); - buffer.resize(10); - BigEndian::Store32(buffer.data(), static_cast(buffer.size() - 4)); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("failed to deserialize roaring bitmap"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, CacheKeyIncludesOffsetAndLength) { - // Scenario: different Paimon tables or splits may reference the same DV file with different - // ranges; cache keys must include both offset and length to avoid sharing the wrong bitmap. - TPaimonDeletionFileDesc first_deletion_file; - first_deletion_file.__set_path("s3://bucket/table/deletion.dv"); - first_deletion_file.__set_offset(128); - first_deletion_file.__set_length(64); - - TPaimonDeletionFileDesc different_offset = first_deletion_file; - different_offset.__set_offset(256); - - TPaimonDeletionFileDesc different_length = first_deletion_file; - different_length.__set_length(96); - - const auto first_key = build_paimon_deletion_vector_cache_key(first_deletion_file); - EXPECT_NE(first_key, build_paimon_deletion_vector_cache_key(different_offset)); - EXPECT_NE(first_key, build_paimon_deletion_vector_cache_key(different_length)); -} - -TEST(PaimonDeletionVectorTest, ValidateDescriptorRejectsInvalidRange) { - size_t bytes_read = 0; - - TPaimonDeletionFileDesc missing_path; - missing_path.__set_offset(0); - missing_path.__set_length(4); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(missing_path, bytes_read).ok()); - - TPaimonDeletionFileDesc deletion_file; - deletion_file.__set_path("dv.bin"); - deletion_file.__set_offset(-1); - deletion_file.__set_length(4); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_offset(0); - deletion_file.__set_length(-1); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_length(std::numeric_limits::max()); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 4); - EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - EXPECT_EQ(static_cast(MAX_PAIMON_DELETION_VECTOR_BYTES), bytes_read); - - deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 3); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_offset(3); - deletion_file.__set_length(4); - EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - EXPECT_EQ(bytes_read, 8); -} - -TEST(PaimonDeletionVectorTest, DecodedCacheReportsHitSeparatelyFromFileCache) { - // The decoded cache lookup result is reported by ShardedKVCache itself. The creator represents - // the lower File Cache/read/decode path and must only run for the miss. - ShardedKVCache cache(1); - int create_count = 0; - bool cache_hit = true; - auto create = [&]() { - ++create_count; - auto* deletion_vector = new DeletionVector(); - deletion_vector->add(uint64_t {7}); - return deletion_vector; - }; - - const auto* first = cache.get("dv", create, &cache_hit); - EXPECT_FALSE(cache_hit); - ASSERT_NE(first, nullptr); - EXPECT_TRUE(first->contains(uint64_t {7})); - - const auto* second = cache.get("dv", create, &cache_hit); - EXPECT_TRUE(cache_hit); - EXPECT_EQ(first, second); - EXPECT_EQ(create_count, 1); -} - -TEST(PaimonDeletionVectorTest, V1ParquetReaderReadErrorReleasesCacheEntry) { - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("paimon_v1_parquet_dv_test"); - auto scan_params = make_local_paimon_scan_params(TFileFormatType::FORMAT_PARQUET); - const auto range = make_paimon_range_with_deletion_file( - "./be/test/exec/test_data/missing_paimon_v1_delete_vector.bin"); - cctz::time_zone ctz; - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); - io::IOContext io_ctx; - FileMetaCache meta_cache(1024); - ShardedKVCache kv_cache(8); - - PaimonParquetReader reader(&profile, scan_params, range, 1024, &ctz, &kv_cache, &io_ctx, - &runtime_state, &meta_cache); - const auto status = reader.TEST_init_deletion_vector(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(range.table_format_params.paimon_params.deletion_file.path), - std::string::npos); -} - -TEST(PaimonDeletionVectorTest, V1OrcReaderReadErrorReleasesCacheEntry) { - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("paimon_v1_orc_dv_test"); - auto scan_params = make_local_paimon_scan_params(TFileFormatType::FORMAT_ORC); - auto range = make_paimon_range_with_deletion_file( - "./be/test/exec/test_data/missing_paimon_v1_orc_delete_vector.bin"); - range.table_format_params.paimon_params.__set_file_format("orc"); - io::IOContext io_ctx; - FileMetaCache meta_cache(1024); - ShardedKVCache kv_cache(8); - - PaimonOrcReader reader(&profile, &runtime_state, scan_params, range, 1024, - TimezoneUtils::default_time_zone, &kv_cache, &io_ctx, &meta_cache); - const auto status = reader.TEST_init_deletion_vector(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(range.table_format_params.paimon_params.deletion_file.path), - std::string::npos); -} - -} // namespace doris diff --git a/be/test/format/table/paimon_doris_file_system_test.cpp b/be/test/format/table/paimon_doris_file_system_test.cpp deleted file mode 100644 index a2032dff8188c9..00000000000000 --- a/be/test/format/table/paimon_doris_file_system_test.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include - -#include "common/config.h" - -namespace doris { - -class PaimonDorisFileSystemTest : public testing::Test { -protected: - void SetUp() override { saved_mappings_ = config::paimon_file_system_scheme_mappings; } - - void TearDown() override { config::paimon_file_system_scheme_mappings = saved_mappings_; } - - std::vector saved_mappings_; -}; - -TEST_F(PaimonDorisFileSystemTest, UsesDefaultSchemeMappings) { - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("jfs")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("s3a")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("https")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("ofs")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("unknown")); -} - -TEST_F(PaimonDorisFileSystemTest, AllowsOverridingSchemeMappingsFromConfig) { - config::paimon_file_system_scheme_mappings = {"file=local", "jfs = s3", "gs = hdfs", - "custom-http = http", "custom-broker = broker"}; - - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("JFS")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("custom-http")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("custom-broker")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("still-unknown")); -} - -} // namespace doris diff --git a/build.sh b/build.sh index b5b531ce707e86..8d89dc9eaee3c6 100755 --- a/build.sh +++ b/build.sh @@ -486,59 +486,16 @@ else LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a' fi -# The final-library sentinel only proves that some third-party build completed. It cannot -# distinguish an older prebuilt whose Arrow/Paimon closure predates the selected sources. -# shellcheck source=thirdparty/arrow-paimon-vars.sh -. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh" -NEED_ARROW_PAIMON_THIRDPARTY=false -if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 || - "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" || - "${BUILD_INDEX_TOOL}" == "ON" ]]; then - NEED_ARROW_PAIMON_THIRDPARTY=true -fi - -rebuild_thirdparty_libraries() { - local remove_installed="$1" - shift - local build_script="${DORIS_THIRDPARTY}/build-thirdparty.sh" - local build_args=(-j "${PARALLEL}") - local selected_thirdparty_root - local checkout_thirdparty_root - - if [[ ! -f "${build_script}" ]]; then - echo "Cannot rebuild thirdparty libraries: ${build_script} is missing." >&2 - echo "DORIS_THIRDPARTY=${DORIS_THIRDPARTY} is an install-only or incomplete prefix. Use a matching compilation image/prebuilt, or unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree." >&2 - exit 1 - fi - selected_thirdparty_root="$(cd "${DORIS_THIRDPARTY}" && pwd -P)" - checkout_thirdparty_root="$(cd "${DORIS_HOME}/thirdparty" && pwd -P)" - if [[ "${selected_thirdparty_root}" != "${checkout_thirdparty_root}" ]]; then - echo "Cannot rebuild thirdparty libraries with an external source tree: ${selected_thirdparty_root}." >&2 - echo "Unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree, then use the resulting version-matched installation." >&2 - exit 1 - fi - build_script="${checkout_thirdparty_root}/build-thirdparty.sh" - if [[ "${remove_installed}" == "true" ]]; then - # Some libraries, such as lz4, fail when an earlier installation remains. - rm -rf "${DORIS_THIRDPARTY}/installed" - fi - if [[ "${CLEAN}" -eq 1 ]]; then - build_args+=(--clean) - fi - bash "${build_script}" "${build_args[@]}" "$@" - if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 - exit 1 - fi -} - if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." - rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && - ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" + # need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists + rm -rf "${DORIS_THIRDPARTY}/installed" + + if [[ "${CLEAN}" -eq 0 ]]; then + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" + else + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean + fi fi update_submodule() { diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 30c109dfb2fa14..87cc46c9b2f5a1 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -167,10 +167,6 @@ public class PaimonScanPlanProvider implements ConnectorScanPlanProvider { // PaimonScanNode.getSplits gate, sessionVariable.isForceJniScanner()), bypassing the native ORC/Parquet // readers to dodge native-reader bugs. Default false (legacy default). // - // NOTE: enable_paimon_cpp_reader is deliberately NOT read here. Upstream #66008 removed the paimon-cpp - // arm from PaimonScanNode.setPaimonParams (file-scanner-v2 has no split-aware paimon-cpp adapter and - // hard-rejects a PAIMON_CPP range), so the flag no longer influences planning — see - // PaimonScanRange.populateRangeParams. private static final String FORCE_JNI_SCANNER = "force_jni_scanner"; // Session variable name (byte-identical to SessionVariable.IGNORE_SPLIT_TYPE) surfaced through the same @@ -642,8 +638,8 @@ private List planScanInternal( // FIX-L14: honor the ignore_split_type debugging escape hatch (legacy PaimonScanNode.getSplits): // IGNORE_JNI drops JNI splits (nonDataSplit + DataSplit-JNI arms), IGNORE_NATIVE drops native splits. - // The COUNT(*) arm is never dropped (legacy parity); IGNORE_PAIMON_CPP stays a no-op (legacy getSplits - // never consulted it). Read once here so discarded JNI splits bypass carrier compatibility checks. + // The COUNT(*) arm is never dropped. The deprecated IGNORE_PAIMON_CPP compatibility value stays + // a no-op. Read once here so discarded JNI splits bypass carrier compatibility checks. String ignoreSplitType = resolveIgnoreSplitType(session); boolean ignoreJni = IGNORE_SPLIT_TYPE_JNI.equals(ignoreSplitType); boolean ignoreNative = IGNORE_SPLIT_TYPE_NATIVE.equals(ignoreSplitType); @@ -1433,11 +1429,8 @@ private PaimonScanRange buildJniScanRange(Split split, String defaultFileFormat, String serializedSplit = encodeSplit(split); - // FIX-JNI-FILE-FORMAT (P7-1) + FIX-L11: emit the real data-file format (orc/parquet/avro), NOT "jni". - // JNI routing is gated by the paimon.split property (PaimonScanRange.populateRangeParams), so this - // string only feeds fileDesc.file_format, which BE's paimon_cpp_reader backfills into - // FILE_FORMAT/MANIFEST_FORMAT (an invalid "jni" breaks the manifest read). Mirrors legacy - // PaimonScanNode.setPaimonParams's fileDesc.setFileFormat(getFileFormat(getPathString())): for a + // Emit the real data-file format (orc/parquet/avro), not the reader transport name "jni". + // JNI routing is gated by the paimon.split property. For a // DataSplit the format is the FIRST data-file suffix (falling back to the table default); a // non-DataSplit has no data file and falls back to the table default (legacy DUMMY_PATH -> orElse). String fileFormat = isDataSplit @@ -1883,7 +1876,7 @@ Map getBackendPaimonOptions() { if (!PaimonCatalogProperties.JDBC.equals(catalogProps.getFlavor())) { return options; } - // Forward relevant JDBC catalog properties for BE's paimon-cpp reader + // Forward relevant JDBC catalog properties for the BE Paimon JNI reader. for (Map.Entry entry : properties.entrySet()) { String key = entry.getKey(); if (key.startsWith("jdbc.") || key.equals("warehouse") @@ -2375,12 +2368,9 @@ private static TField buildField(DataType dataType) { } /** - * Serializes a paimon {@link Split} for the BE JNI reader: ALWAYS Java object serialization, which is - * what BE's PaimonJniScanner deserializes. Mirrors upstream {@code PaimonScanNode.setPaimonParams} + - * {@code PaimonUtil.encodeObjectToString} after #66008 removed the paimon-cpp arm — a logical - * {@link DataSplit} may span several files, and file-scanner-v2 has no split-aware paimon-cpp adapter, - * so the native-binary ({@code DataSplit.serialize} / {@code paimon::Split::Deserialize}) encoding is - * never emitted and {@code enable_paimon_cpp_reader} no longer influences the wire format. + * Serializes a paimon {@link Split} for the BE JNI reader using Java object serialization, which is + * what {@code PaimonJniScanner} deserializes. The native-binary + * ({@code DataSplit.serialize}) encoding is not part of this wire path. */ static String encodeSplit(Split split) { return encodeObjectToString(split); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java index 3bc3cbe78cf4c1..220e8ada42e7e3 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java @@ -209,13 +209,7 @@ public void populateRangeParams(TTableFormatFileDesc formatDesc, if (paimonSplitVal != null) { // JNI reader path rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); - // FIX-READER-TYPE (3645dc94306): tell BE's file-scanner-v2 which paimon reader stack to use. - // ALWAYS the Java JNI reader (upstream #66008 removed the paimon-cpp arm from - // PaimonScanNode.setPaimonParams): a logical DataSplit may span several files, and - // file-scanner-v2 has no split-aware paimon-cpp adapter, so it HARD-REJECTS a PAIMON_CPP range - // ("FileScannerV2 does not support table format paimon", file_scanner_v2.cpp - // is_supported_jni_table_format -> _validate_scan_range) with no per-range V1 fallback. - // enable_paimon_cpp_reader is therefore a no-op on the plan path, exactly like on master. + // A serialized logical split is always consumed by the Java JNI reader. fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI); fileDesc.setPaimonSplit(paimonSplitVal); String weightStr = props.get("paimon.self_split_weight"); @@ -297,10 +291,7 @@ public static class Builder { private long start; private long length = -1; private long fileSize = -1; - // Every production caller sets fileFormat explicitly (the real orc/parquet). Default empty (NOT - // "jni", an invalid paimon format): BE's paimon_cpp_reader skips its FILE_FORMAT/MANIFEST_FORMAT - // backfill when this is empty (guarded !file_format.empty()), so a missing set can never inject an - // invalid format (FIX-JNI-FILE-FORMAT). + // Every production caller sets fileFormat explicitly to the real data-file format. private String fileFormat = ""; private Map partitionValues; private long selfSplitWeight; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index e66a38c021d243..fe98a5d9c4f750 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -34,11 +34,8 @@ import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.StorageKind; import org.apache.doris.filesystem.properties.StorageProperties; -import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; -import org.apache.doris.thrift.TPaimonReaderType; import org.apache.doris.thrift.TPrimitiveType; -import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.schema.external.TField; import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TSchema; @@ -1435,7 +1432,7 @@ private static DataSplit buildRealDataSplit(Path warehouse) throws Exception { } } - /** The paimon-cpp NATIVE binary split encoding (DataSplit.serialize + Base64) — what FE must NEVER emit. */ + /** The native binary split encoding (DataSplit.serialize + Base64), which is not used by JNI. */ private static String nativeBinaryEncode(DataSplit dataSplit) throws Exception { java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); dataSplit.serialize(new org.apache.paimon.io.DataOutputViewStreamWrapper(baos)); @@ -1446,21 +1443,14 @@ private static String nativeBinaryEncode(DataSplit dataSplit) throws Exception { public void encodeSplitAlwaysUsesJavaSerializationForDataSplit(@TempDir Path warehouse) throws Exception { DataSplit dataSplit = buildRealDataSplit(warehouse); - // WHY: upstream #66008 removed the paimon-cpp arm from PaimonScanNode.setPaimonParams, so the ONLY - // split wire format FE emits is Java object serialization (what BE's PaimonJniScanner deserializes). - // Emitting the native binary format would now be fatal, not just different: file-scanner-v2 (default - // ON) has no split-aware paimon-cpp adapter, so is_supported_jni_table_format rejects a PAIMON_CPP - // range and _validate_scan_range fails the query — there is no per-range V1 fallback. - // MUTATION: re-adding a cpp/native-binary branch -> the wire stops matching the Java encoding and - // starts matching the native one -> both assertions red. + // The JNI split wire format is Java object serialization, which PaimonJniScanner deserializes. String wire = PaimonScanPlanProvider.encodeSplit(dataSplit); Assertions.assertEquals(feJavaEncode(dataSplit), wire, "a DataSplit must be Java-object-serialized byte-for-byte (the Java JNI reader's format)"); Assertions.assertNotEquals(nativeBinaryEncode(dataSplit), wire, - "FE must never emit the paimon-cpp native binary split format (file-scanner-v2 rejects it)"); + "FE must not emit the native binary split format for a JNI range"); - // Sanity-check the negative reference really is the paimon-cpp format (else the assertion above - // would pass vacuously): it decodes back to an equal DataSplit via paimon's native deserializer. + // Sanity-check that the negative reference decodes through Paimon's native deserializer. byte[] nativeBytes = Base64.getDecoder().decode( nativeBinaryEncode(dataSplit).getBytes(StandardCharsets.UTF_8)); Assertions.assertEquals(dataSplit, DataSplit.deserialize( @@ -1605,10 +1595,8 @@ public void countPushdownCollapsesMultipleSplitsToOneRangeBearingSummedTotal( @Test public void jniAndCountRangesCarryRealFileFormatNotJni(@TempDir Path warehouse) throws Exception { // FIX-JNI-FILE-FORMAT (P7-1): a JNI-serialized split (the default reader path AND the COUNT(*) - // collapse range) must emit the REAL data-file format in fileDesc.file_format, NOT "jni" — BE's - // paimon_cpp_reader backfills paimon FILE_FORMAT/MANIFEST_FORMAT from it (an invalid "jni" breaks - // the manifest read). JNI routing is gated by the paimon.split property, NOT this string, so the - // real format is safe to emit (legacy PaimonScanNode.setPaimonParams does the same). The table is + // collapse range) must emit the REAL data-file format in fileDesc.file_format, NOT "jni". JNI + // routing is gated by the paimon.split property, NOT this string. The table is // created with explicit file.format=orc so the asserted value is the table option (distinct from // the "parquet" fallback) — proving the real option is read, not a constant. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -1683,8 +1671,8 @@ public void jniAndCountRangesUseFileSuffixNotAlteredTableDefault(@TempDir Path w // file_format from the split's FIRST data-file SUFFIX (legacy PaimonScanNode.getFileFormat(getPathString) // -> dataSplitFileFormat), NOT the table-level file.format option. These DIVERGE for an altered / // mixed-format table: the option is changed to parquet while historical data files remain .orc. HEAD - // regressed to emitting the bare table default, so BE's paimon_cpp_reader would backfill the WRONG - // format for those files. WHY it matters: unlike jniAndCountRangesCarryRealFileFormatNotJni (where the + // regressed to emitting the bare table default, which would describe those files incorrectly. + // WHY it matters: unlike jniAndCountRangesCarryRealFileFormatNotJni (where the // table default == the .orc suffix, so it cannot distinguish default from suffix), this test forces a // mismatch and thus is the one that actually goes RED if the emission points revert to defaultFileFormat. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -2148,7 +2136,7 @@ public void ignoreNativeDropsNativeSplit(@TempDir Path warehouse) throws Excepti public void ignorePaimonCppIsNoOpParity(@TempDir Path warehouse) throws Exception { // FIX-L14: IGNORE_PAIMON_CPP is a documented ignore_split_type value that legacy // PaimonScanNode.getSplits NEVER consulted, so it stays a no-op (legacy parity) — the scan emits the - // same ranges as NONE. Pins that a future change does not add a half-implemented CPP arm. + // same ranges as NONE. Retain this compatibility behavior while the enum value remains on the wire. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), new org.apache.paimon.fs.Path(warehouse.toUri()))) { catalog.createDatabase("db", false); @@ -2190,81 +2178,6 @@ public void ignorePaimonCppIsNoOpParity(@TempDir Path warehouse) throws Exceptio } } - @Test - public void cppReaderSessionFlagNoLongerChangesThePlan(@TempDir Path warehouse) throws Exception { - // WHY (upstream #66008): enable_paimon_cpp_reader must be a NO-OP on the plan path. It stays a - // documented (fuzzy=true!) session variable, so the regression fuzzer and the upstream suites - // test_paimon_cpp_reader / test_paimon_partition_*_refs do set it to true — and if the connector - // still answered with PAIMON_CPP, every such query would HARD-FAIL under the default - // enable_file_scanner_v2=true ("FileScannerV2 does not support table format paimon with file format - // FORMAT_JNI": file_scanner_v2.cpp is_supported_jni_table_format -> _validate_scan_range, with no - // per-range fallback to the V1 scanner that still implements PaimonCppReader). - // MUTATION: reinstating a cpp arm keyed off this session flag -> reader_type flips to PAIMON_CPP - // (and paimon_table reappears) -> red. - try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), - new org.apache.paimon.fs.Path(warehouse.toUri()))) { - catalog.createDatabase("db", false); - Identifier id = Identifier.create("db", "t"); - catalog.createTable(id, Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("val", DataTypes.BIGINT()) - .primaryKey("id") - .option("bucket", "1") - .build(), false); - Table table = catalog.getTable(id); - BatchWriteBuilder wb = table.newBatchWriteBuilder(); - try (BatchTableWrite write = wb.newWrite()) { - write.write(GenericRow.of(1, 100L)); - write.write(GenericRow.of(2, 200L)); - List messages = write.prepareCommit(); - try (BatchTableCommit commit = wb.newCommit()) { - commit.commit(messages); - } - } - - RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); - ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); - PaimonTableHandle handle = new PaimonTableHandle( - "db", "t", Collections.emptyList(), Collections.emptyList()); - List noColumns = Collections.emptyList(); - - // force_jni_scanner pins every split onto the JNI arm (the only arm the cpp flag ever touched). - Map cppOn = new HashMap<>(); - cppOn.put("force_jni_scanner", "true"); - cppOn.put("enable_paimon_cpp_reader", "true"); - Map cppOff = new HashMap<>(); - cppOff.put("force_jni_scanner", "true"); - cppOff.put("enable_paimon_cpp_reader", "false"); - - List onRanges = provider.planScan(sessionWithProps(cppOn), - ConnectorScanRequest.builder(handle, noColumns) - .build()); - List offRanges = provider.planScan(sessionWithProps(cppOff), - ConnectorScanRequest.builder(handle, noColumns) - .build()); - Assertions.assertFalse(onRanges.isEmpty(), "baseline scan must emit >=1 JNI range"); - Assertions.assertEquals(offRanges.size(), onRanges.size(), - "the cpp flag must not change the emitted range count"); - - for (int i = 0; i < onRanges.size(); i++) { - TTableFormatFileDesc onDesc = new TTableFormatFileDesc(); - onRanges.get(i).populateRangeParams(onDesc, new TFileRangeDesc()); - TTableFormatFileDesc offDesc = new TTableFormatFileDesc(); - offRanges.get(i).populateRangeParams(offDesc, new TFileRangeDesc()); - - Assertions.assertEquals(TPaimonReaderType.PAIMON_JNI, - onDesc.getPaimonParams().getReaderType(), - "reader_type must stay PAIMON_JNI even with enable_paimon_cpp_reader=true"); - Assertions.assertFalse(onDesc.getPaimonParams().isSetPaimonTable(), - "paimon_table is cpp-reader-only state and must no longer be shipped"); - Assertions.assertEquals(offDesc.getPaimonParams().getPaimonSplit(), - onDesc.getPaimonParams().getPaimonSplit(), - "the split wire format must be identical with the cpp flag on and off"); - } - } - } - // ---- FIX-NATIVE-SUBSPLIT (M-3) ---- private static final long MB = 1024L * 1024L; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java index 67bbfcfe605fc0..ba41ec4c0c6653 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java @@ -37,14 +37,7 @@ * SPI migration to {@code PaimonScanRange} dropped it (the thrift {@code TPaimonFileDesc} was built without * reader_type), so BE could not tell which paimon reader stack a split wanted. * - *

There is deliberately NO {@link TPaimonReaderType#PAIMON_CPP} arm: upstream #66008 removed it from - * {@code PaimonScanNode.setPaimonParams} because a logical {@code DataSplit} may span several files and - * file-scanner-v2 has no split-aware paimon-cpp adapter. Under the default {@code enable_file_scanner_v2 - * = true}, a PAIMON_CPP range is HARD-REJECTED ({@code is_supported_jni_table_format} → - * {@code _validate_scan_range} → "FileScannerV2 does not support table format paimon") with no - * per-range fallback to the V1 scanner that still implements {@code PaimonCppReader}. So the JNI arm must - * answer PAIMON_JNI unconditionally, and {@code enable_paimon_cpp_reader} is a plan-path no-op - * (see {@code PaimonScanPlanProviderTest.cppReaderSessionFlagNoLongerChangesThePlan}). + *

A serialized logical split is always assigned to the Java JNI reader. */ public class PaimonScanRangeReaderTypeTest { @@ -62,18 +55,15 @@ public void jniSplitSetsReaderTypeJniAndNoPaimonTable() { .paimonSplit("java-serialized-split") // JNI marker (paimon.split prop present) .build(); - // MUTATION: dropping setReaderType, or reinstating a cpp arm, turns this red — with reader_type - // absent BE's V2 paimon reader can still infer JNI from paimon_split, but a PAIMON_CPP answer - // fails the query outright (see the class javadoc). + // Pin the explicit reader type so BE does not need to infer it from paimon_split. TTableFormatFileDesc formatDesc = populate(range); Assertions.assertTrue(formatDesc.getPaimonParams().isSetReaderType(), "a JNI split must set reader_type so BE can pick the reader stack"); Assertions.assertEquals(TPaimonReaderType.PAIMON_JNI, formatDesc.getPaimonParams().getReaderType()); - // paimon_table (the table root path) is read ONLY by the V1 PaimonCppReader, so #66008 stopped - // shipping it. MUTATION: re-adding setPaimonTable -> red. + // paimon_table is a deprecated compatibility field and is not needed by the JNI reader. Assertions.assertFalse(formatDesc.getPaimonParams().isSetPaimonTable(), - "paimon_table is cpp-reader-only state and must not be shipped"); + "paimon_table must not be shipped to the JNI reader"); } @Test diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c5b503e9f144b0..cbfcbb93f0e41d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -809,8 +809,6 @@ public String toString() { public static final String FORCE_JNI_SCANNER = "force_jni_scanner"; - public static final String ENABLE_PAIMON_CPP_READER = "enable_paimon_cpp_reader"; - public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table"; @@ -1473,6 +1471,7 @@ public enum IgnoreSplitType { NONE, IGNORE_JNI, IGNORE_NATIVE, + // Deprecated compatibility value. It behaves like NONE because no C++ splits are emitted. IGNORE_PAIMON_CPP } @@ -2927,11 +2926,6 @@ public Map getForceEagerAggHintMap() { description = "Force the use of jni mode to read external table") private boolean forceJniScanner = false; - @VarAttrDef.VarAttr(name = ENABLE_PAIMON_CPP_READER, - fuzzy = true, - description = "Use paimon-cpp for non-native Paimon reads") - private boolean enablePaimonCppReader = false; - @VarAttrDef.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = "enable count(*) pushdown optimization for external table") @@ -3806,8 +3800,6 @@ private void setFuzzyForCatalog(Random random) { // jni this.forceJniScanner = random.nextBoolean(); - this.enablePaimonCppReader = random.nextBoolean(); - // statistics this.fetchHiveRowCountSync = random.nextBoolean(); @@ -5596,7 +5588,6 @@ public TQueryOptions toThrift() { tResult.setEnableParquetFilePageCache(enableParquetFilePageCache); tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax); tResult.setEnableExprZonemapFilter(enableExprZonemapFilter); - tResult.setEnablePaimonCppReader(enablePaimonCppReader); tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds); tResult.setEmbedMaxBatchSize(embedMaxBatchSize); tResult.setAiContextWindowSize(aiContextWindowSize); @@ -6413,10 +6404,6 @@ public boolean isForceJniScanner() { return forceJniScanner; } - public boolean isEnablePaimonCppReader() { - return enablePaimonCppReader; - } - public String getIgnoreSplitType() { return ignoreSplitType; } @@ -6438,10 +6425,6 @@ public void setForceJniScanner(boolean force) { forceJniScanner = force; } - public void setEnablePaimonCppReader(boolean enable) { - enablePaimonCppReader = enable; - } - public boolean isEnableCountPushDownForExternalTable() { return enableCountPushDownForExternalTable; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 07944139285228..afbfe22b02faec 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -439,7 +439,7 @@ struct TQueryOptions { 195: optional bool enable_left_semi_direct_return_opt; 200: optional bool enable_adjust_conjunct_order_by_cost; - // Use paimon-cpp to read Paimon splits on BE + // Deprecated: the paimon-cpp reader has been removed. Retained for wire compatibility. 201: optional bool enable_paimon_cpp_reader = false; // Whether all fragments of this query are assigned to a single backend. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index af4b9c3eb68619..3fa0e56aa4bf14 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -367,6 +367,7 @@ struct TPaimonDeletionFileDesc { enum TPaimonReaderType { PAIMON_NATIVE = 0, PAIMON_JNI = 1, + // Deprecated wire value kept during rolling upgrades. New plans never emit it. PAIMON_CPP = 2, } diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index b866b5d82a42e6..c5d00a47acd4c0 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -451,7 +451,6 @@ _thirdparty_change_affects_artifacts() { if [[ "${af}" == 'thirdparty/test/'* ]] || [[ "${af}" == 'thirdparty/CHANGELOG.md' ]] || [[ "${af}" == 'thirdparty/LICENSE.txt' ]] || - [[ "${af}" == 'thirdparty/arrow-paimon-vars.sh' ]] || [[ "${af}" == 'thirdparty/build-thirdparty.sh' ]]; then return 1 fi diff --git a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md index 5a551b6d3d8e34..4d2f3ae0529c4f 100644 --- a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md @@ -70,7 +70,7 @@ Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contrac | Paimon | Append table, partitioned table, primitive and nested types | Covered | `test_paimon_catalog`, `test_paimon_partition_table`, `test_paimon_full_schema_change` | | Paimon | Primary-key deduplicate, partial-update, aggregation and first-row | Covered | `test_paimon_merge_engine_matrix` | | Paimon | Fixed bucket, dynamic bucket and cross-partition update | Covered | `test_paimon_merge_engine_matrix`, `test_paimon_partition_pk_delete_refs` | -| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_cpp_reader` | +| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_reader_parity` | | Paimon | Snapshot/timestamp/tag/branch and incremental modes | Covered | `paimon_time_travel`, `paimon_incr_read`, `test_paimon_schema_time_travel_matrix` | | Paimon | Schema evolution, partition-key restrictions and historical schema binding | Covered | `test_paimon_schema_time_travel_matrix`, `test_paimon_partition_mutation_atomicity` | | Paimon | Deletion vectors, upsert/delete visibility and data/system tables | Covered | `test_paimon_deletion_vector`, `paimon_data_system_table`, `paimon_system_table` | diff --git a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy index e9f07efdf64d6b..309ad6a48a2c61 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy @@ -138,7 +138,6 @@ suite("paimon_data_system_table", "p0,external") { } sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" // Paimon data system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection @@ -183,7 +182,6 @@ suite("paimon_data_system_table", "p0,external") { qt_jni_native_binlog_rows """select rowkind, id[1], id[2], name[1], name[2] from ${nativeTableName}\$binlog order by id[1]""" qt_jni_native_audit_log_rows """select rowkind, id, name from ${nativeTableName}\$audit_log order by id""" } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" } } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy index 85007d147ba2d0..f8c599804f13f8 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy @@ -321,14 +321,12 @@ suite("test_paimon_partition_pk_delete_refs", where part in ('p1', 'p2') order by part, id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current DV") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_dv"(currentReaderQuery) "qt_${actionSuffix}_jni_historical"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current DV") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_dv"(currentReaderQuery) @@ -336,7 +334,6 @@ suite("test_paimon_partition_pk_delete_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy index 979fbb7784c798..e2b7ffeae99cd4 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy @@ -341,14 +341,12 @@ suite("test_paimon_partition_schema_filter_refs", where part in ('p1', 'p2') order by id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_complex"(currentReaderQuery) "qt_${actionSuffix}_jni_historical_complex"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_complex"(currentReaderQuery) @@ -356,7 +354,6 @@ suite("test_paimon_partition_schema_filter_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy similarity index 81% rename from regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy rename to regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy index 64ef323add6119..4172c8e805b075 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -suite("test_paimon_cpp_reader", "p0,external") { +suite("test_paimon_reader_parity", "p0,external") { String enabled = context.config.otherConfigs.get("enablePaimonTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disabled paimon test") return } - String catalogName = "test_paimon_cpp_reader" + String catalogName = "test_paimon_reader_parity" String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -35,9 +35,6 @@ suite("test_paimon_cpp_reader", "p0,external") { );""" sql """switch ${catalogName}""" sql """use db1""" - // Do not force JNI; keep default selection behavior. - sql """set force_jni_scanner=false""" - def testQueries = [ """select c1 from complex_all order by c1""", """select c1 from complex_all where c1 >= 2 order by c1""", @@ -51,19 +48,17 @@ suite("test_paimon_cpp_reader", "p0,external") { """select * from deletion_vector_parquet""" ] - // Default path is JNI when enable_paimon_cpp_reader=false. - sql """set enable_paimon_cpp_reader=false""" + sql """set force_jni_scanner=true""" def jniResults = testQueries.collect { query -> sql(query) } - sql """set enable_paimon_cpp_reader=true""" - def cppResults = testQueries.collect { query -> sql(query) } + sql """set force_jni_scanner=false""" + def nativeResults = testQueries.collect { query -> sql(query) } - assertTrue(cppResults[0].size() > 0) + assertTrue(nativeResults[0].size() > 0) for (int i = 0; i < testQueries.size(); i++) { - assertEquals(jniResults[i].toString(), cppResults[i].toString()) + assertEquals(jniResults[i].toString(), nativeResults[i].toString()) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index c406b422928179..b13b747e85e3f5 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -367,9 +367,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { sql """refresh catalog ${catalogName}""" // row_tracking exposes Paimon-generated hidden columns, so its incremental scan must - // bypass the C++ reader even when the session otherwise enables native Paimon scans. + // use Paimon-side JNI semantics. sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" String rowTrackingExplain = sql(""" explain verbose select id, name, _ROW_ID, _SEQUENCE_NUMBER @@ -386,8 +385,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { @incr('startSnapshotId'=0, 'endSnapshotId'=1) order by id """ - sql """set enable_paimon_cpp_reader=false""" - // Scenario TC01: validate latest schema/data, explicit new binding, predicate and aggregate. assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], sql("""select id, victim from ${topTable} order by id""")) @@ -571,9 +568,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { // Scenario T14: incremental reads crossing a rename checkpoint bind the end schema. List> incrementalJni - List> incrementalCpp - sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=false""" + List> incrementalAutomatic + sql """set force_jni_scanner=true""" incrementalJni = sql(""" select id, full_name, score from ${pkTable}@incr( @@ -582,8 +578,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - sql """set enable_paimon_cpp_reader=true""" - incrementalCpp = sql(""" + sql """set force_jni_scanner=false""" + incrementalAutomatic = sql(""" select id, full_name, score from ${pkTable}@incr( 'startSnapshotId'='${pkCp0}', @@ -591,7 +587,7 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - assertEquals(incrementalJni, incrementalCpp) + assertEquals(incrementalJni, incrementalAutomatic) // Scenario TC03/S16: partition pruning and renamed payloads bind to their own snapshots. assertEquals([[1, "p1", "old"], [2, "p2", "old"]], @@ -691,7 +687,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { """)) sql """switch ${catalogName}""" sql """use ${dbName}""" - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" List> forcedJniRows = sql(""" select id, old_name, victim, metric @@ -699,21 +694,19 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { order by id """) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" - List> cppRows = sql(""" + List> automaticRows = sql(""" select id, old_name, victim, metric from ${topTable} for version as of ${topCp0} order by id """) - assertEquals(forcedJniRows, cppRows) - // Schema-selecting OPTIONS must bypass paimon-cpp, whose table handle always uses the - // latest schema, even when native Paimon scans are enabled for the session. - List> cppOptionsRows = sql(""" + assertEquals(forcedJniRows, automaticRows) + // Schema-selecting OPTIONS retain the requested historical schema under automatic routing. + List> automaticOptionsRows = sql(""" select id, old_name, victim, metric from ${topTable}@options('scan.snapshot-id'='${topCp0}') order by id """) - assertEquals(forcedJniRows, cppOptionsRows) + assertEquals(forcedJniRows, automaticOptionsRows) // Scenario T10/T11: retained tags survive expiration; missing refs never fall back to latest. spark_paimon """ @@ -741,7 +734,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { exception "tag" } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop database if exists internal.${viewDb} force""" sql """drop catalog if exists ${catalogName}""" diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh deleted file mode 100644 index d532a740e63d68..00000000000000 --- a/thirdparty/arrow-paimon-vars.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/bin/bash -# shellcheck disable=2034 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can -# distinguish this stack from unrelated thirdparty changes. - -# arrow -ARROW_VERSION="24.0.0" -ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" -ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" - -# Arrow bundled dependencies -BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" -BROTLI_NAME="brotli-1.0.9.tar.gz" -BROTLI_SOURCE="brotli-1.0.9" -BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" - -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" -XSIMD_NAME="14.0.0.tar.gz" -XSIMD_SOURCE=xsimd-14.0.0 -XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" - -# paimon-cpp -PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" -PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" - -# Bump the corresponding schema version whenever output-affecting build options or -# helper behavior in build_arrow() or build_paimon_cpp() changes. The fingerprints -# below intentionally describe only this component stack so master and release -# branches can reuse the same shared prebuilt when their semantic inputs match. -ARROW_BUILD_SCHEMA_VERSION="1" -PAIMON_BUILD_SCHEMA_VERSION="1" - -# The shared automation prebuilt is still published from master, which computes the -# former whole-script fingerprint over env.sh, vars.sh, download-thirdparty.sh and -# build-thirdparty.sh. Every republication triggered by an unrelated thirdparty edit -# therefore stamps a different marker, so list each one still in circulation, newest -# first. These markers are only consulted while the selected inputs still hash to the -# pinned semantic fingerprint, and version plus complete artifact validation stay -# mandatory before reuse. Delete the lists once master has published a prebuilt that -# carries the semantic marker itself. -ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" -# Dependency ordering does not change the installed Paimon artifacts, so existing -# complete prebuilts remain compatible with the corrected clean-build graph. -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c" -ARROW_LEGACY_BUILD_FINGERPRINTS=( - # published 2026-08-19, master after apache/doris#66842 - 8356a3a464b93f3a59a6bdae6a51258947ab35d2 - # published 2026-08-05, master after apache/doris#66221 - 9d03645dd1cded5184a8126f5c7f4a6eb9b92b53 -) -PAIMON_LEGACY_BUILD_FINGERPRINTS=( - # published 2026-08-19, master after apache/doris#66842 - 2bbf52e719bdbc8aaa428caab200ac13848f92e5 - # published 2026-08-05, master after apache/doris#66221 - dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28 -) - -# Arrow consumes xsimd and Brotli as bundled source archives, but neither is a -# build target in the focused Arrow/Paimon recovery path. -ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) -ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) -ARROW_PAIMON_DOWNLOAD_PACKAGES=() - -prepare_arrow_paimon_download_packages() { - ARROW_PAIMON_DOWNLOAD_PACKAGES=("$@") - - local package - local source_package - local arrow_requested=false - local source_requested - for package in "$@"; do - if [[ "${package}" == "arrow" ]]; then - arrow_requested=true - break - fi - done - if [[ "${arrow_requested}" != "true" ]]; then - return - fi - - for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do - source_requested=false - for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do - if [[ "${package}" == "${source_package}" ]]; then - source_requested=true - break - fi - done - if [[ "${source_requested}" != "true" ]]; then - ARROW_PAIMON_DOWNLOAD_PACKAGES+=("${source_package}") - fi - done -} - -# Print stable path-and-content records for fingerprint inputs. Including the path -# makes patch selection and ordering part of the contract, not only file contents. -arrow_paimon_fingerprint_files() { - local file - local blob - - # Feed the sorted list through a pipe rather than a process substitution: - # build.sh is documented as "sh build.sh", and bash 3.2 invoked as sh - # (macOS /bin/sh) rejects "< <(...)" at parse time, which would abort - # every build before any component is compiled. - printf '%s\n' "$@" | LC_ALL=C sort | while IFS= read -r file; do - blob="$(git hash-object "${file}")" || exit 1 - printf 'file=%s\n' "${file}" - printf 'blob=%s\n' "${blob}" - done -} - -# Identify only the source, patch, and explicit build-schema inputs selected for -# Arrow. Arrow and Paimon publish separate installed markers so a package-only -# build cannot certify a component that it did not rebuild. -arrow_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}" - printf 'ARROW_VERSION=%s\n' "${ARROW_VERSION}" - printf 'ARROW_NAME=%s\n' "${ARROW_NAME}" - printf 'ARROW_SOURCE=%s\n' "${ARROW_SOURCE}" - printf 'ARROW_MD5SUM=%s\n' "${ARROW_MD5SUM}" - printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}" - printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}" - printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}" - printf 'XSIMD_NAME=%s\n' "${XSIMD_NAME}" - printf 'XSIMD_SOURCE=%s\n' "${XSIMD_SOURCE}" - printf 'XSIMD_MD5SUM=%s\n' "${XSIMD_MD5SUM}" - arrow_paimon_fingerprint_files \ - patches/apache-arrow-"${ARROW_VERSION}"-*.patch - } | git hash-object --stdin - ) -} - -paimon_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - arrow_build_fingerprint - printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}" - printf 'PAIMON_CPP_NAME=%s\n' "${PAIMON_CPP_NAME}" - printf 'PAIMON_CPP_SOURCE=%s\n' "${PAIMON_CPP_SOURCE}" - printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}" - arrow_paimon_fingerprint_files \ - paimon-cpp-cache.cmake \ - patches/paimon-cpp-*.patch - } | git hash-object --stdin - ) -} - -# Source patch markers use a combined value because either component's inputs -# may change the external-Arrow contract applied to both source trees. -arrow_paimon_build_fingerprint() { - { - arrow_build_fingerprint - paimon_build_fingerprint - } | git hash-object --stdin -} - -arrow_paimon_fingerprint_matches() { - local installed_fingerprint="$1" - local expected_fingerprint="$2" - local legacy_compatible_expected_fingerprint="$3" - shift 3 - local compatible_fingerprint - - if [[ "${installed_fingerprint}" == "${expected_fingerprint}" ]]; then - return 0 - fi - if [[ "${expected_fingerprint}" != "${legacy_compatible_expected_fingerprint}" ]]; then - return 1 - fi - for compatible_fingerprint in "$@"; do - if [[ "${installed_fingerprint}" == "${compatible_fingerprint}" ]]; then - return 0 - fi - done - return 1 -} - -ARROW_REQUIRED_LIBRARIES=( - libbrotlicommon.a - libbrotlidec.a - libbrotlienc.a - libarrow.a - libarrow_compute.a - libarrow_flight.a - libarrow_flight_sql.a - libarrow_dataset.a - libarrow_acero.a - libarrow_bundled_dependencies.a - libparquet.a -) - -PAIMON_REQUIRED_LIBRARIES=( - libpaimon.a - libpaimon_parquet_file_format.a - libpaimon_orc_file_format.a - libpaimon_blob_file_format.a - libpaimon_local_file_system.a - libpaimon_file_index.a - libpaimon_global_index.a - libroaring_bitmap_paimon.a - libxxhash_paimon.a - libfmt_paimon.a - libtbb_paimon.a -) - -ARROW_PAIMON_REQUIRED_LIBRARIES=( - "${ARROW_REQUIRED_LIBRARIES[@]}" - "${PAIMON_REQUIRED_LIBRARIES[@]}" -) - -arrow_artifacts_valid() { - local install_dir="$1" - local installed_arrow_version - local library - - if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then - echo "Missing installed Arrow version header" >&2 - return 1 - fi - installed_arrow_version="$( - awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { - gsub(/"/, "", $3); print $3; exit - }' "${install_dir}/include/arrow/util/config.h" - )" - if [[ "${installed_arrow_version}" != "${ARROW_VERSION}" ]]; then - echo "Installed Arrow version ${installed_arrow_version} does not match ${ARROW_VERSION}" >&2 - return 1 - fi - - for library in "${ARROW_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Arrow library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -paimon_artifacts_valid() { - local install_dir="$1" - local library - - for library in "${PAIMON_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Paimon library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -arrow_prebuilt_valid() { - local install_dir="$1" - local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${arrow_fingerprint_mark}" ]]; then - echo "Missing Arrow build fingerprint: ${arrow_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(arrow_build_fingerprint)" - installed_fingerprint="$(<"${arrow_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${ARROW_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Arrow build fingerprint does not match selected inputs" >&2 - return 1 - fi - arrow_artifacts_valid "${install_dir}" -} - -paimon_prebuilt_valid() { - local install_dir="$1" - local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${paimon_fingerprint_mark}" ]]; then - echo "Missing Paimon build fingerprint: ${paimon_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(paimon_build_fingerprint)" - installed_fingerprint="$(<"${paimon_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${PAIMON_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Paimon build fingerprint does not match selected inputs" >&2 - return 1 - fi - paimon_artifacts_valid "${install_dir}" -} - -arrow_paimon_prebuilt_valid() { - local install_dir="$1" - arrow_prebuilt_valid "${install_dir}" && paimon_prebuilt_valid "${install_dir}" -} - -invalidate_arrow_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/arrow-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_arrow_prebuilt_marker() { - local install_dir="$1" - arrow_artifacts_valid "${install_dir}" - arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" -} - -invalidate_paimon_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/paimon-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_paimon_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid "${install_dir}" - paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" -} - -require_arrow_prebuilt_for_paimon() { - local install_dir="$1" - if ! arrow_prebuilt_valid "${install_dir}"; then - echo "Paimon requires Arrow to be built from the currently selected inputs first" >&2 - return 1 - fi -} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index b1d84b1fe78e0f..70dbdf8d8772dc 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -196,8 +196,7 @@ if [[ "${CLEAN}" -eq 1 ]] && [[ -d "${TP_SOURCE_DIR}" ]]; then fi # Download thirdparties. -prepare_arrow_paimon_download_packages "${packages[@]}" -bash "${TP_DIR}/download-thirdparty.sh" "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}" +bash "${TP_DIR}/download-thirdparty.sh" "${packages[@]}" export LD_LIBRARY_PATH="${TP_DIR}/installed/lib:${LD_LIBRARY_PATH}" @@ -1110,7 +1109,6 @@ build_grpc() { # arrow build_arrow() { check_if_source_exist "${ARROW_SOURCE}" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" mkdir -p release @@ -1197,8 +1195,6 @@ build_arrow() { strip_lib libparquet.a strip_lib libarrow_dataset.a strip_lib libarrow_acero.a - - publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } # arrow-adbc @@ -2305,94 +2301,6 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# paimon-cpp -build_paimon_cpp() { - check_if_source_exist "${PAIMON_CPP_SOURCE}" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" - - # Darwin doesn't build GNU libunwind in this script, so don't force -lunwind there. - local paimon_linker_flags="-L${TP_LIB_DIR} -lbrotlienc -lbrotlidec -lbrotlicommon -llzma" - if [[ "${KERNEL}" != 'Darwin' ]]; then - paimon_linker_flags="${paimon_linker_flags} -lunwind" - fi - - CXXFLAGS="-Wno-nontrivial-memcall" \ - "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ - -G "${GENERATOR}" \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ - -DPAIMON_BUILD_SHARED=OFF \ - -DPAIMON_BUILD_STATIC=ON \ - -DPAIMON_BUILD_TESTS=OFF \ - -DPAIMON_ENABLE_ORC=ON \ - -DPAIMON_ENABLE_AVRO=OFF \ - -DPAIMON_ENABLE_LANCE=OFF \ - -DPAIMON_ENABLE_JINDO=OFF \ - -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_LUCENE=OFF \ - -DCMAKE_EXE_LINKER_FLAGS="${paimon_linker_flags}" \ - -DCMAKE_SHARED_LINKER_FLAGS="${paimon_linker_flags}" \ - .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install - - # Install paimon-cpp internal dependencies with renamed versions - # These libraries are built but not installed by default - echo "Installing paimon-cpp internal dependencies..." - - # Arrow deps: When PAIMON_USE_EXTERNAL_ARROW=ON (Plan B), paimon-cpp - # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps - # directory is not needed. When building its own Arrow (legacy), copy - # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" - if [ -d "arrow_ep-install/lib" ]; then - mkdir -p "${paimon_deps_dir}" - for paimon_arrow_dep in \ - libarrow.a \ - libarrow_compute.a \ - libarrow_filesystem.a \ - libarrow_dataset.a \ - libarrow_acero.a \ - libparquet.a; do - if [ -f "arrow_ep-install/lib/${paimon_arrow_dep}" ]; then - cp -v "arrow_ep-install/lib/${paimon_arrow_dep}" "${paimon_deps_dir}/${paimon_arrow_dep}" - fi - done - else - echo " arrow_ep-install not found (PAIMON_USE_EXTERNAL_ARROW=ON?) – skipping paimon_deps Arrow copy" - fi - - # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap - if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" - fi - - # Install xxhash, renamed to avoid conflict with Doris's xxhash - if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" - fi - - # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) - if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" - fi - - # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) - if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" - fi - - echo "Paimon-cpp internal dependencies installed successfully" - publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" -} - # lance-c build_lance_c() { check_if_source_exist "${LANCE_C_SOURCE}" @@ -2532,7 +2440,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then icu mecab_ipadic pugixml - paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" @@ -2640,7 +2547,6 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; - paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; lzma) src_var="LZMA_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 95c67e2cdad209..6b8d90c726f431 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -85,6 +85,15 @@ while [[ $# -gt 0 ]]; do ) done if [[ "${SPEC_LIB}" != "" ]]; then + # Arrow builds xsimd and Brotli from their source archives. + if [[ " ${SPEC_ARCHIVES[*]} " == *' ARROW '* ]]; then + for arrow_companion in XSIMD BROTLI; do + if [[ " ${SPEC_ARCHIVES[*]} " != *" ${arrow_companion} "* ]]; then + SPEC_ARCHIVES+=("${arrow_companion}") + fi + done + fi + # ARROW_ADBC_FLIGHTSQL is a companion archive of arrow_adbc rather than a # package of its own: it has no build function, build_arrow_adbc() only copies # the prebuilt driver out of it. Its name therefore never appears on a command @@ -367,20 +376,6 @@ echo "===== Patching thirdparty archives..." # This is to avoid duplicated patch. ################################################################################### PATCHED_MARK="patched_mark" -ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" -ARROW_PAIMON_BUILD_FINGERPRINT="" -if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || - " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" -fi - -reset_arrow_paimon_source() { - local archive_name="$1" - local source_name="$2" - echo "Resetting ${source_name} because its patch state is incomplete or stale" - rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" -} # glog patch if [[ " ${TP_ARCHIVES[*]} " =~ " GLOG " ]]; then @@ -486,16 +481,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then cd - fi if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-24.0.0" ]]; then - arrow_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_SOURCE}/${PATCHED_MARK}" && - -f "${arrow_fingerprint_mark}" ]] || - [[ "$(<"${arrow_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${ARROW_NAME}" "${ARROW_SOURCE}" - cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" - # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size, - # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-paimon.patch" - + cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then # Introducing the parameter that forces writing INT96 timestamps for # compatibility with the Doris Parquet writer. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-force-write-int96-timestamps.patch" @@ -503,10 +490,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then # Add Parquet LZO page decompression support used by file scanner v2. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-lzo.patch" touch "${PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - fi + cd - fi echo "Finished patching ${ARROW_SOURCE}" fi @@ -778,31 +763,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# patch paimon-cpp -if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" - PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK="patched_mark_arrow_24_compute" - paimon_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" && - -f "${paimon_fingerprint_mark}" ]] || - [[ "$(<"${paimon_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${PAIMON_CPP_NAME}" "${PAIMON_CPP_SOURCE}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" - touch "${PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - - fi - echo "Finished patching ${PAIMON_CPP_SOURCE}" -fi - # Apply Doris lance-c patches. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.8" ]]; then diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake deleted file mode 100644 index 6ab6049c59b0d5..00000000000000 --- a/thirdparty/paimon-cpp-cache.cmake +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# CMake Initial Cache for paimon-cpp -# Configures paimon-cpp to reuse selected Doris third-party libraries -# Usage: cmake -C paimon-cpp-cache.cmake ... - -# Get the Doris thirdparty installation directory from environment -set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") - -if(NOT DORIS_THIRDPARTY_DIR) - message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") -endif() - -message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") - -# Set CMAKE_PREFIX_PATH to help find_package locate our libraries -set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") - -# Library and include paths -set(DORIS_LIB_DIR "${DORIS_THIRDPARTY_DIR}/lib" CACHE PATH "Doris library directory") -set(DORIS_INCLUDE_DIR "${DORIS_THIRDPARTY_DIR}/include" CACHE PATH "Doris include directory") - -# ============================================================================ -# ZLIB - Reuse from Doris (version 1.3.1) -# ============================================================================ -set(ZLIB_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZLIB root directory") -set(ZLIB_LIBRARY "${DORIS_LIB_DIR}/libz.a" CACHE FILEPATH "ZLIB library") -set(ZLIB_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZLIB include directory") - -# ============================================================================ -# ZSTD - Reuse from Doris (version 1.5.7) -# ============================================================================ -set(ZSTD_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZSTD root directory") -set(ZSTD_LIBRARY "${DORIS_LIB_DIR}/libzstd.a" CACHE FILEPATH "ZSTD library") -set(ZSTD_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZSTD include directory") - -# ============================================================================ -# LZ4 - Reuse from Doris (version 1.9.4) -# ============================================================================ -set(LZ4_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "LZ4 root directory") -set(LZ4_LIBRARY "${DORIS_LIB_DIR}/liblz4.a" CACHE FILEPATH "LZ4 library") -set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") - -# ============================================================================ -# glog - NOT reused from Doris -# paimon-cpp's build_glog() unconditionally calls externalproject_add() to -# build glog 0.7.1. Any GLOG_ROOT/GLOG_LIBRARY/GLOG_INCLUDE_DIR set here -# would be overwritten by that macro, so we skip them entirely. -# ============================================================================ - -# ============================================================================ -# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp -# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). -# ============================================================================ -set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") - -set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") - -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") - -# Protobuf, Thrift - still built separately by paimon-cpp - -# ============================================================================ -# Snappy - Reuse from Doris -# ============================================================================ -set(Snappy_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory") -set(SNAPPY_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory (legacy)") -set(SNAPPY_LIBRARY "${DORIS_LIB_DIR}/libsnappy.a" CACHE FILEPATH "Snappy library") -set(SNAPPY_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Snappy include directory") - -# ============================================================================ -# Build configuration -# ============================================================================ -set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build with -fPIC") -set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type") - -# Symbol visibility control to prevent conflicts with Doris -# paimon-cpp builds Arrow/ORC/etc with hidden symbols to avoid conflicts -set(CMAKE_CXX_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C++ symbols by default") -set(CMAKE_C_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C symbols by default") -set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inline function symbols") - -# Verify that required libraries exist -if(NOT EXISTS "${ZLIB_LIBRARY}") - message(FATAL_ERROR "ZLIB library not found: ${ZLIB_LIBRARY}") -endif() -if(NOT EXISTS "${ZSTD_LIBRARY}") - message(FATAL_ERROR "ZSTD library not found: ${ZSTD_LIBRARY}") -endif() -if(NOT EXISTS "${LZ4_LIBRARY}") - message(FATAL_ERROR "LZ4 library not found: ${LZ4_LIBRARY}") -endif() -if(NOT EXISTS "${SNAPPY_LIBRARY}") - message(FATAL_ERROR "Snappy library not found: ${SNAPPY_LIBRARY}") -endif() - -message(STATUS "========================================") -message(STATUS "Paimon-cpp Library Reuse Configuration") -message(STATUS "========================================") -message(STATUS "Reusing from Doris:") -message(STATUS " ✓ ZLIB, ZSTD, LZ4, Snappy") -if(PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " ✓ Arrow, Parquet, Arrow Dataset, Arrow Acero (Plan B)") -else() - message(STATUS " ✗ Arrow (building separately, symbol visibility=hidden)") -endif() -message(STATUS "") -message(STATUS "Building separately:") -if(NOT PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " - Arrow, Protobuf, Thrift, ORC") -else() - message(STATUS " - Protobuf, Thrift, ORC") -endif() -message(STATUS " - glog, RapidJSON, TBB") -message(STATUS "========================================") diff --git a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch index 1847a50694f014..85ec1f8021cd9f 100644 --- a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch @@ -1,11 +1,10 @@ --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,8 +215,9 @@ +@@ -215,7 +215,9 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && -- target_unit == ::arrow::TimeUnit::NANO) { +- if (arrow_properties.support_deprecated_int96_timestamps()) { + if (arrow_properties.force_write_int96_timestamps() || + (arrow_properties.support_deprecated_int96_timestamps() && + target_unit == ::arrow::TimeUnit::NANO)) { diff --git a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch deleted file mode 100644 index 583f85b4400bbf..00000000000000 --- a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch +++ /dev/null @@ -1,226 +0,0 @@ ---- a/cpp/src/parquet/arrow/schema.cc -+++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,7 +215,8 @@ - - // The user is explicitly asking for Impala int96 encoding, there is no - // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && -+ target_unit == ::arrow::TimeUnit::NANO) { - *physical_type = ParquetType::INT96; - return Status::OK(); - } ---- a/cpp/src/parquet/arrow/reader.cc -+++ b/cpp/src/parquet/arrow/reader.cc -@@ -1041,25 +1041,32 @@ - ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); - } - -- int64_t num_rows = 0; -+ std::vector num_rows; - for (int row_group : row_groups) { -- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); -+ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); - } - - using ::arrow::RecordBatchIterator; -+ size_t row_group_idx = 0; - - // NB: This lambda will be invoked outside the scope of this call to - // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. - // `this` is a non-owning pointer so we are relying on the parent FileReader outliving - // this RecordBatchReader. - ::arrow::Iterator batches = ::arrow::MakeFunctionIterator( -- [readers, batch_schema, num_rows, -+ [readers, batch_schema, num_rows, row_group_idx, - this]() mutable -> ::arrow::Result { - ::arrow::ChunkedArrayVector columns(readers.size()); - - // don't reserve more rows than necessary -- int64_t batch_size = std::min(properties().batch_size(), num_rows); -- num_rows -= batch_size; -+ int64_t batch_size = 0; -+ if (!num_rows.empty()) { -+ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); -+ num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && row_group_idx + 1 < num_rows.size()) { -+ ++row_group_idx; -+ } -+ } - - RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( - reader_properties_.use_threads(), static_cast(readers.size()), ---- a/cpp/src/parquet/arrow/writer.cc -+++ b/cpp/src/parquet/arrow/writer.cc -@@ -341,6 +341,14 @@ - return Status::OK(); - } - -+ int64_t GetBufferedSize() override { -+ if (row_group_writer_ == nullptr) { -+ return 0; -+ } -+ return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); -+ } -+ - Status Close() override { - if (!closed_) { - // Make idempotent -@@ -444,10 +452,14 @@ - - // Max number of rows allowed in a row group. - const int64_t max_row_group_length = this->properties().max_row_group_length(); -+ const int64_t max_row_group_size = this->properties().max_row_group_size(); - - // Initialize a new buffered row group writer if necessary. - if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || -- row_group_writer_->num_rows() >= max_row_group_length) { -+ row_group_writer_->num_rows() >= max_row_group_length || -+ row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= -+ max_row_group_size) { - RETURN_NOT_OK(NewBufferedRowGroup()); - } - ---- a/cpp/src/parquet/arrow/writer.h -+++ b/cpp/src/parquet/arrow/writer.h -@@ -124,6 +124,9 @@ - /// option in this case. - virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; - -+ /// \brief Return the buffered size in bytes. -+ virtual int64_t GetBufferedSize() = 0; -+ - /// \brief Write the footer and close the file. - virtual ::arrow::Status Close() = 0; - virtual ~FileWriter(); ---- a/cpp/src/parquet/properties.h -+++ b/cpp/src/parquet/properties.h -@@ -160,6 +160,7 @@ - static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; - static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; - static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; -+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; - static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; - static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; - static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -343,6 +344,7 @@ - dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), - write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), - max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), -+ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), - pagesize_(kDefaultDataPageSize), - max_rows_per_page_(kDefaultMaxRowsPerPage), - version_(ParquetVersion::PARQUET_2_6), -@@ -359,6 +361,7 @@ - dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), - write_batch_size_(properties.write_batch_size()), - max_row_group_length_(properties.max_row_group_length()), -+ max_row_group_size_(properties.max_row_group_size()), - pagesize_(properties.data_pagesize()), - max_rows_per_page_(properties.max_rows_per_page()), - version_(properties.version()), -@@ -468,6 +471,13 @@ - return this; - } - -+ /// Specify the max bytes size to put in a single row group. -+ /// Default 128MB. -+ Builder* max_row_group_size(int64_t max_row_group_size) { -+ max_row_group_size_ = max_row_group_size; -+ return this; -+ } -+ - /// Specify the data page size. - /// Default 1MB. - Builder* data_pagesize(int64_t pg_size) { -@@ -867,11 +877,12 @@ - - return std::shared_ptr(new WriterProperties( - pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, -- size_statistics_level_, std::move(file_encryption_properties_), -- default_column_properties_, column_properties, data_page_version_, -- store_decimal_as_integer_, std::move(sorting_columns_), -- content_defined_chunking_enabled_, content_defined_chunking_options_)); -+ max_row_group_size_, pagesize_, max_rows_per_page_, version_, created_by_, -+ page_checksum_enabled_, size_statistics_level_, -+ std::move(file_encryption_properties_), default_column_properties_, -+ column_properties, data_page_version_, store_decimal_as_integer_, -+ std::move(sorting_columns_), content_defined_chunking_enabled_, -+ content_defined_chunking_options_)); - } - - private: -@@ -881,6 +892,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetVersion::type version_; -@@ -917,6 +929,8 @@ - - inline int64_t max_row_group_length() const { return max_row_group_length_; } - -+ inline int64_t max_row_group_size() const { return max_row_group_size_; } -+ - inline int64_t data_pagesize() const { return pagesize_; } - - inline int64_t max_rows_per_page() const { return max_rows_per_page_; } -@@ -1046,9 +1060,10 @@ - private: - explicit WriterProperties( - MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, -- ParquetVersion::type version, const std::string& created_by, -- bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, -+ int64_t max_rows_per_page, ParquetVersion::type version, -+ const std::string& created_by, bool page_write_checksum_enabled, -+ SizeStatisticsLevel size_statistics_level, - std::shared_ptr file_encryption_properties, - const ColumnProperties& default_column_properties, - const std::unordered_map& column_properties, -@@ -1059,6 +1074,7 @@ - dictionary_pagesize_limit_(dictionary_pagesize_limit), - write_batch_size_(write_batch_size), - max_row_group_length_(max_row_group_length), -+ max_row_group_size_(max_row_group_size), - pagesize_(pagesize), - max_rows_per_page_(max_rows_per_page), - parquet_data_page_version_(data_page_version), -@@ -1078,6 +1094,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetDataPageVersion parquet_data_page_version_; ---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1884,7 +1884,20 @@ - REQUIRED_VERSION - 0.11.0) - -- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -+ if(NOT Thrift_VERSION) -+ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -+ set(Thrift_VERSION "${thrift_PC_VERSION}") -+ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -+ set(Thrift_VERSION "${ThriftAlt_VERSION}") -+ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -+ set(Thrift_VERSION "${THRIFT_VERSION}") -+ endif() -+ endif() -+ if(NOT Thrift_VERSION) -+ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -+ endif() -+ -+ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") - list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) - list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) - list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch deleted file mode 100644 index d0e723a34d98ef..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/include/paimon/status.h b/include/paimon/status.h ---- a/include/paimon/status.h -+++ b/include/paimon/status.h -@@ -458,9 +458,4 @@ - } \ - } while (false) - --// This is an internal-use macro and should not be used in public headers. --#ifndef RETURN_NOT_OK --#define RETURN_NOT_OK(s) PAIMON_RETURN_NOT_OK(s) --#endif -- - } // namespace paimon -diff --git a/src/paimon/format/parquet/parquet_input_stream_impl.cpp b/src/paimon/format/parquet/parquet_input_stream_impl.cpp ---- a/src/paimon/format/parquet/parquet_input_stream_impl.cpp -+++ b/src/paimon/format/parquet/parquet_input_stream_impl.cpp -@@ -59,7 +59,7 @@ arrow::Result> ParquetInputStreamImpl::Read(int64 - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, Read(nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -@@ -78,7 +78,7 @@ arrow::Result> ParquetInputStreamImpl::ReadAt(int - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, ReadAt(position, nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp ---- a/src/paimon/format/parquet/file_reader_wrapper.cpp -+++ b/src/paimon/format/parquet/file_reader_wrapper.cpp -@@ -81,8 +81,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); - } -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- target_row_group_indices, target_column_indices_, &batch_reader_)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader_, file_reader_->GetRecordBatchReader(target_row_group_indices, -+ target_column_indices_)); - return Status::OK(); - } - } -@@ -152,9 +153,11 @@ Status FileReaderWrapper::PrepareForReading( - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - std::unique_ptr batch_reader; -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -- column_indices, &batch_reader)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader, -+ file_reader_->GetRecordBatchReader( -+ std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -+ column_indices)); - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch b/thirdparty/patches/paimon-cpp-arrow-24-compute.patch deleted file mode 100644 index 1b581b8f93ee92..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -38,6 +38,8 @@ - set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH - "Include directory for external Arrow/Parquet headers") - set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "" CACHE FILEPATH -+ "Path to external libarrow_compute.a") - set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") - set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") - set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -@@ -1008,6 +1010,7 @@ - - foreach(_paimon_external_lib - IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_COMPUTE_LIB - PAIMON_EXTERNAL_ARROW_DATASET_LIB - PAIMON_EXTERNAL_ARROW_ACERO_LIB - PAIMON_EXTERNAL_PARQUET_LIB -@@ -1028,6 +1031,13 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -+ add_library(arrow_compute STATIC IMPORTED) -+ set_target_properties(arrow_compute -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_COMPUTE_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ - add_library(arrow_dataset STATIC IMPORTED) - set_target_properties(arrow_dataset - PROPERTIES IMPORTED_LOCATION -@@ -1055,7 +1065,9 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -- target_link_libraries(arrow_acero INTERFACE arrow) -+ target_link_libraries(arrow_compute INTERFACE arrow) -+ -+ target_link_libraries(arrow_acero INTERFACE arrow_compute) - - target_link_libraries(arrow_dataset INTERFACE arrow_acero) diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch deleted file mode 100644 index 213cc279f52e5c..00000000000000 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ /dev/null @@ -1,362 +0,0 @@ -diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake -index 74654a4..4065297 100644 ---- a/cmake_modules/BuildUtils.cmake -+++ b/cmake_modules/BuildUtils.cmake -@@ -55,12 +55,18 @@ function(add_paimon_lib LIB_NAME) - # Necessary to make static linking into other shared libraries work properly - set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) - if(ARG_DEPENDENCIES) -- # Avoid add_dependencies on non-existent targets (e.g. when building static only). -+ # In static-only builds, some dependency names are still declared as -+ # *_shared. Map them to *_static when the shared target is unavailable. - set(_paimon_objlib_deps) - foreach(_paimon_dep IN LISTS ARG_DEPENDENCIES) -- if(TARGET ${_paimon_dep}) -- list(APPEND _paimon_objlib_deps ${_paimon_dep}) -+ set(_paimon_mapped_dep "${_paimon_dep}") -+ if(NOT TARGET ${_paimon_mapped_dep} AND _paimon_dep MATCHES "_shared$") -+ string(REGEX REPLACE "_shared$" "_static" _paimon_mapped_dep "${_paimon_dep}") - endif() -+ if(TARGET ${_paimon_mapped_dep}) -+ list(APPEND _paimon_objlib_deps ${_paimon_mapped_dep}) -+ endif() -+ unset(_paimon_mapped_dep) - endforeach() - if(_paimon_objlib_deps) - add_dependencies(${LIB_NAME}_objlib ${_paimon_objlib_deps}) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 57f6c50..d8e3cd0 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -38,6 +38,8 @@ string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) - --set(CMAKE_CXX_STANDARD 17) -+if(NOT DEFINED CMAKE_CXX_STANDARD) -+ set(CMAKE_CXX_STANDARD 17) -+endif() - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -909,9 +909,6 @@ macro(build_orc) - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" -- "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" - "-DLZ4_HOME=${ORC_LZ4_ROOT}" - "-DZSTD_HOME=${ORC_ZSTD_ROOT}" -@@ -923,6 +920,13 @@ macro(build_orc) - -DBUILD_TOOLS=OFF - -DBUILD_CPP_ENABLE_METRICS=ON) - -+ if(ORC_RPATH) -+ list(APPEND ORC_CMAKE_ARGS -+ "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}") -+ endif() -+ - set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/orc.diff") - externalproject_add(orc_ep - URL ${ORC_SOURCE_URL} - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1024,6 +1024,14 @@ macro(build_arrow) - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" - -DARROW_DEPENDENCY_USE_SHARED=OFF -+ # Avoid forcing CONDA dependency mode when CONDA_PREFIX is present. -+ # AUTO keeps the normal "find system first, fallback to bundled" -+ # behavior and prevents accidental pickup of conda's thrift/zstd. -+ -DARROW_DEPENDENCY_SOURCE=AUTO -+ # Isolate from user/system CMake package registries to improve -+ # reproducibility in CI and local mixed environments. -+ -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF -+ -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF - -DARROW_BUILD_SHARED=OFF - -DARROW_BUILD_STATIC=ON - -DARROW_BUILD_TESTS=OFF - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -761,6 +761,9 @@ macro(build_protobuf) - get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}" - DIRECTORY) -+ set(THIRDPARTY_ZLIB_STATIC_LIB -+ "${THIRDPARTY_ZLIB_ROOT}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" -+ ) - - # Strip lto flags (which may be added by dh_auto_configure) - # See https://github.com/protocolbuffers/protobuf/issues/7092 -@@ -778,6 +781,10 @@ macro(build_protobuf) - "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}" - "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}" -+ "-DZLIB_INCLUDE_DIR=${THIRDPARTY_ZLIB_INCLUDE_DIR}" -+ "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_DEBUG=${THIRDPARTY_ZLIB_STATIC_LIB}" - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) - set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS}) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -34,6 +34,16 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") - -+option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF) -+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH -+ "Include directory for external Arrow/Parquet headers") -+set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") -+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") -+set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH -+ "Path to external libarrow_bundled_dependencies.a") -+ - macro(set_urls URLS) - set(${URLS} ${ARGN}) - endmacro() - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -961,5 +961,95 @@ macro(build_orc) - endmacro() - - macro(build_arrow) -- message(STATUS "Building Arrow from source") -+ if(PAIMON_USE_EXTERNAL_ARROW) -+ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include") -+ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") -+ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow" -+ "${ARROW_INCLUDE_DIR}/arrow") -+ endif() -+ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ "${ARROW_INCLUDE_DIR}/parquet") -+ endif() -+ -+ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR) -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON" -+ ) -+ endif() -+ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}") -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}" -+ ) -+ endif() -+ -+ foreach(_paimon_external_lib -+ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_DATASET_LIB -+ PAIMON_EXTERNAL_ARROW_ACERO_LIB -+ PAIMON_EXTERNAL_PARQUET_LIB -+ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB) -+ if(NOT ${_paimon_external_lib}) -+ message(FATAL_ERROR -+ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON") -+ endif() -+ if(NOT EXISTS "${${_paimon_external_lib}}") -+ message(FATAL_ERROR -+ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}") -+ endif() -+ endforeach() -+ -+ add_library(arrow STATIC IMPORTED) -+ set_target_properties(arrow -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_dataset STATIC IMPORTED) -+ set_target_properties(arrow_dataset -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_acero STATIC IMPORTED) -+ set_target_properties(arrow_acero -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(parquet STATIC IMPORTED) -+ set_target_properties(parquet -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_bundled_dependencies STATIC IMPORTED) -+ set_target_properties(arrow_bundled_dependencies -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ target_link_libraries(arrow_acero INTERFACE arrow) -+ -+ target_link_libraries(arrow_dataset INTERFACE arrow_acero) -+ -+ target_link_libraries(arrow -+ INTERFACE zstd -+ snappy -+ lz4 -+ zlib -+ arrow_bundled_dependencies) -+ -+ target_link_libraries(parquet -+ INTERFACE zstd snappy lz4 zlib arrow_bundled_dependencies -+ arrow_dataset) -+ else() -+ message(STATUS "Building Arrow from source") - - get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1121,6 +1121,7 @@ macro(build_arrow) - zlib - arrow_bundled_dependencies - arrow_dataset) -+ endif() - - endmacro(build_arrow) - -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -326,10 +326,10 @@ if(PAIMON_ENABLE_LUMINA) - include_directories("${CMAKE_SOURCE_DIR}/third_party/lumina/include") - endif() - -+include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) - include_directories(SYSTEM ${TBB_INCLUDE_DIR}) - --include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - add_compile_definitions("GLOG_USE_GLOG_EXPORT") - - set(THREADS_PREFER_PTHREAD_FLAG ON) - -diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp ---- a/src/paimon/common/logging/logging.cpp -+++ b/src/paimon/common/logging/logging.cpp -@@ -83,7 +83,7 @@ std::unique_ptr Logger::GetLogger(const std::string& path) { - } - std::unique_lock ulock(getRegistryLock()); - if (!google::IsGoogleLoggingInitialized()) { -- google::InitGoogleLogging(program_invocation_name); -+ google::InitGoogleLogging("paimon-cpp"); - } - return std::make_unique(); - } - -diff --git a/src/paimon/common/memory/memory_pool.cpp b/src/paimon/common/memory/memory_pool.cpp ---- a/src/paimon/common/memory/memory_pool.cpp -+++ b/src/paimon/common/memory/memory_pool.cpp -@@ -55,7 +55,7 @@ void* MemoryPoolImpl::Malloc(uint64_t size, uint64_t alignment) { - return memptr; - } - --void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, size_t alignment) { -+void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment) { - if (alignment == 0) { - void* memptr = ::realloc(p, new_size); - total_allocated_size.fetch_add(new_size - old_size); - -diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp ---- a/src/paimon/format/blob/blob_format_writer.cpp -+++ b/src/paimon/format/blob/blob_format_writer.cpp -@@ -138,7 +138,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - uint64_t total_read_length = 0; -- uint32_t read_len = std::min(file_length, tmp_buffer_->size()); -+ uint32_t read_len = -+ static_cast(std::min(file_length, tmp_buffer_->size())); - while (read_len > 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); - if (static_cast(actual_read_len) != read_len) { -@@ -149,7 +150,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); - total_read_length += actual_read_len; -- read_len = std::min(file_length - total_read_length, tmp_buffer_->size()); -+ read_len = static_cast( -+ std::min(file_length - total_read_length, tmp_buffer_->size())); - } - - // write bin length - ---- a/cmake_modules/arrow.diff -+++ b/cmake_modules/arrow.diff -@@ -196,3 +196,29 @@ - int64_t pagesize_; - ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+index 9df922afa2..5c8b3d4d07 100644 -+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) -+ REQUIRED_VERSION -+ 0.11.0) -+ -+- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -++ if(NOT Thrift_VERSION) -++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -++ set(Thrift_VERSION "${thrift_PC_VERSION}") -++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -++ set(Thrift_VERSION "${ThriftAlt_VERSION}") -++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -++ set(Thrift_VERSION "${THRIFT_VERSION}") -++ endif() -++ endif() -++ if(NOT Thrift_VERSION) -++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -++ endif() -++ -++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") -+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) -+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) -+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) - -diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt -index 76dd03b..9317324 100644 ---- a/src/paimon/CMakeLists.txt -+++ b/src/paimon/CMakeLists.txt -@@ -281,6 +281,11 @@ add_paimon_lib(paimon - ${PAIMON_COMMON_SRCS} - ${PAIMON_CORE_SRCS} - DEPENDENCIES -+ # Codec headers come from ExternalProject install steps, so object -+ # compilation must not race those steps in clean parallel builds. -+ zstd -+ snappy -+ lz4 - arrow - tbb - glog diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh deleted file mode 100755 index f946be5c7631d3..00000000000000 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -create_fingerprint_fixture() { - local destination="$1" - - mkdir -p "${destination}/patches" - cp "${ROOT}/arrow-paimon-vars.sh" "${destination}/arrow-paimon-vars.sh" - cp "${ROOT}/paimon-cpp-cache.cmake" "${destination}/paimon-cpp-cache.cmake" - cp "${ROOT}"/patches/apache-arrow-24.0.0-*.patch "${destination}/patches/" - cp "${ROOT}"/patches/paimon-cpp-*.patch "${destination}/patches/" -} - -fingerprint_from_fixture() { - local fixture="$1" - local component="$2" - local result_variable="$3" - local fingerprint - - fingerprint="$( - set -e - # shellcheck source=/dev/null - . "${fixture}/arrow-paimon-vars.sh" - "${component}_build_fingerprint" - )" - printf -v "${result_variable}" '%s' "${fingerprint}" -} - -exercise_semantic_fingerprints() { - local first_fixture="${tmpdir}/fingerprint-first" - local second_fixture="${tmpdir}/fingerprint-second" - local first_arrow - local first_paimon - local second_arrow - local second_paimon - local changed_arrow - local changed_paimon - - create_fingerprint_fixture "${first_fixture}" - create_fingerprint_fixture "${second_fixture}" - - printf '%s\n' first-env >"${first_fixture}/env.sh" - printf '%s\n' second-env >"${second_fixture}/env.sh" - printf '%s\n' first-vars >"${first_fixture}/vars.sh" - printf '%s\n' second-vars >"${second_fixture}/vars.sh" - printf '%s\n' first-download >"${first_fixture}/download-thirdparty.sh" - printf '%s\n' second-download >"${second_fixture}/download-thirdparty.sh" - printf '%s\n' first-build >"${first_fixture}/build-thirdparty.sh" - printf '%s\n' second-build >"${second_fixture}/build-thirdparty.sh" - printf '%s\n' '# release-branch-only comment' >>"${second_fixture}/arrow-paimon-vars.sh" - - fingerprint_from_fixture "${first_fixture}" arrow first_arrow - fingerprint_from_fixture "${first_fixture}" paimon first_paimon - [[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Arrow legacy marker migration target is stale" - [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Paimon legacy marker migration target is stale" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon second_paimon - [[ "${first_arrow}" == "${second_arrow}" ]] || - fail "unrelated branch scripts changed the Arrow fingerprint" - [[ "${first_paimon}" == "${second_paimon}" ]] || - fail "unrelated branch scripts changed the Paimon fingerprint" - - printf '%s\n' semantic-change \ - >>"${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - fingerprint_from_fixture "${second_fixture}" arrow changed_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow patch change did not change the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "an Arrow patch change did not change the Paimon fingerprint" - - cp "${first_fixture}/patches/apache-arrow-24.0.0-lzo.patch" \ - "${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - printf '%s\n' semantic-change >>"${second_fixture}/paimon-cpp-cache.cmake" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${second_arrow}" == "${first_arrow}" ]] || - fail "a Paimon-only cache change changed the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "a Paimon cache change did not change the Paimon fingerprint" - - changed_arrow="$( - set -e - # shellcheck source=/dev/null - . "${first_fixture}/arrow-paimon-vars.sh" - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_build_fingerprint - )" - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow build-schema change did not change its fingerprint" -} - -# shellcheck source=../arrow-paimon-vars.sh -. "${ROOT}/arrow-paimon-vars.sh" -exercise_semantic_fingerprints - -harness="${tmpdir}/harness" -mkdir -p "${harness}/src" "${harness}/patches" "${harness}/installed" -cp "${ROOT}/download-thirdparty.sh" "${harness}/download-thirdparty.sh" - -create_patch() { - local patch_file="$1" - local source_file="$2" - { - printf '%s\n' "--- a/${source_file}" - printf '%s\n' "+++ b/${source_file}" - printf '%s\n' '@@ -1 +1 @@' - printf '%s\n' '-original' - printf '%s\n' '+patched' - } >"${patch_file}" -} - -create_archive() { - local source_name="$1" - local archive_name="$2" - local prefix="$3" - local index - - mkdir -p "${harness}/src/${source_name}" - for index in 1 2 3; do - printf '%s\n' original >"${harness}/src/${source_name}/${prefix}-${index}.txt" - done - tar -czf "${harness}/src/${archive_name}" -C "${harness}/src" "${source_name}" - rm -rf "${harness}/src/${source_name}" -} - -arrow_source="arrow-apache-arrow-24.0.0" -arrow_archive="apache-arrow-24.0.0.tar.gz" -paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" -paimon_archive="paimon-cpp-0a4f4e2.tar.gz" - -create_archive "${arrow_source}" "${arrow_archive}" arrow -create_archive "${paimon_source}" "${paimon_archive}" paimon - -arrow_patches=( - apache-arrow-24.0.0-paimon.patch - apache-arrow-24.0.0-force-write-int96-timestamps.patch - apache-arrow-24.0.0-lzo.patch -) -paimon_patches=( - paimon-cpp-buildutils-static-deps.patch - paimon-cpp-arrow-24-compatibility.patch - paimon-cpp-arrow-24-compute.patch -) - -for index in 0 1 2; do - create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" - create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" -done - -arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" -paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" -{ - printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" - printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" - printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' - printf 'ARROW_NAME="%s"\n' "${arrow_archive}" - printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" - printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" - printf '%s\n' 'ARROW_DOWNLOAD="unused"' - printf 'PAIMON_CPP_NAME="%s"\n' "${paimon_archive}" - printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" - printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" - printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' - printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' -} >"${harness}/vars.sh" - -exercise_interrupted_patch_set() { - local package="$1" - local source_name="$2" - local archive_name="$3" - local prefix="$4" - shift 4 - local patches=("$@") - local applied - local index - - for applied in 1 2 3; do - rm -rf "${harness}/src/${source_name}" - tar -xzf "${harness}/src/${archive_name}" -C "${harness}/src" - for ((index = 0; index < applied; ++index)); do - ( - cd "${harness}/src/${source_name}" - patch -s -p1 <"${harness}/patches/${patches[${index}]}" - ) - done - - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - - for index in 1 2 3; do - [[ "$(<"${harness}/src/${source_name}/${prefix}-${index}.txt")" == "patched" ]] || - fail "${package} did not recover after interruption boundary ${applied}" - done - [[ "$(<"${harness}/src/${source_name}/patched_mark_arrow_paimon_fingerprint")" == "test-fingerprint" ]] || - fail "${package} fingerprint marker is missing" - - touch "${harness}/src/${source_name}/idempotence-sentinel" - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - [[ -f "${harness}/src/${source_name}/idempotence-sentinel" ]] || - fail "${package} reset a completely patched source tree" - done -} - -exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow \ - "${arrow_patches[@]}" -exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ - "${paimon_patches[@]}" - -exercise_generic_recovery_dispatch() { - local generic="${tmpdir}/generic-recovery" - local thirdparty_dir="${generic}/thirdparty" - local external_thirdparty_dir="${generic}/external-thirdparty" - local args_file="${generic}/build-args.txt" - local output_file="${generic}/build-output.txt" - local external_builder_called="${generic}/external-builder-called" - local fake_mvn="${generic}/fake-mvn" - local non_native_status=79 - local non_native_target - local status - local flag - local parallel - local clean - local package1 - local package2 - local extra - - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ - "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ - "${generic}/be/output" - cp "${ROOT}/../build.sh" "${generic}/build.sh" - cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" - printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" - # shellcheck disable=SC2016 - printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" - chmod +x "${fake_mvn}" - { - printf '%s\n' '#!/usr/bin/env bash' - printf 'exit %q\n' "${non_native_status}" - } >"${generic}/generated-source.sh" - chmod +x "${generic}/generated-source.sh" - { - printf '%s\n' 'DORIS_BUILD_PROFILE=0' - printf '%s\n' 'TARGET_SYSTEM=Linux' - printf 'MVN_CMD=%q\n' "${fake_mvn}" - } >"${generic}/env.sh" - { - printf '%s\n' '#!/usr/bin/env bash' - # RECOVERY_ARGS_FILE is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'printf "%s\n" "$*" >"${RECOVERY_ARGS_FILE:?}"' - # shellcheck disable=SC2016 - printf '%s\n' 'exit "${RECOVERY_EXIT_STATUS:-73}"' - } >"${thirdparty_dir}/build-thirdparty.sh" - - DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --clean >"${output_file}" 2>&1 || - fail "bare clean failed before reaching its clean-only exit" - [[ ! -e "${args_file}" ]] || fail "bare clean invoked the thirdparty builder" - [[ ! -d "${generic}/be/build_Release" && ! -d "${generic}/be/output" ]] || - fail "bare clean did not remove BE build outputs" - - for non_native_target in --fe --hive-udf; do - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" "${non_native_target}" >"${output_file}" 2>&1; then - fail "${non_native_target} did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "${non_native_target} failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "${non_native_target} invoked the native Arrow/Paimon builder" - done - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then - fail "--fe --clean did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "--fe --clean failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "--fe --clean invoked the native Arrow/Paimon builder" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic stale-prebuilt recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be --clean >"${output_file}" 2>&1; then - fail "generic clean recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic clean recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - RECOVERY_EXIT_STATUS=0 bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted artifacts that failed current-checkout validation" - fi - grep -Fq "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs" \ - "${output_file}" || fail "generic recovery did not validate artifacts after its builder returned" - - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an invalid install-only thirdparty prefix" - fi - grep -Fq "is an install-only or incomplete prefix" "${output_file}" || - fail "generic recovery did not explain how to refresh an install-only prebuilt" - - { - printf '%s\n' '#!/usr/bin/env bash' - # EXTERNAL_BUILDER_CALLED is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'touch "${EXTERNAL_BUILDER_CALLED:?}"' - } >"${external_thirdparty_dir}/build-thirdparty.sh" - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - EXTERNAL_BUILDER_CALLED="${external_builder_called}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an external thirdparty source tree" - fi - grep -Fq "Cannot rebuild thirdparty libraries with an external source tree" "${output_file}" || - fail "generic recovery did not reject an external thirdparty source tree" - [[ ! -e "${external_builder_called}" ]] || - fail "generic recovery invoked an external thirdparty builder" -} - -exercise_generic_recovery_dispatch - -# A Paimon-only build may publish only its own fingerprint. It must not make a -# stale Arrow installation pass the shared prebuilt validation. -prebuilt="${tmpdir}/prebuilt" -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" -printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" -for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" -done - -prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" -[[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || - fail "focused recovery dispatches a bundled source package as a build target" -[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || - fail "focused recovery does not download the complete Arrow source closure" - -# A legacy prebuilt may have the old combined marker but no component markers. -# Generic build.sh consumers must reject it before importing Arrow Compute. -arrow_paimon_build_fingerprint >"${prebuilt}/arrow-paimon-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "legacy combined marker certified an unversioned component closure" -fi - -printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/arrow-build-fingerprint.txt" -printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/paimon-build-fingerprint.txt" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "the complete shared prebuilt was rejected during fingerprint migration" -if ( - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Arrow marker survived a semantic fingerprint change" -fi -if ( - PAIMON_BUILD_SCHEMA_VERSION="${PAIMON_BUILD_SCHEMA_VERSION}-changed" - paimon_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Paimon marker survived a semantic fingerprint change" -fi - -publish_arrow_prebuilt_marker "${prebuilt}" -publish_paimon_prebuilt_marker "${prebuilt}" -rm "${prebuilt}/lib64/libarrow_compute.a" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "prebuilt validation accepted a missing Arrow Compute archive" -fi -touch "${prebuilt}/lib64/libarrow_compute.a" - -printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon-only marker update certified a stale Arrow build" -fi -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon build accepted a stale installed Arrow" -fi - -publish_arrow_prebuilt_marker "${prebuilt}" -require_arrow_prebuilt_for_paimon "${prebuilt}" || - fail "Paimon build rejected the selected installed Arrow" -arrow_paimon_prebuilt_valid "${prebuilt}" || fail "matching component markers were rejected" - -invalidate_paimon_prebuilt_marker "${prebuilt}" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Paimon rebuild left its old marker valid" -fi -publish_paimon_prebuilt_marker "${prebuilt}" - -invalidate_arrow_prebuilt_marker "${prebuilt}" -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Arrow rebuild left its old marker valid" -fi -publish_arrow_prebuilt_marker "${prebuilt}" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "republished component markers were rejected" - -echo "PASS" diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh deleted file mode 100755 index 4d3b92458b8c26..00000000000000 --- a/thirdparty/test/paimon-codec-dependency-test.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -fixture="${tmpdir}/paimon-codec-dependency" -focused_patch="${fixture}/codec-dependency.patch" -mkdir -p "${fixture}/src/paimon" -{ - printf '%s\n' 'add_paimon_lib(paimon' - printf '%s\n' ' SOURCES' - printf '%s\n' ' ${PAIMON_COMMON_SRCS}' - printf '%s\n' ' ${PAIMON_CORE_SRCS}' - printf '%s\n' ' DEPENDENCIES' - printf '%s\n' ' arrow' - printf '%s\n' ' tbb' - printf '%s\n' ' glog' - printf '%s\n' ' fmt' - printf '%s\n' ' roaring_bitmap' - printf '%s\n' ' xxhash' - printf '%s\n' ' Threads::Threads' - printf '%s\n' ' RapidJSON' - printf '%s\n' ' STATIC_LINK_LIBS' - printf '%s\n' ' arrow)' -} >"${fixture}/src/paimon/CMakeLists.txt" - -awk ' - /^diff --git / { - if (found) { - exit - } - if ($0 == "diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt") { - found = 1 - } - } - found { print } -' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}" -[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is missing" -( - cd "${fixture}" - patch -s -p1 <"${focused_patch}" -) || fail "the Paimon codec dependency patch did not apply" - -dependencies="$(sed -n '/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \ - "${fixture}/src/paimon/CMakeLists.txt")" -grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the ZSTD headers" -grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the Snappy headers" -grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the LZ4 headers" - -echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 84f4a73edaabf9..fa7643d62c33bb 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -50,10 +50,6 @@ export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" # export REPOSITORY_URL= DORIS_THIRDPARTY_REPOSITORY_URL="${DORIS_THIRDPARTY_REPOSITORY_URL:-https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/thirdparty/juicefs}" -DORIS_THIRDPARTY_VARS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=arrow-paimon-vars.sh -. "${DORIS_THIRDPARTY_VARS_DIR}/arrow-paimon-vars.sh" - ##################################################### # Download url, filename and unpaced filename # of all thirdparties @@ -145,7 +141,6 @@ LZO2_SOURCE=lzo-2.10 LZO2_MD5SUM="39d3f3f9c55c87b1e5d6888e1420f4b5" # rapidjson -# Updated to match paimon-cpp version (commit 232389d4f1012dddec4ef84861face2d2ba85709) RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" RAPIDJSON_NAME=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz RAPIDJSON_SOURCE=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709 @@ -239,6 +234,12 @@ ZSTD_NAME=zstd-1.5.7.tar.gz ZSTD_SOURCE=zstd-1.5.7 ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48" +# brotli +BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" +BROTLI_NAME="brotli-1.0.9.tar.gz" +BROTLI_SOURCE="brotli-1.0.9" +BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" + # flatbuffers FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz" FLATBUFFERS_NAME=flatbuffers-23.5.26.tar.gz @@ -258,6 +259,12 @@ GRPC_NAME="grpc-v1.54.3.tar.gz" GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" +# arrow +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz" +ARROW_NAME="apache-arrow-24.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-24.0.0" +ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" + # arrow-adbc # One source tree, three artifacts: the driver manager (statically linked into # doris_be), the JNI bridge (loaded by the FE adbc connector) and the SQLite @@ -455,6 +462,13 @@ BENCHMARK_NAME=benchmark-v1.8.0.tar.gz BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" +# xsimd +# for arrow-24.0.0, if arrow upgrade, this version may also need to be changed +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" +XSIMD_NAME="14.0.0.tar.gz" +XSIMD_SOURCE=xsimd-14.0.0 +XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" + # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" SIMDJSON_NAME=simdjson-3.11.6.tar.gz @@ -679,7 +693,6 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' - 'PAIMON_CPP' 'LANCE_C' )