From 2a327056de8c3fc98166f589733aa97df7d88ae6 Mon Sep 17 00:00:00 2001 From: suxiaogang Date: Tue, 1 Sep 2026 17:45:34 +0800 Subject: [PATCH] [chore](paimon) Remove paimon-cpp dependency ### What problem does this PR solve? Issue Number: N/A Related PR: #67378, #67432 Problem Summary: Remove the deprecated paimon-cpp package, reader implementation, build integration, tests, and Paimon-specific Arrow patch. Retain deprecated wire and configuration values for compatibility. Fold Arrow source variables back into the standard thirdparty manifest and remove the temporary fingerprint, recovery, and lifecycle scripts so the next master automation package is the final cleaned artifact. ### Release note Remove the deprecated paimon-cpp reader and build dependency. Paimon reads continue through the JNI and native readers. ### Check List (For Author) - Test: Unit Test and manual test - Paimon FE connector reactor: 553 tests passed, 1 live test skipped - Clean Arrow 24 patch application and Arrow package build - Thirdparty script tests, build hygiene, shell syntax, workflow YAML, and diff checks - Behavior changed: Yes. The paimon-cpp reader and enable_paimon_cpp_reader session variable are removed; deprecated wire values remain for compatibility. - Does this need documentation: No --- .github/workflows/be-ut-mac.yml | 21 +- .github/workflows/build-thirdparty.yml | 36 +- be/CMakeLists.txt | 50 -- be/cmake/thirdparty.cmake | 15 - be/src/common/config.cpp | 38 +- be/src/common/config.h | 5 +- be/src/exec/scan/file_scanner.cpp | 51 +- be/src/exec/scan/file_scanner_v2.cpp | 8 +- be/src/format/generic_reader.h | 2 +- be/src/format/table/paimon_cpp_reader.cpp | 417 ----------- be/src/format/table/paimon_cpp_reader.h | 101 --- .../format/table/paimon_doris_file_system.cpp | 679 ------------------ .../format/table/paimon_doris_file_system.h | 36 - .../table/paimon_predicate_converter.cpp | 657 ----------------- .../format/table/paimon_predicate_converter.h | 88 --- be/test/exec/scan/file_scanner_v2_test.cpp | 18 - .../format/table/paimon_cpp_reader_test.cpp | 350 --------- .../table/paimon_doris_file_system_test.cpp | 59 -- build.sh | 59 +- .../paimon/PaimonScanPlanProvider.java | 26 +- .../connector/paimon/PaimonScanRange.java | 13 +- .../paimon/PaimonScanPlanProviderTest.java | 105 +-- .../paimon/PaimonScanRangeReaderTypeTest.java | 18 +- .../org/apache/doris/qe/SessionVariable.java | 19 +- gensrc/thrift/PaloInternalService.thrift | 2 +- gensrc/thrift/PlanNodes.thrift | 1 + .../pipeline/common/github-utils.sh | 1 - .../PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md | 2 +- .../paimon/paimon_data_system_table.groovy | 2 - ...est_paimon_partition_pk_delete_refs.groovy | 3 - ...paimon_partition_schema_filter_refs.groovy | 3 - ...roovy => test_paimon_reader_parity.groovy} | 19 +- ...st_paimon_schema_time_travel_matrix.groovy | 30 +- thirdparty/arrow-paimon-vars.sh | 361 ---------- thirdparty/build-thirdparty.sh | 96 +-- thirdparty/download-thirdparty.sh | 64 +- thirdparty/paimon-cpp-cache.cmake | 137 ---- ...-24.0.0-force-write-int96-timestamps.patch | 5 +- .../patches/apache-arrow-24.0.0-paimon.patch | 226 ------ .../paimon-cpp-arrow-24-compatibility.patch | 64 -- .../patches/paimon-cpp-arrow-24-compute.patch | 44 -- .../paimon-cpp-buildutils-static-deps.patch | 362 ---------- .../test/arrow-paimon-lifecycle-test.sh | 455 ------------ .../test/paimon-codec-dependency-test.sh | 78 -- thirdparty/vars.sh | 25 +- 45 files changed, 111 insertions(+), 4740 deletions(-) delete mode 100644 be/src/format/table/paimon_cpp_reader.cpp delete mode 100644 be/src/format/table/paimon_cpp_reader.h delete mode 100644 be/src/format/table/paimon_doris_file_system.cpp delete mode 100644 be/src/format/table/paimon_doris_file_system.h delete mode 100644 be/src/format/table/paimon_predicate_converter.cpp delete mode 100644 be/src/format/table/paimon_predicate_converter.h delete mode 100644 be/test/format/table/paimon_cpp_reader_test.cpp delete mode 100644 be/test/format/table/paimon_doris_file_system_test.cpp rename regression-test/suites/external_table_p0/paimon/{test_paimon_cpp_reader.groovy => test_paimon_reader_parity.groovy} (81%) delete mode 100644 thirdparty/arrow-paimon-vars.sh delete mode 100644 thirdparty/paimon-cpp-cache.cmake delete mode 100644 thirdparty/patches/apache-arrow-24.0.0-paimon.patch delete mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch delete mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compute.patch delete mode 100644 thirdparty/patches/paimon-cpp-buildutils-static-deps.patch delete mode 100755 thirdparty/test/arrow-paimon-lifecycle-test.sh delete mode 100755 thirdparty/test/paimon-codec-dependency-test.sh diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index d970796dd21cb7..201e8d0a6404c8 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -75,9 +75,8 @@ jobs: ) brew install "${cellars[@]}" || true - # Paimon's bundled dependencies still require pre-CMake-4 policy - # compatibility. Match the supported version used by the full - # thirdparty jobs instead of relying on Homebrew's latest CMake. + # Match the supported version used by the full thirdparty jobs instead + # of relying on Homebrew's latest CMake. brew unlink cmake || true wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz tar -xzf cmake-3.25.3-macos-universal.tar.gz @@ -101,22 +100,6 @@ jobs: fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz - # Rebuild the Arrow/Paimon stack when the shared prebuilt predates the - # selected Arrow version/component closure. - # shellcheck source=thirdparty/arrow-paimon-vars.sh - . ./arrow-paimon-vars.sh - arrow_paimon_prebuilt_is_valid=false - if arrow_paimon_prebuilt_valid installed; then - arrow_paimon_prebuilt_is_valid=true - fi - if [[ "${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ - -o doris-thirdparty-source.tgz - tar -zxvf doris-thirdparty-source.tgz - ./download-thirdparty.sh arrow paimon_cpp xsimd brotli - export MACOSX_DEPLOYMENT_TARGET=12.0 - ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp - fi popd # macos-15 runners are Apple Silicon (arm64), so the JDK env var is diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 03244548080032..6baff66f3b7aed 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -32,7 +32,7 @@ jobs: if: github.event_name == 'pull_request' outputs: thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }} - arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }} + focused_test_changes: ${{ steps.filter.outputs.focused_test_changes }} steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 @@ -48,41 +48,15 @@ jobs: thirdparty_changes: - 'thirdparty/**' - 'env.sh' - arrow_paimon_lifecycle_changes: - - 'build.sh' - - 'env.sh' - - 'thirdparty/arrow-paimon-vars.sh' - - 'thirdparty/vars.sh' - - 'thirdparty/build-thirdparty.sh' - - 'thirdparty/download-thirdparty.sh' - - 'thirdparty/paimon-cpp-cache.cmake' - - 'thirdparty/patches/apache-arrow-*.patch' - - 'thirdparty/patches/paimon-cpp-*.patch' + focused_test_changes: - 'regression-test/pipeline/external/conf/fe.conf' - - 'thirdparty/test/arrow-paimon-lifecycle-test.sh' - 'thirdparty/test/adbc-jni-config-test.sh' - - 'thirdparty/test/paimon-codec-dependency-test.sh' - '.github/workflows/build-thirdparty.yml' - arrow_paimon_lifecycle_test: - name: Arrow/Paimon Lifecycle Test - needs: changes - if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout ${{ github.ref }} - uses: actions/checkout@v4 - - - name: Test focused thirdparty lifecycle - run: | - thirdparty/test/arrow-paimon-lifecycle-test.sh - thirdparty/test/adbc-jni-config-test.sh - thirdparty/test/paimon-codec-dependency-test.sh - script_test: name: Thirdparty Script Test needs: changes - if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} + if: ${{ needs.changes.outputs.thirdparty_changes == 'true' || needs.changes.outputs.focused_test_changes == 'true' }} runs-on: ubuntu-22.04 steps: - name: Checkout ${{ github.ref }} @@ -104,6 +78,10 @@ jobs: run: | thirdparty/test/azure-vcpkg-retry-test.sh + - name: Test thirdparty configuration + run: | + thirdparty/test/adbc-jni-config-test.sh + build_linux: name: Build Third Party Libraries (Linux) needs: changes diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 0dc2196dc81534..6b7df1b43fc3ab 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -199,8 +199,6 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF) message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") -option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) -set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -218,14 +216,6 @@ if (ENABLE_VARIANT_NESTED_GROUP AND "${VARIANT_NESTED_GROUP_MODULE_DIR}" STREQUA message(FATAL_ERROR "ENABLE_VARIANT_NESTED_GROUP requires VARIANT_NESTED_GROUP_MODULE_DIR") endif() -# Allow env to override when reconfiguring (avoid picking /usr/local). -if (DEFINED ENV{ENABLE_PAIMON_CPP}) - set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) -endif() -if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) - set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) -endif() - set(CMAKE_SKIP_RPATH TRUE) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) @@ -696,10 +686,6 @@ set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ) -if (ENABLE_PAIMON_CPP) - message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode") -endif() - if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) @@ -759,19 +745,6 @@ if (BUILD_BENCHMARK) ) endif() -set(PAIMON_FACTORY_REGISTRY_LIBS) -if (ENABLE_PAIMON_CPP) - # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via - # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). - # No dual-stack selection needed — single Arrow for everything. - - # paimon_parquet_file_format depends on Arrow Dataset symbols. - # Force-link it with --whole-archive so its factory registration runs. - set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format) - list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS}) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_START_GROUP} @@ -789,29 +762,6 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib) -if (ENABLE_PAIMON_CPP) - if (PAIMON_FACTORY_REGISTRY_LIBS) - if (APPLE) - foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS}) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,-force_load,$) - endforeach() - else() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} - -Wl,--whole-archive - ${PAIMON_FACTORY_REGISTRY_LIBS} - -Wl,--no-whole-archive) - endif() - endif() - - # paimon-cpp internal dependencies (renamed with _paimon suffix) - # These must come after paimon libraries to resolve symbols. - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon) - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon) -endif() - set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP}) # Add all external dependencies. They should come after the palo libs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index f3bd9867ee0fdd..fc7a38bf7b790e 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -189,18 +189,3 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) - -if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64) - add_thirdparty(paimon_parquet_file_format LIB64) - add_thirdparty(paimon_orc_file_format LIB64) - add_thirdparty(paimon_blob_file_format LIB64) - add_thirdparty(paimon_local_file_system LIB64) - add_thirdparty(paimon_file_index LIB64) - add_thirdparty(paimon_global_index LIB64) - - add_thirdparty(roaring_bitmap_paimon LIB64) - add_thirdparty(xxhash_paimon LIB64) - add_thirdparty(fmt_paimon LIB64) - add_thirdparty(tbb_paimon LIB64) -endif() diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index b4d7ef6116525f..faec885e0ef2e1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1721,40 +1721,10 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB /** Iceberg sink configurations **/ DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB -// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. -// Each entry uses the format "=", and file_type must be one of: -// local, hdfs, s3, http, broker. -DEFINE_Strings(paimon_file_system_scheme_mappings, - "file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs," - "s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3," - "abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http," - "ofs=broker,gfs=broker"); -DEFINE_Validator(paimon_file_system_scheme_mappings, - ([](const std::vector& mappings) -> bool { - doris::StringCaseUnorderedSet seen_schemes; - static const doris::StringCaseUnorderedSet supported_types = { - "local", "hdfs", "s3", "http", "broker"}; - for (const auto& raw_entry : mappings) { - std::string_view entry = doris::trim(raw_entry); - size_t separator = entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - std::string scheme = std::string(doris::trim(entry.substr(0, separator))); - std::string file_type = - std::string(doris::trim(entry.substr(separator + 1))); - if (scheme.empty() || file_type.empty()) { - return false; - } - if (supported_types.find(file_type) == supported_types.end()) { - return false; - } - if (!seen_schemes.insert(scheme).second) { - return false; - } - } - return true; - })); +/** Paimon sink configurations **/ +DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB +DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, + [](int64_t bytes) -> bool { return bytes > 0; }); DEFINE_mInt32(thrift_client_open_num_tries, "1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index c99be92dc23181..2deadaa9f61f1e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1787,8 +1787,9 @@ DECLARE_mInt64(hive_sink_max_file_size); /** Iceberg sink configurations **/ DECLARE_mInt64(iceberg_sink_max_file_size); -/** Paimon file system configurations **/ -DECLARE_Strings(paimon_file_system_scheme_mappings); +/** Paimon sink configurations **/ +// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. +DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); // Number of open tries, default 1 means only try to open once. // Retry the Open num_retries time waiting 100 milliseconds between retries. diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index d15b2c2b5f7c81..b367a799459d3e 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -76,9 +76,7 @@ #include "format/table/iceberg_sys_table_jni_reader.h" #include "format/table/jdbc_jni_reader.h" #include "format/table/max_compute_jni_reader.h" -#include "format/table/paimon_cpp_reader.h" #include "format/table/paimon_jni_reader.h" -#include "format/table/paimon_predicate_converter.h" #include "format/table/paimon_reader.h" #include "format/table/partition_column_filler.h" #include "format/table/remote_doris_reader.h" @@ -1085,50 +1083,11 @@ Status FileScanner::_get_next_reader() { _cur_reader = std::move(mc_reader); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "paimon") { - const auto& paimon_params = range.table_format_params.paimon_params; - bool use_paimon_cpp_reader = false; - if (paimon_params.__isset.reader_type) { - switch (paimon_params.reader_type) { - case TPaimonReaderType::PAIMON_CPP: - use_paimon_cpp_reader = true; - break; - case TPaimonReaderType::PAIMON_JNI: - break; - case TPaimonReaderType::PAIMON_NATIVE: - return Status::InternalError( - "invalid PAIMON_NATIVE reader_type for paimon FORMAT_JNI split, " - "possibly caused by FE/BE protocol mismatch"); - default: - return Status::InternalError( - "unknown paimon reader_type for paimon FORMAT_JNI split, possibly " - "caused by FE/BE protocol mismatch"); - } - } else { - // TODO: Remove this fallback after all FE versions set TPaimonReaderType. - use_paimon_cpp_reader = - _state->query_options().__isset.enable_paimon_cpp_reader && - _state->query_options().enable_paimon_cpp_reader; - } - if (use_paimon_cpp_reader) { - auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state, - _profile, range, _params); - if (!_is_load && !_push_down_conjuncts.empty()) { - PaimonPredicateConverter predicate_converter(_file_slot_descs, _state); - auto predicate = predicate_converter.build(_push_down_conjuncts); - if (predicate) { - cpp_reader->set_predicate(std::move(predicate)); - } - } - init_status = - static_cast(cpp_reader.get())->init_reader(&jni_ctx); - _cur_reader = std::move(cpp_reader); - } else { - auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, - _profile, range, _params); - init_status = - static_cast(paimon_reader.get())->init_reader(&jni_ctx); - _cur_reader = std::move(paimon_reader); - } + auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, + _profile, range, _params); + init_status = + static_cast(paimon_reader.get())->init_reader(&jni_ctx); + _cur_reader = std::move(paimon_reader); } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "hudi") { auto hudi_reader = HudiJniReader::create_unique( diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index af574ae216f065..52d71d5f8004ed 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -142,12 +142,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { } const auto& params = range.table_format_params.paimon_params; if (params.__isset.reader_type) { - if (params.reader_type == TPaimonReaderType::PAIMON_JNI) { - return params.__isset.paimon_split; - } - // V2 cannot pass a logical DataSplit through a raw native child without silently - // dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback. - return false; + return params.reader_type == TPaimonReaderType::PAIMON_JNI && + params.__isset.paimon_split; } if (params.__isset.paimon_split) { // Before reader_type was added, an encoded split unambiguously selected the Java diff --git a/be/src/format/generic_reader.h b/be/src/format/generic_reader.h index cc21ce82c2805e..10bad9e463a732 100644 --- a/be/src/format/generic_reader.h +++ b/be/src/format/generic_reader.h @@ -178,7 +178,7 @@ class GenericReader : public ProfileCollector { /// cannot be accidentally overridden. Status init_reader(ReaderInitContext* ctx) { // Apply push_down_agg_type early so _open_file_reader and _do_init_reader - // can use it (e.g., PaimonCppReader skips full init on COUNT pushdown). + // can use it before format-specific initialization. // on_after_init_reader may reset this (e.g., Iceberg with equality deletes). set_push_down_agg_type(ctx->push_down_agg_type); diff --git a/be/src/format/table/paimon_cpp_reader.cpp b/be/src/format/table/paimon_cpp_reader.cpp deleted file mode 100644 index d4164238ec371f..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.cpp +++ /dev/null @@ -1,417 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "arrow/record_batch.h" -#include "arrow/result.h" -#include "core/block/block.h" -#include "core/block/column_with_type_and_name.h" -#include "format/table/paimon_doris_file_system.h" -#include "format/table/partition_column_filler.h" -#include "paimon/defs.h" -#include "paimon/memory/memory_pool.h" -#include "paimon/read_context.h" -#include "paimon/table/source/table_read.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/url_coding.h" - -namespace doris { - -namespace { -constexpr const char* VALUE_KIND_FIELD = "_VALUE_KIND"; - -} // namespace - -PaimonCppReader::PaimonCppReader(const std::vector& file_slot_descs, - RuntimeState* state, RuntimeProfile* profile, - const TFileRangeDesc& range, - const TFileScanRangeParams* range_params) - : _file_slot_descs(file_slot_descs), - _state(state), - _profile(profile), - _range(range), - _range_params(range_params) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz); - if (range.__isset.table_format_params && - range.table_format_params.__isset.table_level_row_count) { - _remaining_table_level_row_count = range.table_format_params.table_level_row_count; - } else { - _remaining_table_level_row_count = -1; - } -} - -PaimonCppReader::~PaimonCppReader() = default; - -Status PaimonCppReader::on_before_init_reader(ReaderInitContext* ctx) { - _column_descs = ctx->column_descs; - _partition_values.clear(); - _partition_value_is_null.clear(); - if (ctx->range == nullptr || ctx->tuple_descriptor == nullptr || - !ctx->range->__isset.columns_from_path_keys) { - return Status::OK(); - } - - DORIS_CHECK(ctx->range->__isset.columns_from_path); - DORIS_CHECK(ctx->range->columns_from_path.size() == ctx->range->columns_from_path_keys.size()); - const bool has_null_flags = ctx->range->__isset.columns_from_path_is_null; - if (has_null_flags) { - DORIS_CHECK(ctx->range->columns_from_path_is_null.size() == - ctx->range->columns_from_path_keys.size()); - } - - std::unordered_map name_to_slot; - for (auto* slot : ctx->tuple_descriptor->slots()) { - name_to_slot.emplace(slot->col_name(), slot); - } - for (size_t i = 0; i < ctx->range->columns_from_path_keys.size(); ++i) { - const auto& key = ctx->range->columns_from_path_keys[i]; - auto slot_it = name_to_slot.find(key); - if (slot_it == name_to_slot.end()) { - continue; - } - _partition_values.emplace( - key, std::make_tuple(ctx->range->columns_from_path[i], slot_it->second)); - _partition_value_is_null.emplace( - key, has_null_flags ? ctx->range->columns_from_path_is_null[i] : false); - } - return Status::OK(); -} - -Status PaimonCppReader::on_after_read_block(Block* block, size_t* read_rows) { - if (_column_descs == nullptr || _partition_values.empty() || *read_rows == 0 || - _push_down_agg_type == TPushAggOp::type::COUNT) { - return Status::OK(); - } - return _fill_partition_columns(block, *read_rows); -} - -Status PaimonCppReader::init_reader() { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - return Status::OK(); - } - return _init_paimon_reader(); -} - -Status PaimonCppReader::_do_get_next_block(Block* block, size_t* read_rows, bool* eof) { - if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) { - auto rows = std::min(_remaining_table_level_row_count, - (int64_t)_state->query_options().batch_size); - _remaining_table_level_row_count -= rows; - auto mutable_columns_guard = block->mutate_columns_scoped(); - auto& mutate_columns = mutable_columns_guard.mutable_columns(); - for (auto& col : mutate_columns) { - col->resize(rows); - } - *read_rows = rows; - *eof = false; - if (_remaining_table_level_row_count == 0) { - *eof = true; - } - return Status::OK(); - } - - if (!_batch_reader) { - return Status::InternalError("paimon-cpp reader is not initialized"); - } - - if (_col_name_to_block_idx.empty()) { - _col_name_to_block_idx = block->get_name_to_pos_map(); - } - - auto batch_result = _batch_reader->NextBatch(); - if (!batch_result.ok()) { - return Status::InternalError("paimon-cpp read batch failed: {}", - batch_result.status().ToString()); - } - auto batch = std::move(batch_result).value(); - if (paimon::BatchReader::IsEofBatch(batch)) { - *read_rows = 0; - *eof = true; - return Status::OK(); - } - - arrow::Result> import_result = - arrow::ImportRecordBatch(batch.first.get(), batch.second.get()); - if (!import_result.ok()) { - return Status::InternalError("failed to import paimon-cpp arrow batch: {}", - import_result.status().message()); - } - - auto record_batch = std::move(import_result).ValueUnsafe(); - const auto num_rows = static_cast(record_batch->num_rows()); - const auto num_columns = record_batch->num_columns(); - auto columns_guard = block->mutate_columns_scoped(); - auto& columns = columns_guard.mutable_columns(); - for (int c = 0; c < num_columns; ++c) { - const auto& field = record_batch->schema()->field(c); - if (field->name() == VALUE_KIND_FIELD) { - continue; - } - - auto it = _col_name_to_block_idx.find(field->name()); - if (it == _col_name_to_block_idx.end()) { - // Skip columns that are not in the block (e.g., partition columns handled elsewhere) - continue; - } - const auto block_pos = it->second; - try { - RETURN_IF_ERROR(columns_guard.get_datatype_by_position(block_pos) - ->get_serde() - ->read_column_from_arrow(*columns[block_pos], - record_batch->column(c).get(), 0, - num_rows, _ctzz)); - } catch (Exception& e) { - return Status::InternalError("Failed to convert from arrow to block: {}", e.what()); - } - } - - *read_rows = num_rows; - *eof = false; - return Status::OK(); -} - -Status PaimonCppReader::_get_columns_impl( - std::unordered_map* name_to_type) { - for (const auto& slot : _file_slot_descs) { - name_to_type->emplace(slot->col_name(), slot->type()); - } - return Status::OK(); -} - -Status PaimonCppReader::_fill_partition_columns(Block* block, size_t num_rows) { - if (_col_name_to_block_idx.empty()) { - _col_name_to_block_idx = block->get_name_to_pos_map(); - } - - for (const auto& desc : *_column_descs) { - if (desc.category != ColumnCategory::PARTITION_KEY) { - continue; - } - auto value_it = _partition_values.find(desc.name); - if (value_it == _partition_values.end()) { - continue; - } - auto col_it = _col_name_to_block_idx.find(desc.name); - if (col_it == _col_name_to_block_idx.end()) { - return Status::InternalError("Missing partition column {} in block {}", desc.name, - block->dump_structure()); - } - - auto& column_with_type_and_name = block->get_by_position(col_it->second); - auto mutable_column = std::move(*column_with_type_and_name.column).mutate(); - const auto& [value, slot_desc] = value_it->second; - auto null_it = _partition_value_is_null.find(desc.name); - DORIS_CHECK(null_it != _partition_value_is_null.end()); - RETURN_IF_ERROR(fill_partition_column_from_path_value(*mutable_column, *slot_desc, value, - num_rows, null_it->second)); - column_with_type_and_name.column = std::move(mutable_column); - } - return Status::OK(); -} - -Status PaimonCppReader::close() { - if (_batch_reader) { - _batch_reader->Close(); - } - return Status::OK(); -} - -Status PaimonCppReader::_init_paimon_reader() { - register_paimon_doris_file_system(); - RETURN_IF_ERROR(_decode_split(&_split)); - - auto table_path_opt = _resolve_table_path(); - if (!table_path_opt.has_value()) { - return Status::InternalError( - "paimon-cpp missing paimon_table; cannot resolve paimon table root path"); - } - auto options = _build_options(); - auto read_columns = _build_read_columns(); - - // Avoid moving strings across module boundaries to prevent allocator mismatches in ASAN builds. - std::string table_path = table_path_opt.value(); - static std::once_flag options_log_once; - std::call_once(options_log_once, [&]() { - auto has_key = [&](const char* key) { - auto it = options.find(key); - return (it != options.end() && !it->second.empty()) ? "set" : "empty"; - }; - auto value_or = [&](const char* key) { - auto it = options.find(key); - return it != options.end() ? it->second : std::string(""); - }; - LOG(INFO) << "paimon-cpp options summary: table_path=" << table_path - << " AWS_ACCESS_KEY=" << has_key("AWS_ACCESS_KEY") - << " AWS_SECRET_KEY=" << has_key("AWS_SECRET_KEY") - << " AWS_TOKEN=" << has_key("AWS_TOKEN") - << " AWS_ENDPOINT=" << value_or("AWS_ENDPOINT") - << " AWS_REGION=" << value_or("AWS_REGION") - << " use_path_style=" << value_or("use_path_style") - << " fs.oss.endpoint=" << value_or("fs.oss.endpoint") - << " fs.s3a.endpoint=" << value_or("fs.s3a.endpoint"); - }); - paimon::ReadContextBuilder builder(table_path); - if (!read_columns.empty()) { - builder.SetReadSchema(read_columns); - } - if (!options.empty()) { - builder.SetOptions(options); - } - if (_predicate) { - builder.SetPredicate(_predicate); - builder.EnablePredicateFilter(true); - } - - auto context_result = builder.Finish(); - if (!context_result.ok()) { - return Status::InternalError("paimon-cpp build read context failed: {}", - context_result.status().ToString()); - } - auto context = std::move(context_result).value(); - - auto table_read_result = paimon::TableRead::Create(std::move(context)); - if (!table_read_result.ok()) { - return Status::InternalError("paimon-cpp create table read failed: {}", - table_read_result.status().ToString()); - } - auto table_read = std::move(table_read_result).value(); - auto reader_result = table_read->CreateReader(_split); - if (!reader_result.ok()) { - return Status::InternalError("paimon-cpp create reader failed: {}", - reader_result.status().ToString()); - } - _table_read = std::move(table_read); - _batch_reader = std::move(reader_result).value(); - return Status::OK(); -} - -Status PaimonCppReader::_decode_split(std::shared_ptr* split) { - if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.paimon_params || - !_range.table_format_params.paimon_params.__isset.paimon_split) { - return Status::InternalError("paimon-cpp missing paimon_split in scan range"); - } - const auto& encoded_split = _range.table_format_params.paimon_params.paimon_split; - std::string decoded_split; - if (!base64_decode(encoded_split, &decoded_split)) { - return Status::InternalError("paimon-cpp base64 decode paimon_split failed"); - } - auto pool = paimon::GetDefaultPool(); - auto split_result = - paimon::Split::Deserialize(decoded_split.data(), decoded_split.size(), pool); - if (!split_result.ok()) { - return Status::InternalError("paimon-cpp deserialize split failed: {}", - split_result.status().ToString()); - } - *split = std::move(split_result).value(); - return Status::OK(); -} - -std::optional PaimonCppReader::_resolve_table_path() const { - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_table && - !_range.table_format_params.paimon_params.paimon_table.empty()) { - return _range.table_format_params.paimon_params.paimon_table; - } - return std::nullopt; -} - -std::vector PaimonCppReader::_build_read_columns() const { - std::vector columns; - columns.reserve(_file_slot_descs.size()); - for (const auto& slot : _file_slot_descs) { - columns.emplace_back(slot->col_name()); - } - return columns; -} - -std::map PaimonCppReader::_build_options() const { - std::map options; - if (_range_params && _range_params->__isset.paimon_options && - !_range_params->paimon_options.empty()) { - options.insert(_range_params->paimon_options.begin(), _range_params->paimon_options.end()); - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.paimon_options) { - options.insert(_range.table_format_params.paimon_params.paimon_options.begin(), - _range.table_format_params.paimon_params.paimon_options.end()); - } - - if (_range_params && _range_params->__isset.properties && !_range_params->properties.empty()) { - for (const auto& kv : _range_params->properties) { - options[kv.first] = kv.second; - } - } else if (_range.__isset.table_format_params && - _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.hadoop_conf) { - for (const auto& kv : _range.table_format_params.paimon_params.hadoop_conf) { - options[kv.first] = kv.second; - } - } - - auto copy_if_missing = [&](const char* from_key, const char* to_key) { - if (options.find(to_key) != options.end()) { - return; - } - auto it = options.find(from_key); - if (it != options.end() && !it->second.empty()) { - options[to_key] = it->second; - } - }; - - // Map common OSS/S3 Hadoop configs to Doris S3 property keys. - copy_if_missing("fs.oss.accessKeyId", "AWS_ACCESS_KEY"); - copy_if_missing("fs.oss.accessKeySecret", "AWS_SECRET_KEY"); - copy_if_missing("fs.oss.sessionToken", "AWS_TOKEN"); - copy_if_missing("fs.oss.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.oss.region", "AWS_REGION"); - copy_if_missing("fs.s3a.access.key", "AWS_ACCESS_KEY"); - copy_if_missing("fs.s3a.secret.key", "AWS_SECRET_KEY"); - copy_if_missing("fs.s3a.session.token", "AWS_TOKEN"); - copy_if_missing("fs.s3a.endpoint", "AWS_ENDPOINT"); - copy_if_missing("fs.s3a.region", "AWS_REGION"); - copy_if_missing("fs.s3a.path.style.access", "use_path_style"); - - // Backfill file.format/manifest.format from split file_format to avoid - // paimon-cpp falling back to default manifest.format=avro. - if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params && - _range.table_format_params.paimon_params.__isset.file_format && - !_range.table_format_params.paimon_params.file_format.empty()) { - const auto& split_file_format = _range.table_format_params.paimon_params.file_format; - auto file_format_it = options.find(paimon::Options::FILE_FORMAT); - if (file_format_it == options.end() || file_format_it->second.empty()) { - options[paimon::Options::FILE_FORMAT] = split_file_format; - } - auto manifest_format_it = options.find(paimon::Options::MANIFEST_FORMAT); - if (manifest_format_it == options.end() || manifest_format_it->second.empty()) { - options[paimon::Options::MANIFEST_FORMAT] = split_file_format; - } - } - - options[paimon::Options::FILE_SYSTEM] = "doris"; - return options; -} - -} // namespace doris diff --git a/be/src/format/table/paimon_cpp_reader.h b/be/src/format/table/paimon_cpp_reader.h deleted file mode 100644 index f695234614104c..00000000000000 --- a/be/src/format/table/paimon_cpp_reader.h +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "common/status.h" -#include "format/generic_reader.h" -#include "paimon/reader/batch_reader.h" -#include "paimon/table/source/split.h" -#include "storage/olap_scan_common.h" - -namespace paimon { -class TableRead; -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeProfile; -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { - -class Block; - -class PaimonCppReader : public GenericReader { - ENABLE_FACTORY_CREATOR(PaimonCppReader); - -public: - PaimonCppReader(const std::vector& file_slot_descs, RuntimeState* state, - RuntimeProfile* profile, const TFileRangeDesc& range, - const TFileScanRangeParams* range_params); - ~PaimonCppReader() override; - - Status init_reader(); - Status _do_get_next_block(Block* block, size_t* read_rows, bool* eof) override; - Status _get_columns_impl(std::unordered_map* name_to_type) override; - Status close() override; - void set_predicate(std::shared_ptr predicate) { - _predicate = std::move(predicate); - } - -protected: - Status on_before_init_reader(ReaderInitContext* ctx) override; - Status on_after_read_block(Block* block, size_t* read_rows) override; - Status _do_init_reader(ReaderInitContext* /*ctx*/) override { return init_reader(); } - -private: - Status _fill_partition_columns(Block* block, size_t num_rows); - Status _init_paimon_reader(); - Status _decode_split(std::shared_ptr* split); - // Resolve paimon table root path for schema/manifest lookup. - std::optional _resolve_table_path() const; - std::vector _build_read_columns() const; - std::map _build_options() const; - - const std::vector& _file_slot_descs; - RuntimeState* _state = nullptr; - [[maybe_unused]] RuntimeProfile* _profile = nullptr; - const TFileRangeDesc& _range; - const TFileScanRangeParams* _range_params = nullptr; - - std::shared_ptr _split; - std::unique_ptr _table_read; - std::unique_ptr _batch_reader; - std::shared_ptr _predicate; - - std::unordered_map> - _partition_values; - std::unordered_map _partition_value_is_null; - std::unordered_map _col_name_to_block_idx; - int64_t _remaining_table_level_row_count = -1; - cctz::time_zone _ctzz; -}; - -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.cpp b/be/src/format/table/paimon_doris_file_system.cpp deleted file mode 100644 index bd303b5e26afb8..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.cpp +++ /dev/null @@ -1,679 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common/config.h" -#include "common/status.h" -#include "io/file_factory.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_system.h" -#include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" -#include "paimon/factories/factory.h" -#include "paimon/fs/file_system.h" -#include "paimon/fs/file_system_factory.h" -#include "paimon/result.h" -#include "paimon/status.h" -#include "util/string_util.h" - -namespace paimon { - -struct ParsedUri { - std::string scheme; - std::string authority; -}; - -ParsedUri parse_uri(const std::string& path) { - ParsedUri parsed; - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos || scheme_pos == 0) { - return parsed; - } - parsed.scheme = doris::to_lower(path.substr(0, scheme_pos)); - size_t authority_start = scheme_pos + delim_len; - if (authority_start >= path.size() || path[authority_start] == '/') { - return parsed; - } - size_t next_slash = path.find('/', authority_start); - if (next_slash == std::string::npos) { - parsed.authority = path.substr(authority_start); - } else { - parsed.authority = path.substr(authority_start, next_slash - authority_start); - } - return parsed; -} - -bool parse_scheme_mapping_target(std::string_view raw_target, doris::TFileType::type* type) { - std::string target = doris::to_lower(std::string(doris::trim(raw_target))); - if (target == "local") { - *type = doris::TFileType::FILE_LOCAL; - return true; - } - if (target == "hdfs") { - *type = doris::TFileType::FILE_HDFS; - return true; - } - if (target == "s3") { - *type = doris::TFileType::FILE_S3; - return true; - } - if (target == "http") { - *type = doris::TFileType::FILE_HTTP; - return true; - } - if (target == "broker") { - *type = doris::TFileType::FILE_BROKER; - return true; - } - return false; -} - -bool parse_scheme_mapping_entry(std::string_view raw_entry, std::string* scheme, - doris::TFileType::type* type) { - size_t separator = raw_entry.find('='); - if (separator == std::string_view::npos) { - return false; - } - *scheme = doris::to_lower(std::string(doris::trim(raw_entry.substr(0, separator)))); - if (scheme->empty()) { - return false; - } - return parse_scheme_mapping_target(raw_entry.substr(separator + 1), type); -} - -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme) { - if (scheme.empty()) { - return doris::TFileType::FILE_HDFS; - } - std::string normalized_scheme = doris::to_lower(scheme); - for (const auto& mapping_entry : doris::config::paimon_file_system_scheme_mappings) { - std::string configured_scheme; - doris::TFileType::type configured_type; - if (!parse_scheme_mapping_entry(mapping_entry, &configured_scheme, &configured_type)) { - continue; - } - if (configured_scheme == normalized_scheme) { - return configured_type; - } - } - return doris::TFileType::FILE_HDFS; -} - -std::string replace_scheme(const std::string& path, const std::string& scheme) { - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return path; - } - return scheme + "://" + path.substr(scheme_pos + delim_len); -} - -std::string normalize_local_path(const std::string& path) { - if (!path.starts_with("file:")) { - return path; - } - constexpr size_t file_prefix_len = 5; - size_t start = file_prefix_len; - if (path.compare(start, 2, "//") == 0 && path.size() - start > 2) { - size_t next_slash = path.find('/', start + 2); - if (next_slash == std::string::npos) { - return ""; - } - start = next_slash; - } - return path.substr(start); -} - -std::string normalize_path_for_type(const std::string& path, const std::string& scheme, - doris::TFileType::type type) { - if (type == doris::TFileType::FILE_LOCAL) { - return normalize_local_path(path); - } - if (type == doris::TFileType::FILE_S3 && scheme != "s3") { - return replace_scheme(path, "s3"); - } - return path; -} - -std::string build_fs_cache_key(doris::TFileType::type type, const ParsedUri& uri, - const std::string& default_fs_name) { - switch (type) { - case doris::TFileType::FILE_LOCAL: - return "local"; - case doris::TFileType::FILE_S3: - return "s3://" + uri.authority; - case doris::TFileType::FILE_HTTP: - return "http://" + uri.authority; - case doris::TFileType::FILE_BROKER: - return "broker"; - case doris::TFileType::FILE_HDFS: - default: - if (!uri.scheme.empty() || !uri.authority.empty()) { - return uri.scheme + "://" + uri.authority; - } - return default_fs_name; - } -} - -paimon::Status to_paimon_status(const doris::Status& status) { - if (status.ok()) { - return paimon::Status::OK(); - } - switch (status.code()) { - case doris::ErrorCode::NOT_FOUND: - case doris::ErrorCode::DIR_NOT_EXIST: - return paimon::Status::NotExist(status.to_string()); - case doris::ErrorCode::ALREADY_EXIST: - case doris::ErrorCode::FILE_ALREADY_EXIST: - return paimon::Status::Exist(status.to_string()); - case doris::ErrorCode::INVALID_ARGUMENT: - case doris::ErrorCode::INVALID_INPUT_SYNTAX: - return paimon::Status::Invalid(status.to_string()); - case doris::ErrorCode::NOT_IMPLEMENTED_ERROR: - return paimon::Status::NotImplemented(status.to_string()); - default: - return paimon::Status::IOError(status.to_string()); - } -} - -std::string join_path(const std::string& base, const std::string& child) { - if (base.empty()) { - return child; - } - if (base.back() == '/') { - return base + child; - } - return base + "/" + child; -} - -std::string parent_path_no_scheme(const std::string& path) { - if (path.empty()) { - return ""; - } - size_t end = path.size(); - while (end > 1 && path[end - 1] == '/') { - --end; - } - size_t pos = path.rfind('/', end - 1); - if (pos == std::string::npos) { - return ""; - } - if (pos == 0) { - return "/"; - } - return path.substr(0, pos); -} - -std::string parent_path(const std::string& path) { - ParsedUri uri = parse_uri(path); - if (uri.scheme.empty()) { - return parent_path_no_scheme(path); - } - size_t scheme_pos = path.find("://"); - size_t delim_len = 3; - if (scheme_pos == std::string::npos) { - scheme_pos = path.find(":/"); - delim_len = 2; - } - if (scheme_pos == std::string::npos) { - return parent_path_no_scheme(path); - } - size_t start = scheme_pos + delim_len; - size_t slash = path.find('/', start); - if (slash == std::string::npos) { - return ""; - } - std::string path_part = path.substr(slash); - std::string parent_part = parent_path_no_scheme(path_part); - if (parent_part.empty()) { - return ""; - } - std::string prefix = uri.scheme + "://"; - if (!uri.authority.empty()) { - prefix += uri.authority; - } - return prefix + parent_part; -} - -class DorisInputStream : public InputStream { -public: - DorisInputStream(doris::io::FileReaderSPtr reader, std::string path) - : reader_(std::move(reader)), path_(std::move(path)) {} - - Status Seek(int64_t offset, SeekOrigin origin) override { - int64_t target = 0; - if (origin == SeekOrigin::FS_SEEK_SET) { - target = offset; - } else if (origin == SeekOrigin::FS_SEEK_CUR) { - target = position_ + offset; - } else if (origin == SeekOrigin::FS_SEEK_END) { - target = static_cast(reader_->size()) + offset; - } else { - return Status::Invalid("unknown seek origin"); - } - if (target < 0) { - return Status::Invalid("seek position is negative"); - } - position_ = target; - return Status::OK(); - } - - Result GetPos() const override { return position_; } - - Result Read(char* buffer, uint32_t size) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(position_, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - position_ += static_cast(bytes_read); - return static_cast(bytes_read); - } - - Result Read(char* buffer, uint32_t size, uint64_t offset) override { - size_t bytes_read = 0; - doris::Status status = reader_->read_at(offset, doris::Slice(buffer, size), &bytes_read); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(bytes_read); - } - - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, - std::function&& callback) override { - Result result = Read(buffer, size, offset); - Status status = Status::OK(); - if (!result.ok()) { - status = result.status(); - } - callback(status); - } - - Result GetUri() const override { return path_; } - - Result Length() const override { return static_cast(reader_->size()); } - - Status Close() override { return to_paimon_status(reader_->close()); } - -private: - doris::io::FileReaderSPtr reader_; - std::string path_; - int64_t position_ = 0; -}; - -class DorisOutputStream : public OutputStream { -public: - DorisOutputStream(doris::io::FileWriterPtr writer, std::string path) - : writer_(std::move(writer)), path_(std::move(path)) {} - - Result Write(const char* buffer, uint32_t size) override { - doris::Status status = writer_->append(doris::Slice(buffer, size)); - if (!status.ok()) { - return to_paimon_status(status); - } - return static_cast(size); - } - - Status Flush() override { return Status::OK(); } - - Result GetPos() const override { - return static_cast(writer_->bytes_appended()); - } - - Result GetUri() const override { return path_; } - - Status Close() override { return to_paimon_status(writer_->close()); } - -private: - doris::io::FileWriterPtr writer_; - std::string path_; -}; - -class DorisBasicFileStatus : public BasicFileStatus { -public: - DorisBasicFileStatus(std::string path, bool is_dir) : path_(std::move(path)), is_dir_(is_dir) {} - - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - -private: - std::string path_; - bool is_dir_; -}; - -class DorisFileStatus : public FileStatus { -public: - DorisFileStatus(std::string path, bool is_dir, uint64_t length, int64_t mtime) - : path_(std::move(path)), is_dir_(is_dir), length_(length), mtime_(mtime) {} - - uint64_t GetLen() const override { return length_; } - bool IsDir() const override { return is_dir_; } - std::string GetPath() const override { return path_; } - int64_t GetModificationTime() const override { return mtime_; } - -private: - std::string path_; - bool is_dir_; - uint64_t length_; - int64_t mtime_; -}; - -class DorisFileSystem : public FileSystem { -public: - explicit DorisFileSystem(std::map options) - : options_(std::move(options)) { - auto it = options_.find("fs.defaultFS"); - if (it != options_.end()) { - default_fs_name_ = it->second; - } - } - - Result> Open(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - doris::io::FileReaderSPtr reader; - doris::io::FileReaderOptions reader_options = doris::io::FileReaderOptions::DEFAULT; - doris::Status status = fs->open_file(normalized_path, &reader, &reader_options); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(reader), normalized_path); - } - - Result> Create(const std::string& path, - bool overwrite) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto& fs = resolved.first; - auto& normalized_path = resolved.second; - if (!overwrite) { - bool exists = false; - doris::Status exists_status = fs->exists(normalized_path, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (exists) { - return Status::Exist("file already exists: ", normalized_path); - } - } - std::string parent = parent_path(normalized_path); - if (!parent.empty()) { - doris::Status mkdir_status = fs->create_directory(parent); - if (!mkdir_status.ok()) { - return to_paimon_status(mkdir_status); - } - } - doris::io::FileWriterPtr writer; - doris::Status status = fs->create_file(normalized_path, &writer); - if (!status.ok()) { - return to_paimon_status(status); - } - return std::make_unique(std::move(writer), normalized_path); - } - - Status Mkdirs(const std::string& path) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - doris::Status status = resolved.first->create_directory(resolved.second); - return to_paimon_status(status); - } - - Status Rename(const std::string& src, const std::string& dst) const override { - PAIMON_ASSIGN_OR_RAISE(auto src_resolved, resolve_path(src)); - PAIMON_ASSIGN_OR_RAISE(auto dst_resolved, resolve_path(dst)); - doris::Status status = src_resolved.first->rename(src_resolved.second, dst_resolved.second); - return to_paimon_status(status); - } - - Status Delete(const std::string& path, bool recursive = true) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - return Status::OK(); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return to_paimon_status(resolved.first->delete_file(resolved.second)); - } - if (recursive) { - return to_paimon_status(resolved.first->delete_directory(resolved.second)); - } - return to_paimon_status(size_status); - } - - Result> GetFileStatus(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status exists_status = resolved.first->exists(resolved.second, &exists); - if (!exists_status.ok()) { - return to_paimon_status(exists_status); - } - if (!exists) { - if (type != doris::TFileType::FILE_S3) { - return Status::NotExist("path not exists: ", resolved.second); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - int64_t size = 0; - doris::Status size_status = resolved.first->file_size(resolved.second, &size); - if (size_status.ok()) { - return std::make_unique(resolved.second, false, - static_cast(size), 0); - } - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!list_exists && files.empty()) { - return Status::NotExist("path not exists: ", resolved.second); - } - return std::make_unique(resolved.second, true, 0, 0); - } - - Status ListDir(const std::string& directory, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(directory)); - auto file_status = GetFileStatus(directory); - if (file_status.ok() && !file_status.value()->IsDir()) { - return Status::IOError("path is not a directory: ", directory); - } - std::vector files; - bool exists = false; - doris::Status status = resolved.first->list(resolved.second, false, &files, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file)); - } - return Status::OK(); - } - - Status ListFileStatus(const std::string& path, - std::vector>* status_list) const override { - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - auto self_status = GetFileStatus(path); - if (!self_status.ok()) { - if (self_status.status().IsNotExist()) { - return Status::OK(); - } - return self_status.status(); - } - if (!self_status.value()->IsDir()) { - status_list->emplace_back(std::move(self_status).value()); - return Status::OK(); - } - std::vector files; - bool exists = false; - doris::Status list_status = resolved.first->list(resolved.second, false, &files, &exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - if (!exists) { - return Status::OK(); - } - status_list->reserve(status_list->size() + files.size()); - for (const auto& file : files) { - uint64_t length = file.is_file ? static_cast(file.file_size) : 0; - status_list->emplace_back(std::make_unique( - join_path(resolved.second, file.file_name), !file.is_file, length, 0)); - } - return Status::OK(); - } - - Result Exists(const std::string& path) const override { - ParsedUri uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path)); - bool exists = false; - doris::Status status = resolved.first->exists(resolved.second, &exists); - if (!status.ok()) { - return to_paimon_status(status); - } - if (!exists && type == doris::TFileType::FILE_S3) { - std::vector files; - bool list_exists = false; - doris::Status list_status = - resolved.first->list(resolved.second, false, &files, &list_exists); - if (!list_status.ok()) { - return to_paimon_status(list_status); - } - return list_exists || !files.empty(); - } - return exists; - } - -private: - Result> resolve_path( - const std::string& path) const { - auto uri = parse_uri(path); - doris::TFileType::type type = map_scheme_to_file_type(uri.scheme); - std::string normalized_path = normalize_path_for_type(path, uri.scheme, type); - if (type == doris::TFileType::FILE_LOCAL) { - doris::io::FileSystemSPtr fs = doris::io::global_local_filesystem(); - return std::make_pair(std::move(fs), normalized_path); - } - std::string fs_key = build_fs_cache_key(type, uri, default_fs_name_); - { - std::lock_guard lock(fs_lock_); - auto it = fs_cache_.find(fs_key); - if (it != fs_cache_.end()) { - return std::make_pair(it->second, normalized_path); - } - } - doris::io::FSPropertiesRef fs_properties(type); - const std::map* properties = &options_; - std::map properties_override; - if (type == doris::TFileType::FILE_HTTP && !options_.contains("uri") && - !uri.scheme.empty()) { - properties_override = options_; - properties_override["uri"] = uri.scheme + "://" + uri.authority; - properties = &properties_override; - } - fs_properties.properties = properties; - if (!broker_addresses_.empty()) { - fs_properties.broker_addresses = &broker_addresses_; - } - doris::io::FileDescription file_description = { - .path = normalized_path, .file_size = -1, .mtime = 0, .fs_name = default_fs_name_}; - auto fs_result = doris::FileFactory::create_fs(fs_properties, file_description); - if (!fs_result.has_value()) { - return to_paimon_status(fs_result.error()); - } - doris::io::FileSystemSPtr fs = std::move(fs_result).value(); - { - std::lock_guard lock(fs_lock_); - fs_cache_.emplace(std::move(fs_key), fs); - } - return std::make_pair(std::move(fs), std::move(normalized_path)); - } - - std::map options_; - std::vector broker_addresses_; - std::string default_fs_name_; - mutable std::mutex fs_lock_; - mutable std::unordered_map fs_cache_; -}; - -class DorisFileSystemFactory : public FileSystemFactory { -public: - static const char IDENTIFIER[]; - - const char* Identifier() const override { return IDENTIFIER; } - - Result> Create( - const std::string& path, - const std::map& options) const override { - return std::make_unique(options); - } -}; - -const char DorisFileSystemFactory::IDENTIFIER[] = "doris"; - -REGISTER_PAIMON_FACTORY(DorisFileSystemFactory); - -} // namespace paimon - -namespace doris { - -void register_paimon_doris_file_system() {} - -} // namespace doris diff --git a/be/src/format/table/paimon_doris_file_system.h b/be/src/format/table/paimon_doris_file_system.h deleted file mode 100644 index 561e7aeac30fe4..00000000000000 --- a/be/src/format/table/paimon_doris_file_system.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include - -namespace paimon { - -// Visible for tests: maps a URI scheme to the Doris file type used by paimon-cpp. -doris::TFileType::type map_scheme_to_file_type(const std::string& scheme); - -} // namespace paimon - -namespace doris { - -// Force-link helper so the paimon-cpp file system factory registration is kept. -void register_paimon_doris_file_system(); - -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.cpp b/be/src/format/table/paimon_predicate_converter.cpp deleted file mode 100644 index 6fa6d4907e7340..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.cpp +++ /dev/null @@ -1,657 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_predicate_converter.h" - -#include -#include -#include - -#include "core/column/column_const.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/field.h" -#include "core/types.h" -#include "core/value/decimalv2_value.h" -#include "core/value/timestamptz_value.h" -#include "core/value/vdatetime_value.h" -#include "exprs/vcompound_pred.h" -#include "exprs/vdirect_in_predicate.h" -#include "exprs/vectorized_fn_call.h" -#include "exprs/vexpr.h" -#include "exprs/vin_predicate.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" -#include "paimon/data/decimal.h" -#include "paimon/data/timestamp.h" -#include "paimon/predicate/predicate_builder.h" -#include "runtime/descriptors.h" -#include "runtime/runtime_state.h" -#include "util/timezone_utils.h" - -namespace doris { - -PaimonPredicateConverter::PaimonPredicateConverter( - const std::vector& file_slot_descs, RuntimeState* state) - : _state(state) { - _field_index_by_name.reserve(file_slot_descs.size()); - for (size_t i = 0; i < file_slot_descs.size(); ++i) { - const auto& name = file_slot_descs[i]->col_name(); - auto normalized = _normalize_name(name); - if (_field_index_by_name.find(normalized) == _field_index_by_name.end()) { - _field_index_by_name.emplace(std::move(normalized), static_cast(i)); - } - } - - if (!TimezoneUtils::find_cctz_time_zone("GMT", _gmt_tz)) { - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _gmt_tz); - } -} - -std::shared_ptr PaimonPredicateConverter::build( - const VExprContextSPtrs& conjuncts) { - std::vector> predicates; - predicates.reserve(conjuncts.size()); - for (const auto& conjunct : conjuncts) { - if (!conjunct || !conjunct->root()) { - continue; - } - auto root = conjunct->root(); - if (root->is_rf_wrapper()) { - if (auto impl = root->get_impl()) { - root = impl; - } - } - auto predicate = _convert_expr(root); - if (predicate) { - predicates.emplace_back(std::move(predicate)); - } - } - - if (predicates.empty()) { - return nullptr; - } - if (predicates.size() == 1) { - return predicates.front(); - } - auto and_result = paimon::PredicateBuilder::And(predicates); - if (!and_result.ok()) { - return nullptr; - } - return std::move(and_result).value(); -} - -std::shared_ptr PaimonPredicateConverter::_convert_expr(const VExprSPtr& expr) { - if (!expr) { - return nullptr; - } - - auto uncast = VExpr::expr_without_cast(expr); - - if (auto* direct_in = dynamic_cast(uncast.get())) { - VExprSPtr in_expr; - if (direct_in->get_slot_in_expr(in_expr)) { - return _convert_in(in_expr); - } - return nullptr; - } - - if (dynamic_cast(uncast.get()) != nullptr) { - return _convert_in(uncast); - } - - switch (uncast->op()) { - case TExprOpcode::COMPOUND_AND: - case TExprOpcode::COMPOUND_OR: - return _convert_compound(uncast); - case TExprOpcode::COMPOUND_NOT: - return nullptr; - case TExprOpcode::EQ: - case TExprOpcode::EQ_FOR_NULL: - case TExprOpcode::NE: - case TExprOpcode::GE: - case TExprOpcode::GT: - case TExprOpcode::LE: - case TExprOpcode::LT: - return _convert_binary(uncast); - default: - break; - } - - if (auto* fn = dynamic_cast(uncast.get())) { - auto fn_name = _normalize_name(fn->function_name()); - if (fn_name == "is_null_pred" || fn_name == "is_not_null_pred") { - return _convert_is_null(uncast, fn_name); - } - if (fn_name == "like") { - return _convert_like_prefix(uncast); - } - } - - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_compound( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto left = _convert_expr(expr->get_child(0)); - if (!left) { - return nullptr; - } - auto right = _convert_expr(expr->get_child(1)); - if (!right) { - return nullptr; - } - - if (expr->op() == TExprOpcode::COMPOUND_AND) { - auto and_result = paimon::PredicateBuilder::And({left, right}); - return and_result.ok() ? std::move(and_result).value() : nullptr; - } - if (expr->op() == TExprOpcode::COMPOUND_OR) { - auto or_result = paimon::PredicateBuilder::Or({left, right}); - return or_result.ok() ? std::move(or_result).value() : nullptr; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_in(const VExprSPtr& expr) { - auto* in_pred = dynamic_cast(expr.get()); - if (!in_pred || expr->get_num_children() < 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - std::vector literals; - literals.reserve(expr->get_num_children() - 1); - for (uint16_t i = 1; i < expr->get_num_children(); ++i) { - auto literal = _convert_literal(expr->get_child(i), *field_meta->slot_desc, - field_meta->field_type); - if (!literal) { - return nullptr; - } - literals.emplace_back(std::move(*literal)); - } - - if (literals.empty()) { - return nullptr; - } - if (in_pred->is_not_in()) { - return paimon::PredicateBuilder::NotIn(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); - } - return paimon::PredicateBuilder::In(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, literals); -} - -std::shared_ptr PaimonPredicateConverter::_convert_binary( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - - if (expr->op() == TExprOpcode::EQ_FOR_NULL) { - return paimon::PredicateBuilder::IsNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - - auto literal = - _convert_literal(expr->get_child(1), *field_meta->slot_desc, field_meta->field_type); - if (!literal) { - return nullptr; - } - - switch (expr->op()) { - case TExprOpcode::EQ: - return paimon::PredicateBuilder::Equal(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::NE: - return paimon::PredicateBuilder::NotEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GE: - return paimon::PredicateBuilder::GreaterOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::GT: - return paimon::PredicateBuilder::GreaterThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LE: - return paimon::PredicateBuilder::LessOrEqual(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - case TExprOpcode::LT: - return paimon::PredicateBuilder::LessThan(field_meta->index, - field_meta->slot_desc->col_name(), - field_meta->field_type, *literal); - default: - break; - } - return nullptr; -} - -std::shared_ptr PaimonPredicateConverter::_convert_is_null( - const VExprSPtr& expr, const std::string& fn_name) { - if (!expr || expr->get_num_children() != 1) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta) { - return nullptr; - } - if (fn_name == "is_not_null_pred") { - return paimon::PredicateBuilder::IsNotNull( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type); - } - return paimon::PredicateBuilder::IsNull(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type); -} - -std::shared_ptr PaimonPredicateConverter::_convert_like_prefix( - const VExprSPtr& expr) { - if (!expr || expr->get_num_children() != 2) { - return nullptr; - } - auto field_meta = _resolve_field(expr->get_child(0)); - if (!field_meta || field_meta->field_type != paimon::FieldType::STRING) { - return nullptr; - } - - auto pattern_opt = _extract_string_literal(expr->get_child(1)); - if (!pattern_opt) { - return nullptr; - } - const std::string& pattern = *pattern_opt; - if (!pattern.empty() && pattern.front() == '%') { - return nullptr; - } - if (pattern.empty() || pattern.back() != '%') { - return nullptr; - } - - std::string prefix = pattern.substr(0, pattern.size() - 1); - paimon::Literal lower_literal(paimon::FieldType::STRING, prefix.data(), prefix.size()); - auto lower_pred = paimon::PredicateBuilder::GreaterOrEqual( - field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type, - lower_literal); - - auto upper_prefix = _next_prefix(prefix); - if (!upper_prefix) { - return lower_pred; - } - - paimon::Literal upper_literal(paimon::FieldType::STRING, upper_prefix->data(), - upper_prefix->size()); - auto upper_pred = - paimon::PredicateBuilder::LessThan(field_meta->index, field_meta->slot_desc->col_name(), - field_meta->field_type, upper_literal); - auto and_result = paimon::PredicateBuilder::And({lower_pred, upper_pred}); - return and_result.ok() ? std::move(and_result).value() : nullptr; -} - -std::optional PaimonPredicateConverter::_resolve_field( - const VExprSPtr& expr) const { - if (!_state || !expr) { - return std::nullopt; - } - auto slot_expr = VExpr::expr_without_cast(expr); - auto* slot_ref = dynamic_cast(slot_expr.get()); - if (!slot_ref) { - return std::nullopt; - } - auto* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id()); - if (!slot_desc) { - return std::nullopt; - } - auto normalized = _normalize_name(slot_desc->col_name()); - auto it = _field_index_by_name.find(normalized); - if (it == _field_index_by_name.end()) { - return std::nullopt; - } - auto slot_type = slot_desc->type(); - auto field_type = - _to_paimon_field_type(slot_type->get_primitive_type(), slot_type->get_precision()); - if (!field_type) { - return std::nullopt; - } - return FieldMeta {it->second, *field_type, slot_desc}; -} - -std::optional PaimonPredicateConverter::_convert_literal( - const VExprSPtr& expr, const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - PrimitiveType slot_primitive = slot_desc.type()->get_primitive_type(); - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - - Field field; - col->get(0, field); - - switch (slot_primitive) { - case TYPE_BOOLEAN: { - if (literal_primitive != TYPE_BOOLEAN) { - return std::nullopt; - } - return paimon::Literal(static_cast(field.get())); - } - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: { - if (!_is_integer_type(literal_primitive)) { - return std::nullopt; - } - int64_t value = 0; - switch (literal_primitive) { - case TYPE_TINYINT: - value = field.get(); - break; - case TYPE_SMALLINT: - value = field.get(); - break; - case TYPE_INT: - value = field.get(); - break; - case TYPE_BIGINT: - value = field.get(); - break; - default: - return std::nullopt; - } - if (slot_primitive == TYPE_TINYINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_SMALLINT) { - return paimon::Literal(static_cast(value)); - } - if (slot_primitive == TYPE_INT) { - return paimon::Literal(static_cast(value)); - } - return paimon::Literal(value); - } - case TYPE_DOUBLE: { - if (literal_primitive != TYPE_DOUBLE && literal_primitive != TYPE_FLOAT) { - return std::nullopt; - } - double value = 0; - if (literal_primitive == TYPE_FLOAT) { - value = static_cast(field.get()); - } else { - value = field.get(); - } - return paimon::Literal(value); - } - case TYPE_DATE: - case TYPE_DATEV2: { - if (!_is_date_type(literal_primitive)) { - return std::nullopt; - } - int64_t seconds = 0; - if (literal_primitive == TYPE_DATE) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } else if (literal_primitive == TYPE_DATEV2) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&seconds, _gmt_tz); - } - int32_t days = _seconds_to_days(seconds); - return paimon::Literal(paimon::FieldType::DATE, days); - } - case TYPE_DATETIME: - case TYPE_DATETIMEV2: { - if (!_is_datetime_type(literal_primitive)) { - return std::nullopt; - } - if (literal_primitive == TYPE_DATETIME) { - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - int64_t seconds = 0; - dt.unix_timestamp(&seconds, _gmt_tz); - return paimon::Literal(paimon::Timestamp::FromEpochMillis(seconds * 1000)); - } - std::pair ts; - const auto& dt = field.get(); - if (!dt.is_valid_date()) { - return std::nullopt; - } - dt.unix_timestamp(&ts, _gmt_tz); - int64_t millis = ts.first * 1000 + ts.second / 1000; - return paimon::Literal(paimon::Timestamp::FromEpochMillis(millis)); - } - case TYPE_VARCHAR: - case TYPE_STRING: { - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - const auto& value = field.get(); - return paimon::Literal(field_type, value.data(), value.size()); - } - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: { - if (!_is_decimal_type(literal_primitive)) { - return std::nullopt; - } - int32_t precision = static_cast(literal_type->get_precision()); - int32_t scale = static_cast(literal_type->get_scale()); - if (precision <= 0 || precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - - paimon::Decimal::int128_t value = 0; - switch (literal_primitive) { - case TYPE_DECIMALV2: { - const auto& dec = field.get(); - value = dec.value(); - break; - } - case TYPE_DECIMAL32: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL64: { - const auto& dec = field.get(); - value = dec.value; - break; - } - case TYPE_DECIMAL128I: { - const auto& dec = field.get(); - value = dec.value; - break; - } - default: - return std::nullopt; - } - return paimon::Literal(paimon::Decimal(precision, scale, value)); - } - default: - break; - } - return std::nullopt; -} - -std::optional PaimonPredicateConverter::_extract_string_literal( - const VExprSPtr& expr) const { - auto literal_expr = VExpr::expr_without_cast(expr); - auto* literal = dynamic_cast(literal_expr.get()); - if (!literal) { - return std::nullopt; - } - auto literal_type = remove_nullable(literal->get_data_type()); - PrimitiveType literal_primitive = literal_type->get_primitive_type(); - if (!_is_string_type(literal_primitive)) { - return std::nullopt; - } - - ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const(); - if (const auto* nullable = check_and_get_column(*col)) { - if (nullable->is_null_at(0)) { - return std::nullopt; - } - col = nullable->get_nested_column_ptr(); - } - Field field; - col->get(0, field); - const auto& value = field.get(); - return value; -} - -std::string PaimonPredicateConverter::_normalize_name(std::string_view name) { - std::string out(name); - std::transform(out.begin(), out.end(), out.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return out; -} - -std::optional PaimonPredicateConverter::_next_prefix(const std::string& prefix) { - if (prefix.empty()) { - return std::nullopt; - } - std::string upper = prefix; - for (int i = static_cast(upper.size()) - 1; i >= 0; --i) { - auto c = static_cast(upper[i]); - if (c != 0xFF) { - upper[i] = static_cast(c + 1); - upper.resize(i + 1); - return upper; - } - } - return std::nullopt; -} - -int32_t PaimonPredicateConverter::_seconds_to_days(int64_t seconds) { - static constexpr int64_t kSecondsPerDay = 24 * 60 * 60; - int64_t days = seconds / kSecondsPerDay; - if (seconds < 0 && seconds % kSecondsPerDay != 0) { - --days; - } - return static_cast(days); -} - -bool PaimonPredicateConverter::_is_integer_type(PrimitiveType type) { - switch (type) { - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_string_type(PrimitiveType type) { - return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING; -} - -bool PaimonPredicateConverter::_is_decimal_type(PrimitiveType type) { - switch (type) { - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - return true; - default: - return false; - } -} - -bool PaimonPredicateConverter::_is_date_type(PrimitiveType type) { - return type == TYPE_DATE || type == TYPE_DATEV2; -} - -bool PaimonPredicateConverter::_is_datetime_type(PrimitiveType type) { - return type == TYPE_DATETIME || type == TYPE_DATETIMEV2; -} - -std::optional PaimonPredicateConverter::_to_paimon_field_type( - PrimitiveType type, uint32_t precision) { - switch (type) { - case TYPE_BOOLEAN: - return paimon::FieldType::BOOLEAN; - case TYPE_TINYINT: - return paimon::FieldType::TINYINT; - case TYPE_SMALLINT: - return paimon::FieldType::SMALLINT; - case TYPE_INT: - return paimon::FieldType::INT; - case TYPE_BIGINT: - return paimon::FieldType::BIGINT; - case TYPE_DOUBLE: - return paimon::FieldType::DOUBLE; - case TYPE_VARCHAR: - case TYPE_STRING: - return paimon::FieldType::STRING; - case TYPE_DATE: - case TYPE_DATEV2: - return paimon::FieldType::DATE; - case TYPE_DATETIME: - case TYPE_DATETIMEV2: - return paimon::FieldType::TIMESTAMP; - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - if (precision > 0 && precision > paimon::Decimal::MAX_PRECISION) { - return std::nullopt; - } - return paimon::FieldType::DECIMAL; - case TYPE_FLOAT: - case TYPE_CHAR: - default: - return std::nullopt; - } -} - -} // namespace doris diff --git a/be/src/format/table/paimon_predicate_converter.h b/be/src/format/table/paimon_predicate_converter.h deleted file mode 100644 index cb391077cd2e7f..00000000000000 --- a/be/src/format/table/paimon_predicate_converter.h +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "cctz/time_zone.h" -#include "core/data_type/define_primitive_type.h" -#include "exprs/vexpr_fwd.h" -#include "paimon/defs.h" -#include "paimon/predicate/literal.h" - -namespace paimon { -class Predicate; -} // namespace paimon - -namespace doris { -class RuntimeState; -class SlotDescriptor; -} // namespace doris - -namespace doris { - -class PaimonPredicateConverter { -public: - PaimonPredicateConverter(const std::vector& file_slot_descs, - RuntimeState* state); - - std::shared_ptr build(const VExprContextSPtrs& conjuncts); - -private: - struct FieldMeta { - int32_t index = -1; - paimon::FieldType field_type = paimon::FieldType::UNKNOWN; - const SlotDescriptor* slot_desc = nullptr; - }; - - std::shared_ptr _convert_expr(const VExprSPtr& expr); - std::shared_ptr _convert_compound(const VExprSPtr& expr); - std::shared_ptr _convert_in(const VExprSPtr& expr); - std::shared_ptr _convert_binary(const VExprSPtr& expr); - std::shared_ptr _convert_is_null(const VExprSPtr& expr, - const std::string& fn_name); - std::shared_ptr _convert_like_prefix(const VExprSPtr& expr); - - std::optional _resolve_field(const VExprSPtr& expr) const; - std::optional _convert_literal(const VExprSPtr& expr, - const SlotDescriptor& slot_desc, - paimon::FieldType field_type) const; - std::optional _extract_string_literal(const VExprSPtr& expr) const; - - static std::string _normalize_name(std::string_view name); - static std::optional _next_prefix(const std::string& prefix); - static int32_t _seconds_to_days(int64_t seconds); - static bool _is_integer_type(PrimitiveType type); - static bool _is_string_type(PrimitiveType type); - static bool _is_decimal_type(PrimitiveType type); - static bool _is_date_type(PrimitiveType type); - static bool _is_datetime_type(PrimitiveType type); - static std::optional _to_paimon_field_type(PrimitiveType type, - uint32_t precision); - - std::unordered_map _field_index_by_name; - RuntimeState* _state = nullptr; - cctz::time_zone _gmt_tz; -}; - -} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 56b9bc8923133f..8a6a0df7ee0ee6 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -80,15 +80,6 @@ TFileRangeDesc hudi_range_with_delta_logs() { return range; } -TFileRangeDesc paimon_cpp_jni_range() { - auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); - TPaimonFileDesc paimon_params; - paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_CPP); - paimon_params.__set_file_format("parquet"); - range.table_format_params.__set_paimon_params(std::move(paimon_params)); - return range; -} - TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { auto range = range_with_format("paimon", TFileFormatType::FORMAT_JNI); TPaimonFileDesc paimon_params; @@ -509,19 +500,10 @@ TEST(FileScannerV2Test, LegacyCountExemptionRequiresMetadataCountOnEveryRange) { TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { TQueryOptions query_options; query_options.__set_enable_file_scanner_v2(true); - query_options.__set_enable_paimon_cpp_reader(true); TFileScanRangeParams params; params.__set_format_type(TFileFormatType::FORMAT_JNI); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - const auto cpp_range = paimon_cpp_jni_range(); - EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_range)); - const auto cpp_status = FileScannerV2::TEST_validate_scan_range(params, cpp_range); - EXPECT_TRUE(cpp_status.is()); - - // Older FE plans without reader_type used Java whenever the C++ option was disabled. - query_options.__set_enable_paimon_cpp_reader(false); - EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } diff --git a/be/test/format/table/paimon_cpp_reader_test.cpp b/be/test/format/table/paimon_cpp_reader_test.cpp deleted file mode 100644 index f2602c34fd010e..00000000000000 --- a/be/test/format/table/paimon_cpp_reader_test.cpp +++ /dev/null @@ -1,350 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_cpp_reader.h" - -#include -#include - -#include -#include -#include -#include - -#include "core/block/block.h" -#include "exec/common/endian.h" -#include "format/format_common.h" -#include "format/table/deletion_vector_reader.h" -#include "format/table/paimon_reader.h" -#include "io/fs/file_meta_cache.h" -#include "io/io_common.h" -#include "roaring/roaring.hh" -#include "runtime/runtime_profile.h" -#include "runtime/runtime_state.h" -#include "util/timezone_utils.h" - -namespace doris { - -namespace { - -std::vector build_paimon_deletion_vector_buffer(const std::vector& positions) { - roaring::Roaring rows; - for (const auto position : positions) { - rows.add(position); - } - - const size_t bitmap_size = rows.getSizeInBytes(); - const uint32_t total_length = static_cast(4 + bitmap_size); - std::vector buffer(4 + total_length); - BigEndian::Store32(buffer.data(), total_length); - constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'}; - memcpy(buffer.data() + 4, PAIMON_BITMAP_MAGIC, 4); - rows.write(buffer.data() + 8); - return buffer; -} - -TFileScanRangeParams make_local_paimon_scan_params(TFileFormatType::type format_type) { - TFileScanRangeParams scan_params; - scan_params.__set_file_type(TFileType::FILE_LOCAL); - scan_params.__set_format_type(format_type); - return scan_params; -} - -TFileRangeDesc make_paimon_range_with_deletion_file(const std::string& deletion_file_path) { - TFileRangeDesc range; - range.__set_path("data.parquet"); - range.__set_start_offset(0); - range.__set_size(0); - range.__set_fs_name(""); - range.__isset.table_format_params = true; - range.table_format_params.__set_table_format_type("paimon"); - - TPaimonFileDesc paimon_params; - paimon_params.__set_file_format("parquet"); - TPaimonDeletionFileDesc deletion_file; - deletion_file.__set_path(deletion_file_path); - deletion_file.__set_offset(0); - deletion_file.__set_length(16); - paimon_params.__set_deletion_file(deletion_file); - range.table_format_params.__set_paimon_params(paimon_params); - return range; -} - -} // namespace - -class PaimonCppReaderTest : public testing::Test { -protected: - void SetUp() override { - _query_options.__set_batch_size(3); - _runtime_state = std::make_unique(_query_options, _query_globals); - } - - TFileRangeDesc _build_range_with_table_level_row_count(int64_t row_count) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.table_level_row_count = true; - range.table_format_params.table_level_row_count = row_count; - return range; - } - - TQueryOptions _query_options; - TQueryGlobals _query_globals; - std::unique_ptr _runtime_state; - RuntimeProfile _profile {"paimon_cpp_reader_test"}; - std::vector _file_slot_descs; -}; - -TEST_F(PaimonCppReaderTest, CountPushDownUsesTableLevelRowCount) { - auto range = _build_range_with_table_level_row_count(5); - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - reader.set_push_down_agg_type(TPushAggOp::type::COUNT); - - auto init_status = reader.init_reader(); - ASSERT_TRUE(init_status.ok()) << init_status; - - Block block; - size_t read_rows = 0; - bool eof = false; - - auto first_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(first_status.ok()) << first_status; - EXPECT_EQ(3, read_rows); - EXPECT_FALSE(eof); - - auto second_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(second_status.ok()) << second_status; - EXPECT_EQ(2, read_rows); - EXPECT_TRUE(eof); - - auto third_status = reader.get_next_block(&block, &read_rows, &eof); - ASSERT_TRUE(third_status.ok()) << third_status; - EXPECT_EQ(0, read_rows); - EXPECT_TRUE(eof); -} - -TEST_F(PaimonCppReaderTest, InitReaderFailsWithoutPaimonSplit) { - TFileRangeDesc range; - range.__isset.table_format_params = true; - range.table_format_params.__isset.paimon_params = true; - range.table_format_params.paimon_params.__isset.paimon_table = true; - range.table_format_params.paimon_params.paimon_table = "s3://bucket/db.tbl"; - - PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr); - auto status = reader.init_reader(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("missing paimon_split"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, DecodeValidBuffer) { - // Scenario: a valid Paimon DV stays compressed after decoding instead of becoming one int64_t - // per deleted row in the query cache. - const auto buffer = build_paimon_deletion_vector_buffer({0, 3, 5}); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(deletion_vector.cardinality(), 3); - EXPECT_TRUE(deletion_vector.contains(uint64_t {0})); - EXPECT_TRUE(deletion_vector.contains(uint64_t {3})); - EXPECT_TRUE(deletion_vector.contains(uint64_t {5})); - EXPECT_FALSE(deletion_vector.contains(uint64_t {4})); -} - -TEST(PaimonDeletionVectorTest, RejectShortBuffer) { - // Scenario: malformed DV content must fail before reading the length and magic fields. - const std::vector buffer(7, '\0'); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("file size too small"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectNullBuffer) { - DeletionVector deletion_vector; - const auto status = decode_paimon_deletion_vector_buffer(nullptr, 8, &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("blob is null"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectLengthMismatch) { - // Scenario: the big-endian length prefix protects against using a truncated or over-read DV - // slice from a shared deletion-vector file. - auto buffer = build_paimon_deletion_vector_buffer({1}); - BigEndian::Store32(buffer.data(), static_cast(buffer.size())); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("length mismatch"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectMagicMismatch) { - // Scenario: Paimon DV buffers have a fixed magic header, so a cache entry or offset pointing - // to unrelated bytes must be rejected. - auto buffer = build_paimon_deletion_vector_buffer({1}); - buffer[4] = '\0'; - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, RejectCorruptRoaringBitmap) { - // Scenario: a buffer with a valid header but incomplete Roaring payload should surface as a - // data-quality error instead of silently producing partial delete rows. - auto buffer = build_paimon_deletion_vector_buffer({1, 2}); - buffer.resize(10); - BigEndian::Store32(buffer.data(), static_cast(buffer.size() - 4)); - DeletionVector deletion_vector; - const auto status = - decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("failed to deserialize roaring bitmap"), std::string::npos); -} - -TEST(PaimonDeletionVectorTest, CacheKeyIncludesOffsetAndLength) { - // Scenario: different Paimon tables or splits may reference the same DV file with different - // ranges; cache keys must include both offset and length to avoid sharing the wrong bitmap. - TPaimonDeletionFileDesc first_deletion_file; - first_deletion_file.__set_path("s3://bucket/table/deletion.dv"); - first_deletion_file.__set_offset(128); - first_deletion_file.__set_length(64); - - TPaimonDeletionFileDesc different_offset = first_deletion_file; - different_offset.__set_offset(256); - - TPaimonDeletionFileDesc different_length = first_deletion_file; - different_length.__set_length(96); - - const auto first_key = build_paimon_deletion_vector_cache_key(first_deletion_file); - EXPECT_NE(first_key, build_paimon_deletion_vector_cache_key(different_offset)); - EXPECT_NE(first_key, build_paimon_deletion_vector_cache_key(different_length)); -} - -TEST(PaimonDeletionVectorTest, ValidateDescriptorRejectsInvalidRange) { - size_t bytes_read = 0; - - TPaimonDeletionFileDesc missing_path; - missing_path.__set_offset(0); - missing_path.__set_length(4); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(missing_path, bytes_read).ok()); - - TPaimonDeletionFileDesc deletion_file; - deletion_file.__set_path("dv.bin"); - deletion_file.__set_offset(-1); - deletion_file.__set_length(4); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_offset(0); - deletion_file.__set_length(-1); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_length(std::numeric_limits::max()); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 4); - EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - EXPECT_EQ(static_cast(MAX_PAIMON_DELETION_VECTOR_BYTES), bytes_read); - - deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 3); - EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - - deletion_file.__set_offset(3); - deletion_file.__set_length(4); - EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); - EXPECT_EQ(bytes_read, 8); -} - -TEST(PaimonDeletionVectorTest, DecodedCacheReportsHitSeparatelyFromFileCache) { - // The decoded cache lookup result is reported by ShardedKVCache itself. The creator represents - // the lower File Cache/read/decode path and must only run for the miss. - ShardedKVCache cache(1); - int create_count = 0; - bool cache_hit = true; - auto create = [&]() { - ++create_count; - auto* deletion_vector = new DeletionVector(); - deletion_vector->add(uint64_t {7}); - return deletion_vector; - }; - - const auto* first = cache.get("dv", create, &cache_hit); - EXPECT_FALSE(cache_hit); - ASSERT_NE(first, nullptr); - EXPECT_TRUE(first->contains(uint64_t {7})); - - const auto* second = cache.get("dv", create, &cache_hit); - EXPECT_TRUE(cache_hit); - EXPECT_EQ(first, second); - EXPECT_EQ(create_count, 1); -} - -TEST(PaimonDeletionVectorTest, V1ParquetReaderReadErrorReleasesCacheEntry) { - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("paimon_v1_parquet_dv_test"); - auto scan_params = make_local_paimon_scan_params(TFileFormatType::FORMAT_PARQUET); - const auto range = make_paimon_range_with_deletion_file( - "./be/test/exec/test_data/missing_paimon_v1_delete_vector.bin"); - cctz::time_zone ctz; - TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); - io::IOContext io_ctx; - FileMetaCache meta_cache(1024); - ShardedKVCache kv_cache(8); - - PaimonParquetReader reader(&profile, scan_params, range, 1024, &ctz, &kv_cache, &io_ctx, - &runtime_state, &meta_cache); - const auto status = reader.TEST_init_deletion_vector(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(range.table_format_params.paimon_params.deletion_file.path), - std::string::npos); -} - -TEST(PaimonDeletionVectorTest, V1OrcReaderReadErrorReleasesCacheEntry) { - RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("paimon_v1_orc_dv_test"); - auto scan_params = make_local_paimon_scan_params(TFileFormatType::FORMAT_ORC); - auto range = make_paimon_range_with_deletion_file( - "./be/test/exec/test_data/missing_paimon_v1_orc_delete_vector.bin"); - range.table_format_params.paimon_params.__set_file_format("orc"); - io::IOContext io_ctx; - FileMetaCache meta_cache(1024); - ShardedKVCache kv_cache(8); - - PaimonOrcReader reader(&profile, &runtime_state, scan_params, range, 1024, - TimezoneUtils::default_time_zone, &kv_cache, &io_ctx, &meta_cache); - const auto status = reader.TEST_init_deletion_vector(); - - ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(range.table_format_params.paimon_params.deletion_file.path), - std::string::npos); -} - -} // namespace doris diff --git a/be/test/format/table/paimon_doris_file_system_test.cpp b/be/test/format/table/paimon_doris_file_system_test.cpp deleted file mode 100644 index a2032dff8188c9..00000000000000 --- a/be/test/format/table/paimon_doris_file_system_test.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format/table/paimon_doris_file_system.h" - -#include - -#include - -#include "common/config.h" - -namespace doris { - -class PaimonDorisFileSystemTest : public testing::Test { -protected: - void SetUp() override { saved_mappings_ = config::paimon_file_system_scheme_mappings; } - - void TearDown() override { config::paimon_file_system_scheme_mappings = saved_mappings_; } - - std::vector saved_mappings_; -}; - -TEST_F(PaimonDorisFileSystemTest, UsesDefaultSchemeMappings) { - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("jfs")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("s3a")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("https")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("ofs")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("unknown")); -} - -TEST_F(PaimonDorisFileSystemTest, AllowsOverridingSchemeMappingsFromConfig) { - config::paimon_file_system_scheme_mappings = {"file=local", "jfs = s3", "gs = hdfs", - "custom-http = http", "custom-broker = broker"}; - - EXPECT_EQ(TFileType::FILE_LOCAL, paimon::map_scheme_to_file_type("file")); - EXPECT_EQ(TFileType::FILE_S3, paimon::map_scheme_to_file_type("JFS")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("gs")); - EXPECT_EQ(TFileType::FILE_HTTP, paimon::map_scheme_to_file_type("custom-http")); - EXPECT_EQ(TFileType::FILE_BROKER, paimon::map_scheme_to_file_type("custom-broker")); - EXPECT_EQ(TFileType::FILE_HDFS, paimon::map_scheme_to_file_type("still-unknown")); -} - -} // namespace doris diff --git a/build.sh b/build.sh index b5b531ce707e86..8d89dc9eaee3c6 100755 --- a/build.sh +++ b/build.sh @@ -486,59 +486,16 @@ else LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a' fi -# The final-library sentinel only proves that some third-party build completed. It cannot -# distinguish an older prebuilt whose Arrow/Paimon closure predates the selected sources. -# shellcheck source=thirdparty/arrow-paimon-vars.sh -. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh" -NEED_ARROW_PAIMON_THIRDPARTY=false -if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 || - "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" || - "${BUILD_INDEX_TOOL}" == "ON" ]]; then - NEED_ARROW_PAIMON_THIRDPARTY=true -fi - -rebuild_thirdparty_libraries() { - local remove_installed="$1" - shift - local build_script="${DORIS_THIRDPARTY}/build-thirdparty.sh" - local build_args=(-j "${PARALLEL}") - local selected_thirdparty_root - local checkout_thirdparty_root - - if [[ ! -f "${build_script}" ]]; then - echo "Cannot rebuild thirdparty libraries: ${build_script} is missing." >&2 - echo "DORIS_THIRDPARTY=${DORIS_THIRDPARTY} is an install-only or incomplete prefix. Use a matching compilation image/prebuilt, or unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree." >&2 - exit 1 - fi - selected_thirdparty_root="$(cd "${DORIS_THIRDPARTY}" && pwd -P)" - checkout_thirdparty_root="$(cd "${DORIS_HOME}/thirdparty" && pwd -P)" - if [[ "${selected_thirdparty_root}" != "${checkout_thirdparty_root}" ]]; then - echo "Cannot rebuild thirdparty libraries with an external source tree: ${selected_thirdparty_root}." >&2 - echo "Unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree, then use the resulting version-matched installation." >&2 - exit 1 - fi - build_script="${checkout_thirdparty_root}/build-thirdparty.sh" - if [[ "${remove_installed}" == "true" ]]; then - # Some libraries, such as lz4, fail when an earlier installation remains. - rm -rf "${DORIS_THIRDPARTY}/installed" - fi - if [[ "${CLEAN}" -eq 1 ]]; then - build_args+=(--clean) - fi - bash "${build_script}" "${build_args[@]}" "$@" - if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 - exit 1 - fi -} - if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." - rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && - ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" + # need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists + rm -rf "${DORIS_THIRDPARTY}/installed" + + if [[ "${CLEAN}" -eq 0 ]]; then + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" + else + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean + fi fi update_submodule() { diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 30c109dfb2fa14..87cc46c9b2f5a1 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -167,10 +167,6 @@ public class PaimonScanPlanProvider implements ConnectorScanPlanProvider { // PaimonScanNode.getSplits gate, sessionVariable.isForceJniScanner()), bypassing the native ORC/Parquet // readers to dodge native-reader bugs. Default false (legacy default). // - // NOTE: enable_paimon_cpp_reader is deliberately NOT read here. Upstream #66008 removed the paimon-cpp - // arm from PaimonScanNode.setPaimonParams (file-scanner-v2 has no split-aware paimon-cpp adapter and - // hard-rejects a PAIMON_CPP range), so the flag no longer influences planning — see - // PaimonScanRange.populateRangeParams. private static final String FORCE_JNI_SCANNER = "force_jni_scanner"; // Session variable name (byte-identical to SessionVariable.IGNORE_SPLIT_TYPE) surfaced through the same @@ -642,8 +638,8 @@ private List planScanInternal( // FIX-L14: honor the ignore_split_type debugging escape hatch (legacy PaimonScanNode.getSplits): // IGNORE_JNI drops JNI splits (nonDataSplit + DataSplit-JNI arms), IGNORE_NATIVE drops native splits. - // The COUNT(*) arm is never dropped (legacy parity); IGNORE_PAIMON_CPP stays a no-op (legacy getSplits - // never consulted it). Read once here so discarded JNI splits bypass carrier compatibility checks. + // The COUNT(*) arm is never dropped. The deprecated IGNORE_PAIMON_CPP compatibility value stays + // a no-op. Read once here so discarded JNI splits bypass carrier compatibility checks. String ignoreSplitType = resolveIgnoreSplitType(session); boolean ignoreJni = IGNORE_SPLIT_TYPE_JNI.equals(ignoreSplitType); boolean ignoreNative = IGNORE_SPLIT_TYPE_NATIVE.equals(ignoreSplitType); @@ -1433,11 +1429,8 @@ private PaimonScanRange buildJniScanRange(Split split, String defaultFileFormat, String serializedSplit = encodeSplit(split); - // FIX-JNI-FILE-FORMAT (P7-1) + FIX-L11: emit the real data-file format (orc/parquet/avro), NOT "jni". - // JNI routing is gated by the paimon.split property (PaimonScanRange.populateRangeParams), so this - // string only feeds fileDesc.file_format, which BE's paimon_cpp_reader backfills into - // FILE_FORMAT/MANIFEST_FORMAT (an invalid "jni" breaks the manifest read). Mirrors legacy - // PaimonScanNode.setPaimonParams's fileDesc.setFileFormat(getFileFormat(getPathString())): for a + // Emit the real data-file format (orc/parquet/avro), not the reader transport name "jni". + // JNI routing is gated by the paimon.split property. For a // DataSplit the format is the FIRST data-file suffix (falling back to the table default); a // non-DataSplit has no data file and falls back to the table default (legacy DUMMY_PATH -> orElse). String fileFormat = isDataSplit @@ -1883,7 +1876,7 @@ Map getBackendPaimonOptions() { if (!PaimonCatalogProperties.JDBC.equals(catalogProps.getFlavor())) { return options; } - // Forward relevant JDBC catalog properties for BE's paimon-cpp reader + // Forward relevant JDBC catalog properties for the BE Paimon JNI reader. for (Map.Entry entry : properties.entrySet()) { String key = entry.getKey(); if (key.startsWith("jdbc.") || key.equals("warehouse") @@ -2375,12 +2368,9 @@ private static TField buildField(DataType dataType) { } /** - * Serializes a paimon {@link Split} for the BE JNI reader: ALWAYS Java object serialization, which is - * what BE's PaimonJniScanner deserializes. Mirrors upstream {@code PaimonScanNode.setPaimonParams} + - * {@code PaimonUtil.encodeObjectToString} after #66008 removed the paimon-cpp arm — a logical - * {@link DataSplit} may span several files, and file-scanner-v2 has no split-aware paimon-cpp adapter, - * so the native-binary ({@code DataSplit.serialize} / {@code paimon::Split::Deserialize}) encoding is - * never emitted and {@code enable_paimon_cpp_reader} no longer influences the wire format. + * Serializes a paimon {@link Split} for the BE JNI reader using Java object serialization, which is + * what {@code PaimonJniScanner} deserializes. The native-binary + * ({@code DataSplit.serialize}) encoding is not part of this wire path. */ static String encodeSplit(Split split) { return encodeObjectToString(split); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java index 3bc3cbe78cf4c1..220e8ada42e7e3 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java @@ -209,13 +209,7 @@ public void populateRangeParams(TTableFormatFileDesc formatDesc, if (paimonSplitVal != null) { // JNI reader path rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI); - // FIX-READER-TYPE (3645dc94306): tell BE's file-scanner-v2 which paimon reader stack to use. - // ALWAYS the Java JNI reader (upstream #66008 removed the paimon-cpp arm from - // PaimonScanNode.setPaimonParams): a logical DataSplit may span several files, and - // file-scanner-v2 has no split-aware paimon-cpp adapter, so it HARD-REJECTS a PAIMON_CPP range - // ("FileScannerV2 does not support table format paimon", file_scanner_v2.cpp - // is_supported_jni_table_format -> _validate_scan_range) with no per-range V1 fallback. - // enable_paimon_cpp_reader is therefore a no-op on the plan path, exactly like on master. + // A serialized logical split is always consumed by the Java JNI reader. fileDesc.setReaderType(TPaimonReaderType.PAIMON_JNI); fileDesc.setPaimonSplit(paimonSplitVal); String weightStr = props.get("paimon.self_split_weight"); @@ -297,10 +291,7 @@ public static class Builder { private long start; private long length = -1; private long fileSize = -1; - // Every production caller sets fileFormat explicitly (the real orc/parquet). Default empty (NOT - // "jni", an invalid paimon format): BE's paimon_cpp_reader skips its FILE_FORMAT/MANIFEST_FORMAT - // backfill when this is empty (guarded !file_format.empty()), so a missing set can never inject an - // invalid format (FIX-JNI-FILE-FORMAT). + // Every production caller sets fileFormat explicitly to the real data-file format. private String fileFormat = ""; private Map partitionValues; private long selfSplitWeight; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index e66a38c021d243..fe98a5d9c4f750 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -34,11 +34,8 @@ import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.StorageKind; import org.apache.doris.filesystem.properties.StorageProperties; -import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; -import org.apache.doris.thrift.TPaimonReaderType; import org.apache.doris.thrift.TPrimitiveType; -import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.schema.external.TField; import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TSchema; @@ -1435,7 +1432,7 @@ private static DataSplit buildRealDataSplit(Path warehouse) throws Exception { } } - /** The paimon-cpp NATIVE binary split encoding (DataSplit.serialize + Base64) — what FE must NEVER emit. */ + /** The native binary split encoding (DataSplit.serialize + Base64), which is not used by JNI. */ private static String nativeBinaryEncode(DataSplit dataSplit) throws Exception { java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); dataSplit.serialize(new org.apache.paimon.io.DataOutputViewStreamWrapper(baos)); @@ -1446,21 +1443,14 @@ private static String nativeBinaryEncode(DataSplit dataSplit) throws Exception { public void encodeSplitAlwaysUsesJavaSerializationForDataSplit(@TempDir Path warehouse) throws Exception { DataSplit dataSplit = buildRealDataSplit(warehouse); - // WHY: upstream #66008 removed the paimon-cpp arm from PaimonScanNode.setPaimonParams, so the ONLY - // split wire format FE emits is Java object serialization (what BE's PaimonJniScanner deserializes). - // Emitting the native binary format would now be fatal, not just different: file-scanner-v2 (default - // ON) has no split-aware paimon-cpp adapter, so is_supported_jni_table_format rejects a PAIMON_CPP - // range and _validate_scan_range fails the query — there is no per-range V1 fallback. - // MUTATION: re-adding a cpp/native-binary branch -> the wire stops matching the Java encoding and - // starts matching the native one -> both assertions red. + // The JNI split wire format is Java object serialization, which PaimonJniScanner deserializes. String wire = PaimonScanPlanProvider.encodeSplit(dataSplit); Assertions.assertEquals(feJavaEncode(dataSplit), wire, "a DataSplit must be Java-object-serialized byte-for-byte (the Java JNI reader's format)"); Assertions.assertNotEquals(nativeBinaryEncode(dataSplit), wire, - "FE must never emit the paimon-cpp native binary split format (file-scanner-v2 rejects it)"); + "FE must not emit the native binary split format for a JNI range"); - // Sanity-check the negative reference really is the paimon-cpp format (else the assertion above - // would pass vacuously): it decodes back to an equal DataSplit via paimon's native deserializer. + // Sanity-check that the negative reference decodes through Paimon's native deserializer. byte[] nativeBytes = Base64.getDecoder().decode( nativeBinaryEncode(dataSplit).getBytes(StandardCharsets.UTF_8)); Assertions.assertEquals(dataSplit, DataSplit.deserialize( @@ -1605,10 +1595,8 @@ public void countPushdownCollapsesMultipleSplitsToOneRangeBearingSummedTotal( @Test public void jniAndCountRangesCarryRealFileFormatNotJni(@TempDir Path warehouse) throws Exception { // FIX-JNI-FILE-FORMAT (P7-1): a JNI-serialized split (the default reader path AND the COUNT(*) - // collapse range) must emit the REAL data-file format in fileDesc.file_format, NOT "jni" — BE's - // paimon_cpp_reader backfills paimon FILE_FORMAT/MANIFEST_FORMAT from it (an invalid "jni" breaks - // the manifest read). JNI routing is gated by the paimon.split property, NOT this string, so the - // real format is safe to emit (legacy PaimonScanNode.setPaimonParams does the same). The table is + // collapse range) must emit the REAL data-file format in fileDesc.file_format, NOT "jni". JNI + // routing is gated by the paimon.split property, NOT this string. The table is // created with explicit file.format=orc so the asserted value is the table option (distinct from // the "parquet" fallback) — proving the real option is read, not a constant. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -1683,8 +1671,8 @@ public void jniAndCountRangesUseFileSuffixNotAlteredTableDefault(@TempDir Path w // file_format from the split's FIRST data-file SUFFIX (legacy PaimonScanNode.getFileFormat(getPathString) // -> dataSplitFileFormat), NOT the table-level file.format option. These DIVERGE for an altered / // mixed-format table: the option is changed to parquet while historical data files remain .orc. HEAD - // regressed to emitting the bare table default, so BE's paimon_cpp_reader would backfill the WRONG - // format for those files. WHY it matters: unlike jniAndCountRangesCarryRealFileFormatNotJni (where the + // regressed to emitting the bare table default, which would describe those files incorrectly. + // WHY it matters: unlike jniAndCountRangesCarryRealFileFormatNotJni (where the // table default == the .orc suffix, so it cannot distinguish default from suffix), this test forces a // mismatch and thus is the one that actually goes RED if the emission points revert to defaultFileFormat. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), @@ -2148,7 +2136,7 @@ public void ignoreNativeDropsNativeSplit(@TempDir Path warehouse) throws Excepti public void ignorePaimonCppIsNoOpParity(@TempDir Path warehouse) throws Exception { // FIX-L14: IGNORE_PAIMON_CPP is a documented ignore_split_type value that legacy // PaimonScanNode.getSplits NEVER consulted, so it stays a no-op (legacy parity) — the scan emits the - // same ranges as NONE. Pins that a future change does not add a half-implemented CPP arm. + // same ranges as NONE. Retain this compatibility behavior while the enum value remains on the wire. try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), new org.apache.paimon.fs.Path(warehouse.toUri()))) { catalog.createDatabase("db", false); @@ -2190,81 +2178,6 @@ public void ignorePaimonCppIsNoOpParity(@TempDir Path warehouse) throws Exceptio } } - @Test - public void cppReaderSessionFlagNoLongerChangesThePlan(@TempDir Path warehouse) throws Exception { - // WHY (upstream #66008): enable_paimon_cpp_reader must be a NO-OP on the plan path. It stays a - // documented (fuzzy=true!) session variable, so the regression fuzzer and the upstream suites - // test_paimon_cpp_reader / test_paimon_partition_*_refs do set it to true — and if the connector - // still answered with PAIMON_CPP, every such query would HARD-FAIL under the default - // enable_file_scanner_v2=true ("FileScannerV2 does not support table format paimon with file format - // FORMAT_JNI": file_scanner_v2.cpp is_supported_jni_table_format -> _validate_scan_range, with no - // per-range fallback to the V1 scanner that still implements PaimonCppReader). - // MUTATION: reinstating a cpp arm keyed off this session flag -> reader_type flips to PAIMON_CPP - // (and paimon_table reappears) -> red. - try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), - new org.apache.paimon.fs.Path(warehouse.toUri()))) { - catalog.createDatabase("db", false); - Identifier id = Identifier.create("db", "t"); - catalog.createTable(id, Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("val", DataTypes.BIGINT()) - .primaryKey("id") - .option("bucket", "1") - .build(), false); - Table table = catalog.getTable(id); - BatchWriteBuilder wb = table.newBatchWriteBuilder(); - try (BatchTableWrite write = wb.newWrite()) { - write.write(GenericRow.of(1, 100L)); - write.write(GenericRow.of(2, 200L)); - List messages = write.prepareCommit(); - try (BatchTableCommit commit = wb.newCommit()) { - commit.commit(messages); - } - } - - RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); - ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); - PaimonTableHandle handle = new PaimonTableHandle( - "db", "t", Collections.emptyList(), Collections.emptyList()); - List noColumns = Collections.emptyList(); - - // force_jni_scanner pins every split onto the JNI arm (the only arm the cpp flag ever touched). - Map cppOn = new HashMap<>(); - cppOn.put("force_jni_scanner", "true"); - cppOn.put("enable_paimon_cpp_reader", "true"); - Map cppOff = new HashMap<>(); - cppOff.put("force_jni_scanner", "true"); - cppOff.put("enable_paimon_cpp_reader", "false"); - - List onRanges = provider.planScan(sessionWithProps(cppOn), - ConnectorScanRequest.builder(handle, noColumns) - .build()); - List offRanges = provider.planScan(sessionWithProps(cppOff), - ConnectorScanRequest.builder(handle, noColumns) - .build()); - Assertions.assertFalse(onRanges.isEmpty(), "baseline scan must emit >=1 JNI range"); - Assertions.assertEquals(offRanges.size(), onRanges.size(), - "the cpp flag must not change the emitted range count"); - - for (int i = 0; i < onRanges.size(); i++) { - TTableFormatFileDesc onDesc = new TTableFormatFileDesc(); - onRanges.get(i).populateRangeParams(onDesc, new TFileRangeDesc()); - TTableFormatFileDesc offDesc = new TTableFormatFileDesc(); - offRanges.get(i).populateRangeParams(offDesc, new TFileRangeDesc()); - - Assertions.assertEquals(TPaimonReaderType.PAIMON_JNI, - onDesc.getPaimonParams().getReaderType(), - "reader_type must stay PAIMON_JNI even with enable_paimon_cpp_reader=true"); - Assertions.assertFalse(onDesc.getPaimonParams().isSetPaimonTable(), - "paimon_table is cpp-reader-only state and must no longer be shipped"); - Assertions.assertEquals(offDesc.getPaimonParams().getPaimonSplit(), - onDesc.getPaimonParams().getPaimonSplit(), - "the split wire format must be identical with the cpp flag on and off"); - } - } - } - // ---- FIX-NATIVE-SUBSPLIT (M-3) ---- private static final long MB = 1024L * 1024L; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java index 67bbfcfe605fc0..ba41ec4c0c6653 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeReaderTypeTest.java @@ -37,14 +37,7 @@ * SPI migration to {@code PaimonScanRange} dropped it (the thrift {@code TPaimonFileDesc} was built without * reader_type), so BE could not tell which paimon reader stack a split wanted. * - *

There is deliberately NO {@link TPaimonReaderType#PAIMON_CPP} arm: upstream #66008 removed it from - * {@code PaimonScanNode.setPaimonParams} because a logical {@code DataSplit} may span several files and - * file-scanner-v2 has no split-aware paimon-cpp adapter. Under the default {@code enable_file_scanner_v2 - * = true}, a PAIMON_CPP range is HARD-REJECTED ({@code is_supported_jni_table_format} → - * {@code _validate_scan_range} → "FileScannerV2 does not support table format paimon") with no - * per-range fallback to the V1 scanner that still implements {@code PaimonCppReader}. So the JNI arm must - * answer PAIMON_JNI unconditionally, and {@code enable_paimon_cpp_reader} is a plan-path no-op - * (see {@code PaimonScanPlanProviderTest.cppReaderSessionFlagNoLongerChangesThePlan}). + *

A serialized logical split is always assigned to the Java JNI reader. */ public class PaimonScanRangeReaderTypeTest { @@ -62,18 +55,15 @@ public void jniSplitSetsReaderTypeJniAndNoPaimonTable() { .paimonSplit("java-serialized-split") // JNI marker (paimon.split prop present) .build(); - // MUTATION: dropping setReaderType, or reinstating a cpp arm, turns this red — with reader_type - // absent BE's V2 paimon reader can still infer JNI from paimon_split, but a PAIMON_CPP answer - // fails the query outright (see the class javadoc). + // Pin the explicit reader type so BE does not need to infer it from paimon_split. TTableFormatFileDesc formatDesc = populate(range); Assertions.assertTrue(formatDesc.getPaimonParams().isSetReaderType(), "a JNI split must set reader_type so BE can pick the reader stack"); Assertions.assertEquals(TPaimonReaderType.PAIMON_JNI, formatDesc.getPaimonParams().getReaderType()); - // paimon_table (the table root path) is read ONLY by the V1 PaimonCppReader, so #66008 stopped - // shipping it. MUTATION: re-adding setPaimonTable -> red. + // paimon_table is a deprecated compatibility field and is not needed by the JNI reader. Assertions.assertFalse(formatDesc.getPaimonParams().isSetPaimonTable(), - "paimon_table is cpp-reader-only state and must not be shipped"); + "paimon_table must not be shipped to the JNI reader"); } @Test diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c5b503e9f144b0..cbfcbb93f0e41d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -809,8 +809,6 @@ public String toString() { public static final String FORCE_JNI_SCANNER = "force_jni_scanner"; - public static final String ENABLE_PAIMON_CPP_READER = "enable_paimon_cpp_reader"; - public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table"; @@ -1473,6 +1471,7 @@ public enum IgnoreSplitType { NONE, IGNORE_JNI, IGNORE_NATIVE, + // Deprecated compatibility value. It behaves like NONE because no C++ splits are emitted. IGNORE_PAIMON_CPP } @@ -2927,11 +2926,6 @@ public Map getForceEagerAggHintMap() { description = "Force the use of jni mode to read external table") private boolean forceJniScanner = false; - @VarAttrDef.VarAttr(name = ENABLE_PAIMON_CPP_READER, - fuzzy = true, - description = "Use paimon-cpp for non-native Paimon reads") - private boolean enablePaimonCppReader = false; - @VarAttrDef.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = "enable count(*) pushdown optimization for external table") @@ -3806,8 +3800,6 @@ private void setFuzzyForCatalog(Random random) { // jni this.forceJniScanner = random.nextBoolean(); - this.enablePaimonCppReader = random.nextBoolean(); - // statistics this.fetchHiveRowCountSync = random.nextBoolean(); @@ -5596,7 +5588,6 @@ public TQueryOptions toThrift() { tResult.setEnableParquetFilePageCache(enableParquetFilePageCache); tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax); tResult.setEnableExprZonemapFilter(enableExprZonemapFilter); - tResult.setEnablePaimonCppReader(enablePaimonCppReader); tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds); tResult.setEmbedMaxBatchSize(embedMaxBatchSize); tResult.setAiContextWindowSize(aiContextWindowSize); @@ -6413,10 +6404,6 @@ public boolean isForceJniScanner() { return forceJniScanner; } - public boolean isEnablePaimonCppReader() { - return enablePaimonCppReader; - } - public String getIgnoreSplitType() { return ignoreSplitType; } @@ -6438,10 +6425,6 @@ public void setForceJniScanner(boolean force) { forceJniScanner = force; } - public void setEnablePaimonCppReader(boolean enable) { - enablePaimonCppReader = enable; - } - public boolean isEnableCountPushDownForExternalTable() { return enableCountPushDownForExternalTable; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 07944139285228..afbfe22b02faec 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -439,7 +439,7 @@ struct TQueryOptions { 195: optional bool enable_left_semi_direct_return_opt; 200: optional bool enable_adjust_conjunct_order_by_cost; - // Use paimon-cpp to read Paimon splits on BE + // Deprecated: the paimon-cpp reader has been removed. Retained for wire compatibility. 201: optional bool enable_paimon_cpp_reader = false; // Whether all fragments of this query are assigned to a single backend. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index af4b9c3eb68619..3fa0e56aa4bf14 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -367,6 +367,7 @@ struct TPaimonDeletionFileDesc { enum TPaimonReaderType { PAIMON_NATIVE = 0, PAIMON_JNI = 1, + // Deprecated wire value kept during rolling upgrades. New plans never emit it. PAIMON_CPP = 2, } diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index b866b5d82a42e6..c5d00a47acd4c0 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -451,7 +451,6 @@ _thirdparty_change_affects_artifacts() { if [[ "${af}" == 'thirdparty/test/'* ]] || [[ "${af}" == 'thirdparty/CHANGELOG.md' ]] || [[ "${af}" == 'thirdparty/LICENSE.txt' ]] || - [[ "${af}" == 'thirdparty/arrow-paimon-vars.sh' ]] || [[ "${af}" == 'thirdparty/build-thirdparty.sh' ]]; then return 1 fi diff --git a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md index 5a551b6d3d8e34..4d2f3ae0529c4f 100644 --- a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md @@ -70,7 +70,7 @@ Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contrac | Paimon | Append table, partitioned table, primitive and nested types | Covered | `test_paimon_catalog`, `test_paimon_partition_table`, `test_paimon_full_schema_change` | | Paimon | Primary-key deduplicate, partial-update, aggregation and first-row | Covered | `test_paimon_merge_engine_matrix` | | Paimon | Fixed bucket, dynamic bucket and cross-partition update | Covered | `test_paimon_merge_engine_matrix`, `test_paimon_partition_pk_delete_refs` | -| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_cpp_reader` | +| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_reader_parity` | | Paimon | Snapshot/timestamp/tag/branch and incremental modes | Covered | `paimon_time_travel`, `paimon_incr_read`, `test_paimon_schema_time_travel_matrix` | | Paimon | Schema evolution, partition-key restrictions and historical schema binding | Covered | `test_paimon_schema_time_travel_matrix`, `test_paimon_partition_mutation_atomicity` | | Paimon | Deletion vectors, upsert/delete visibility and data/system tables | Covered | `test_paimon_deletion_vector`, `paimon_data_system_table`, `paimon_system_table` | diff --git a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy index e9f07efdf64d6b..309ad6a48a2c61 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy @@ -138,7 +138,6 @@ suite("paimon_data_system_table", "p0,external") { } sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" // Paimon data system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection @@ -183,7 +182,6 @@ suite("paimon_data_system_table", "p0,external") { qt_jni_native_binlog_rows """select rowkind, id[1], id[2], name[1], name[2] from ${nativeTableName}\$binlog order by id[1]""" qt_jni_native_audit_log_rows """select rowkind, id, name from ${nativeTableName}\$audit_log order by id""" } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" } } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy index 85007d147ba2d0..f8c599804f13f8 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_pk_delete_refs.groovy @@ -321,14 +321,12 @@ suite("test_paimon_partition_pk_delete_refs", where part in ('p1', 'p2') order by part, id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current DV") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_dv"(currentReaderQuery) "qt_${actionSuffix}_jni_historical"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current DV") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_dv"(currentReaderQuery) @@ -336,7 +334,6 @@ suite("test_paimon_partition_pk_delete_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy index 979fbb7784c798..e2b7ffeae99cd4 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_partition_schema_filter_refs.groovy @@ -341,14 +341,12 @@ suite("test_paimon_partition_schema_filter_refs", where part in ('p1', 'p2') order by id """ - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" assertJniPath(currentReaderQuery, "${tableName} current") assertJniPath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_jni_current_complex"(currentReaderQuery) "qt_${actionSuffix}_jni_historical_complex"(historicalReaderQuery) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" assertNativePath(currentReaderQuery, "${tableName} current") assertNativePath(historicalReaderQuery, "${tableName} historical") "qt_${actionSuffix}_native_current_complex"(currentReaderQuery) @@ -356,7 +354,6 @@ suite("test_paimon_partition_schema_filter_refs", assertEquals(finalSnapshot, latestSnapshotId(tableName)) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """set enable_runtime_filter_prune=true""" sql """set enable_runtime_filter_partition_prune=true""" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy similarity index 81% rename from regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy rename to regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy index 64ef323add6119..4172c8e805b075 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_reader_parity.groovy @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -suite("test_paimon_cpp_reader", "p0,external") { +suite("test_paimon_reader_parity", "p0,external") { String enabled = context.config.otherConfigs.get("enablePaimonTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disabled paimon test") return } - String catalogName = "test_paimon_cpp_reader" + String catalogName = "test_paimon_reader_parity" String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -35,9 +35,6 @@ suite("test_paimon_cpp_reader", "p0,external") { );""" sql """switch ${catalogName}""" sql """use db1""" - // Do not force JNI; keep default selection behavior. - sql """set force_jni_scanner=false""" - def testQueries = [ """select c1 from complex_all order by c1""", """select c1 from complex_all where c1 >= 2 order by c1""", @@ -51,19 +48,17 @@ suite("test_paimon_cpp_reader", "p0,external") { """select * from deletion_vector_parquet""" ] - // Default path is JNI when enable_paimon_cpp_reader=false. - sql """set enable_paimon_cpp_reader=false""" + sql """set force_jni_scanner=true""" def jniResults = testQueries.collect { query -> sql(query) } - sql """set enable_paimon_cpp_reader=true""" - def cppResults = testQueries.collect { query -> sql(query) } + sql """set force_jni_scanner=false""" + def nativeResults = testQueries.collect { query -> sql(query) } - assertTrue(cppResults[0].size() > 0) + assertTrue(nativeResults[0].size() > 0) for (int i = 0; i < testQueries.size(); i++) { - assertEquals(jniResults[i].toString(), cppResults[i].toString()) + assertEquals(jniResults[i].toString(), nativeResults[i].toString()) } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index c406b422928179..b13b747e85e3f5 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -367,9 +367,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { sql """refresh catalog ${catalogName}""" // row_tracking exposes Paimon-generated hidden columns, so its incremental scan must - // bypass the C++ reader even when the session otherwise enables native Paimon scans. + // use Paimon-side JNI semantics. sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" String rowTrackingExplain = sql(""" explain verbose select id, name, _ROW_ID, _SEQUENCE_NUMBER @@ -386,8 +385,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { @incr('startSnapshotId'=0, 'endSnapshotId'=1) order by id """ - sql """set enable_paimon_cpp_reader=false""" - // Scenario TC01: validate latest schema/data, explicit new binding, predicate and aggregate. assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], sql("""select id, victim from ${topTable} order by id""")) @@ -571,9 +568,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { // Scenario T14: incremental reads crossing a rename checkpoint bind the end schema. List> incrementalJni - List> incrementalCpp - sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=false""" + List> incrementalAutomatic + sql """set force_jni_scanner=true""" incrementalJni = sql(""" select id, full_name, score from ${pkTable}@incr( @@ -582,8 +578,8 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - sql """set enable_paimon_cpp_reader=true""" - incrementalCpp = sql(""" + sql """set force_jni_scanner=false""" + incrementalAutomatic = sql(""" select id, full_name, score from ${pkTable}@incr( 'startSnapshotId'='${pkCp0}', @@ -591,7 +587,7 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { ) order by id """) - assertEquals(incrementalJni, incrementalCpp) + assertEquals(incrementalJni, incrementalAutomatic) // Scenario TC03/S16: partition pruning and renamed payloads bind to their own snapshots. assertEquals([[1, "p1", "old"], [2, "p2", "old"]], @@ -691,7 +687,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { """)) sql """switch ${catalogName}""" sql """use ${dbName}""" - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=true""" List> forcedJniRows = sql(""" select id, old_name, victim, metric @@ -699,21 +694,19 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { order by id """) sql """set force_jni_scanner=false""" - sql """set enable_paimon_cpp_reader=true""" - List> cppRows = sql(""" + List> automaticRows = sql(""" select id, old_name, victim, metric from ${topTable} for version as of ${topCp0} order by id """) - assertEquals(forcedJniRows, cppRows) - // Schema-selecting OPTIONS must bypass paimon-cpp, whose table handle always uses the - // latest schema, even when native Paimon scans are enabled for the session. - List> cppOptionsRows = sql(""" + assertEquals(forcedJniRows, automaticRows) + // Schema-selecting OPTIONS retain the requested historical schema under automatic routing. + List> automaticOptionsRows = sql(""" select id, old_name, victim, metric from ${topTable}@options('scan.snapshot-id'='${topCp0}') order by id """) - assertEquals(forcedJniRows, cppOptionsRows) + assertEquals(forcedJniRows, automaticOptionsRows) // Scenario T10/T11: retained tags survive expiration; missing refs never fall back to latest. spark_paimon """ @@ -741,7 +734,6 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { exception "tag" } } finally { - sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" sql """drop database if exists internal.${viewDb} force""" sql """drop catalog if exists ${catalogName}""" diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh deleted file mode 100644 index d532a740e63d68..00000000000000 --- a/thirdparty/arrow-paimon-vars.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/bin/bash -# shellcheck disable=2034 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can -# distinguish this stack from unrelated thirdparty changes. - -# arrow -ARROW_VERSION="24.0.0" -ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" -ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" -ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" - -# Arrow bundled dependencies -BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" -BROTLI_NAME="brotli-1.0.9.tar.gz" -BROTLI_SOURCE="brotli-1.0.9" -BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" - -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" -XSIMD_NAME="14.0.0.tar.gz" -XSIMD_SOURCE=xsimd-14.0.0 -XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" - -# paimon-cpp -PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" -PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" - -# Bump the corresponding schema version whenever output-affecting build options or -# helper behavior in build_arrow() or build_paimon_cpp() changes. The fingerprints -# below intentionally describe only this component stack so master and release -# branches can reuse the same shared prebuilt when their semantic inputs match. -ARROW_BUILD_SCHEMA_VERSION="1" -PAIMON_BUILD_SCHEMA_VERSION="1" - -# The shared automation prebuilt is still published from master, which computes the -# former whole-script fingerprint over env.sh, vars.sh, download-thirdparty.sh and -# build-thirdparty.sh. Every republication triggered by an unrelated thirdparty edit -# therefore stamps a different marker, so list each one still in circulation, newest -# first. These markers are only consulted while the selected inputs still hash to the -# pinned semantic fingerprint, and version plus complete artifact validation stay -# mandatory before reuse. Delete the lists once master has published a prebuilt that -# carries the semantic marker itself. -ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" -# Dependency ordering does not change the installed Paimon artifacts, so existing -# complete prebuilts remain compatible with the corrected clean-build graph. -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c" -ARROW_LEGACY_BUILD_FINGERPRINTS=( - # published 2026-08-19, master after apache/doris#66842 - 8356a3a464b93f3a59a6bdae6a51258947ab35d2 - # published 2026-08-05, master after apache/doris#66221 - 9d03645dd1cded5184a8126f5c7f4a6eb9b92b53 -) -PAIMON_LEGACY_BUILD_FINGERPRINTS=( - # published 2026-08-19, master after apache/doris#66842 - 2bbf52e719bdbc8aaa428caab200ac13848f92e5 - # published 2026-08-05, master after apache/doris#66221 - dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28 -) - -# Arrow consumes xsimd and Brotli as bundled source archives, but neither is a -# build target in the focused Arrow/Paimon recovery path. -ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) -ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) -ARROW_PAIMON_DOWNLOAD_PACKAGES=() - -prepare_arrow_paimon_download_packages() { - ARROW_PAIMON_DOWNLOAD_PACKAGES=("$@") - - local package - local source_package - local arrow_requested=false - local source_requested - for package in "$@"; do - if [[ "${package}" == "arrow" ]]; then - arrow_requested=true - break - fi - done - if [[ "${arrow_requested}" != "true" ]]; then - return - fi - - for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do - source_requested=false - for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do - if [[ "${package}" == "${source_package}" ]]; then - source_requested=true - break - fi - done - if [[ "${source_requested}" != "true" ]]; then - ARROW_PAIMON_DOWNLOAD_PACKAGES+=("${source_package}") - fi - done -} - -# Print stable path-and-content records for fingerprint inputs. Including the path -# makes patch selection and ordering part of the contract, not only file contents. -arrow_paimon_fingerprint_files() { - local file - local blob - - # Feed the sorted list through a pipe rather than a process substitution: - # build.sh is documented as "sh build.sh", and bash 3.2 invoked as sh - # (macOS /bin/sh) rejects "< <(...)" at parse time, which would abort - # every build before any component is compiled. - printf '%s\n' "$@" | LC_ALL=C sort | while IFS= read -r file; do - blob="$(git hash-object "${file}")" || exit 1 - printf 'file=%s\n' "${file}" - printf 'blob=%s\n' "${blob}" - done -} - -# Identify only the source, patch, and explicit build-schema inputs selected for -# Arrow. Arrow and Paimon publish separate installed markers so a package-only -# build cannot certify a component that it did not rebuild. -arrow_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}" - printf 'ARROW_VERSION=%s\n' "${ARROW_VERSION}" - printf 'ARROW_NAME=%s\n' "${ARROW_NAME}" - printf 'ARROW_SOURCE=%s\n' "${ARROW_SOURCE}" - printf 'ARROW_MD5SUM=%s\n' "${ARROW_MD5SUM}" - printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}" - printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}" - printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}" - printf 'XSIMD_NAME=%s\n' "${XSIMD_NAME}" - printf 'XSIMD_SOURCE=%s\n' "${XSIMD_SOURCE}" - printf 'XSIMD_MD5SUM=%s\n' "${XSIMD_MD5SUM}" - arrow_paimon_fingerprint_files \ - patches/apache-arrow-"${ARROW_VERSION}"-*.patch - } | git hash-object --stdin - ) -} - -paimon_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - arrow_build_fingerprint - printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}" - printf 'PAIMON_CPP_NAME=%s\n' "${PAIMON_CPP_NAME}" - printf 'PAIMON_CPP_SOURCE=%s\n' "${PAIMON_CPP_SOURCE}" - printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}" - arrow_paimon_fingerprint_files \ - paimon-cpp-cache.cmake \ - patches/paimon-cpp-*.patch - } | git hash-object --stdin - ) -} - -# Source patch markers use a combined value because either component's inputs -# may change the external-Arrow contract applied to both source trees. -arrow_paimon_build_fingerprint() { - { - arrow_build_fingerprint - paimon_build_fingerprint - } | git hash-object --stdin -} - -arrow_paimon_fingerprint_matches() { - local installed_fingerprint="$1" - local expected_fingerprint="$2" - local legacy_compatible_expected_fingerprint="$3" - shift 3 - local compatible_fingerprint - - if [[ "${installed_fingerprint}" == "${expected_fingerprint}" ]]; then - return 0 - fi - if [[ "${expected_fingerprint}" != "${legacy_compatible_expected_fingerprint}" ]]; then - return 1 - fi - for compatible_fingerprint in "$@"; do - if [[ "${installed_fingerprint}" == "${compatible_fingerprint}" ]]; then - return 0 - fi - done - return 1 -} - -ARROW_REQUIRED_LIBRARIES=( - libbrotlicommon.a - libbrotlidec.a - libbrotlienc.a - libarrow.a - libarrow_compute.a - libarrow_flight.a - libarrow_flight_sql.a - libarrow_dataset.a - libarrow_acero.a - libarrow_bundled_dependencies.a - libparquet.a -) - -PAIMON_REQUIRED_LIBRARIES=( - libpaimon.a - libpaimon_parquet_file_format.a - libpaimon_orc_file_format.a - libpaimon_blob_file_format.a - libpaimon_local_file_system.a - libpaimon_file_index.a - libpaimon_global_index.a - libroaring_bitmap_paimon.a - libxxhash_paimon.a - libfmt_paimon.a - libtbb_paimon.a -) - -ARROW_PAIMON_REQUIRED_LIBRARIES=( - "${ARROW_REQUIRED_LIBRARIES[@]}" - "${PAIMON_REQUIRED_LIBRARIES[@]}" -) - -arrow_artifacts_valid() { - local install_dir="$1" - local installed_arrow_version - local library - - if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then - echo "Missing installed Arrow version header" >&2 - return 1 - fi - installed_arrow_version="$( - awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { - gsub(/"/, "", $3); print $3; exit - }' "${install_dir}/include/arrow/util/config.h" - )" - if [[ "${installed_arrow_version}" != "${ARROW_VERSION}" ]]; then - echo "Installed Arrow version ${installed_arrow_version} does not match ${ARROW_VERSION}" >&2 - return 1 - fi - - for library in "${ARROW_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Arrow library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -paimon_artifacts_valid() { - local install_dir="$1" - local library - - for library in "${PAIMON_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Paimon library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -arrow_prebuilt_valid() { - local install_dir="$1" - local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${arrow_fingerprint_mark}" ]]; then - echo "Missing Arrow build fingerprint: ${arrow_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(arrow_build_fingerprint)" - installed_fingerprint="$(<"${arrow_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${ARROW_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Arrow build fingerprint does not match selected inputs" >&2 - return 1 - fi - arrow_artifacts_valid "${install_dir}" -} - -paimon_prebuilt_valid() { - local install_dir="$1" - local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" - local expected_fingerprint - local installed_fingerprint - - if [[ ! -f "${paimon_fingerprint_mark}" ]]; then - echo "Missing Paimon build fingerprint: ${paimon_fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(paimon_build_fingerprint)" - installed_fingerprint="$(<"${paimon_fingerprint_mark}")" - if ! arrow_paimon_fingerprint_matches "${installed_fingerprint}" \ - "${expected_fingerprint}" "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" \ - "${PAIMON_LEGACY_BUILD_FINGERPRINTS[@]}"; then - echo "Paimon build fingerprint does not match selected inputs" >&2 - return 1 - fi - paimon_artifacts_valid "${install_dir}" -} - -arrow_paimon_prebuilt_valid() { - local install_dir="$1" - arrow_prebuilt_valid "${install_dir}" && paimon_prebuilt_valid "${install_dir}" -} - -invalidate_arrow_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/arrow-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_arrow_prebuilt_marker() { - local install_dir="$1" - arrow_artifacts_valid "${install_dir}" - arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" -} - -invalidate_paimon_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/paimon-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" -} - -publish_paimon_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid "${install_dir}" - paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" -} - -require_arrow_prebuilt_for_paimon() { - local install_dir="$1" - if ! arrow_prebuilt_valid "${install_dir}"; then - echo "Paimon requires Arrow to be built from the currently selected inputs first" >&2 - return 1 - fi -} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index b1d84b1fe78e0f..70dbdf8d8772dc 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -196,8 +196,7 @@ if [[ "${CLEAN}" -eq 1 ]] && [[ -d "${TP_SOURCE_DIR}" ]]; then fi # Download thirdparties. -prepare_arrow_paimon_download_packages "${packages[@]}" -bash "${TP_DIR}/download-thirdparty.sh" "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}" +bash "${TP_DIR}/download-thirdparty.sh" "${packages[@]}" export LD_LIBRARY_PATH="${TP_DIR}/installed/lib:${LD_LIBRARY_PATH}" @@ -1110,7 +1109,6 @@ build_grpc() { # arrow build_arrow() { check_if_source_exist "${ARROW_SOURCE}" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" mkdir -p release @@ -1197,8 +1195,6 @@ build_arrow() { strip_lib libparquet.a strip_lib libarrow_dataset.a strip_lib libarrow_acero.a - - publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } # arrow-adbc @@ -2305,94 +2301,6 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# paimon-cpp -build_paimon_cpp() { - check_if_source_exist "${PAIMON_CPP_SOURCE}" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" - - # Darwin doesn't build GNU libunwind in this script, so don't force -lunwind there. - local paimon_linker_flags="-L${TP_LIB_DIR} -lbrotlienc -lbrotlidec -lbrotlicommon -llzma" - if [[ "${KERNEL}" != 'Darwin' ]]; then - paimon_linker_flags="${paimon_linker_flags} -lunwind" - fi - - CXXFLAGS="-Wno-nontrivial-memcall" \ - "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ - -G "${GENERATOR}" \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ - -DPAIMON_BUILD_SHARED=OFF \ - -DPAIMON_BUILD_STATIC=ON \ - -DPAIMON_BUILD_TESTS=OFF \ - -DPAIMON_ENABLE_ORC=ON \ - -DPAIMON_ENABLE_AVRO=OFF \ - -DPAIMON_ENABLE_LANCE=OFF \ - -DPAIMON_ENABLE_JINDO=OFF \ - -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_LUCENE=OFF \ - -DCMAKE_EXE_LINKER_FLAGS="${paimon_linker_flags}" \ - -DCMAKE_SHARED_LINKER_FLAGS="${paimon_linker_flags}" \ - .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install - - # Install paimon-cpp internal dependencies with renamed versions - # These libraries are built but not installed by default - echo "Installing paimon-cpp internal dependencies..." - - # Arrow deps: When PAIMON_USE_EXTERNAL_ARROW=ON (Plan B), paimon-cpp - # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps - # directory is not needed. When building its own Arrow (legacy), copy - # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" - if [ -d "arrow_ep-install/lib" ]; then - mkdir -p "${paimon_deps_dir}" - for paimon_arrow_dep in \ - libarrow.a \ - libarrow_compute.a \ - libarrow_filesystem.a \ - libarrow_dataset.a \ - libarrow_acero.a \ - libparquet.a; do - if [ -f "arrow_ep-install/lib/${paimon_arrow_dep}" ]; then - cp -v "arrow_ep-install/lib/${paimon_arrow_dep}" "${paimon_deps_dir}/${paimon_arrow_dep}" - fi - done - else - echo " arrow_ep-install not found (PAIMON_USE_EXTERNAL_ARROW=ON?) – skipping paimon_deps Arrow copy" - fi - - # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap - if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" - fi - - # Install xxhash, renamed to avoid conflict with Doris's xxhash - if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" - fi - - # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) - if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" - fi - - # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) - if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" - fi - - echo "Paimon-cpp internal dependencies installed successfully" - publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" -} - # lance-c build_lance_c() { check_if_source_exist "${LANCE_C_SOURCE}" @@ -2532,7 +2440,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then icu mecab_ipadic pugixml - paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" @@ -2640,7 +2547,6 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; - paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; lzma) src_var="LZMA_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 95c67e2cdad209..6b8d90c726f431 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -85,6 +85,15 @@ while [[ $# -gt 0 ]]; do ) done if [[ "${SPEC_LIB}" != "" ]]; then + # Arrow builds xsimd and Brotli from their source archives. + if [[ " ${SPEC_ARCHIVES[*]} " == *' ARROW '* ]]; then + for arrow_companion in XSIMD BROTLI; do + if [[ " ${SPEC_ARCHIVES[*]} " != *" ${arrow_companion} "* ]]; then + SPEC_ARCHIVES+=("${arrow_companion}") + fi + done + fi + # ARROW_ADBC_FLIGHTSQL is a companion archive of arrow_adbc rather than a # package of its own: it has no build function, build_arrow_adbc() only copies # the prebuilt driver out of it. Its name therefore never appears on a command @@ -367,20 +376,6 @@ echo "===== Patching thirdparty archives..." # This is to avoid duplicated patch. ################################################################################### PATCHED_MARK="patched_mark" -ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" -ARROW_PAIMON_BUILD_FINGERPRINT="" -if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || - " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" -fi - -reset_arrow_paimon_source() { - local archive_name="$1" - local source_name="$2" - echo "Resetting ${source_name} because its patch state is incomplete or stale" - rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" -} # glog patch if [[ " ${TP_ARCHIVES[*]} " =~ " GLOG " ]]; then @@ -486,16 +481,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then cd - fi if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-24.0.0" ]]; then - arrow_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_SOURCE}/${PATCHED_MARK}" && - -f "${arrow_fingerprint_mark}" ]] || - [[ "$(<"${arrow_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${ARROW_NAME}" "${ARROW_SOURCE}" - cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" - # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size, - # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-paimon.patch" - + cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then # Introducing the parameter that forces writing INT96 timestamps for # compatibility with the Doris Parquet writer. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-force-write-int96-timestamps.patch" @@ -503,10 +490,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then # Add Parquet LZO page decompression support used by file scanner v2. patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-lzo.patch" touch "${PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - fi + cd - fi echo "Finished patching ${ARROW_SOURCE}" fi @@ -778,31 +763,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# patch paimon-cpp -if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then - PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" - PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK="patched_mark_arrow_24_compute" - paimon_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_PATCHED_MARK}" && - -f "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}/${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" && - -f "${paimon_fingerprint_mark}" ]] || - [[ "$(<"${paimon_fingerprint_mark}")" != "${ARROW_PAIMON_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${PAIMON_CPP_NAME}" "${PAIMON_CPP_SOURCE}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" - touch "${PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" - touch "${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - - fi - echo "Finished patching ${PAIMON_CPP_SOURCE}" -fi - # Apply Doris lance-c patches. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.8" ]]; then diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake deleted file mode 100644 index 6ab6049c59b0d5..00000000000000 --- a/thirdparty/paimon-cpp-cache.cmake +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# CMake Initial Cache for paimon-cpp -# Configures paimon-cpp to reuse selected Doris third-party libraries -# Usage: cmake -C paimon-cpp-cache.cmake ... - -# Get the Doris thirdparty installation directory from environment -set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") - -if(NOT DORIS_THIRDPARTY_DIR) - message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") -endif() - -message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") - -# Set CMAKE_PREFIX_PATH to help find_package locate our libraries -set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") - -# Library and include paths -set(DORIS_LIB_DIR "${DORIS_THIRDPARTY_DIR}/lib" CACHE PATH "Doris library directory") -set(DORIS_INCLUDE_DIR "${DORIS_THIRDPARTY_DIR}/include" CACHE PATH "Doris include directory") - -# ============================================================================ -# ZLIB - Reuse from Doris (version 1.3.1) -# ============================================================================ -set(ZLIB_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZLIB root directory") -set(ZLIB_LIBRARY "${DORIS_LIB_DIR}/libz.a" CACHE FILEPATH "ZLIB library") -set(ZLIB_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZLIB include directory") - -# ============================================================================ -# ZSTD - Reuse from Doris (version 1.5.7) -# ============================================================================ -set(ZSTD_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZSTD root directory") -set(ZSTD_LIBRARY "${DORIS_LIB_DIR}/libzstd.a" CACHE FILEPATH "ZSTD library") -set(ZSTD_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZSTD include directory") - -# ============================================================================ -# LZ4 - Reuse from Doris (version 1.9.4) -# ============================================================================ -set(LZ4_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "LZ4 root directory") -set(LZ4_LIBRARY "${DORIS_LIB_DIR}/liblz4.a" CACHE FILEPATH "LZ4 library") -set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") - -# ============================================================================ -# glog - NOT reused from Doris -# paimon-cpp's build_glog() unconditionally calls externalproject_add() to -# build glog 0.7.1. Any GLOG_ROOT/GLOG_LIBRARY/GLOG_INCLUDE_DIR set here -# would be overwritten by that macro, so we skip them entirely. -# ============================================================================ - -# ============================================================================ -# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp -# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). -# ============================================================================ -set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") - -set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") - -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") - -# Protobuf, Thrift - still built separately by paimon-cpp - -# ============================================================================ -# Snappy - Reuse from Doris -# ============================================================================ -set(Snappy_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory") -set(SNAPPY_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory (legacy)") -set(SNAPPY_LIBRARY "${DORIS_LIB_DIR}/libsnappy.a" CACHE FILEPATH "Snappy library") -set(SNAPPY_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Snappy include directory") - -# ============================================================================ -# Build configuration -# ============================================================================ -set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build with -fPIC") -set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type") - -# Symbol visibility control to prevent conflicts with Doris -# paimon-cpp builds Arrow/ORC/etc with hidden symbols to avoid conflicts -set(CMAKE_CXX_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C++ symbols by default") -set(CMAKE_C_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C symbols by default") -set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inline function symbols") - -# Verify that required libraries exist -if(NOT EXISTS "${ZLIB_LIBRARY}") - message(FATAL_ERROR "ZLIB library not found: ${ZLIB_LIBRARY}") -endif() -if(NOT EXISTS "${ZSTD_LIBRARY}") - message(FATAL_ERROR "ZSTD library not found: ${ZSTD_LIBRARY}") -endif() -if(NOT EXISTS "${LZ4_LIBRARY}") - message(FATAL_ERROR "LZ4 library not found: ${LZ4_LIBRARY}") -endif() -if(NOT EXISTS "${SNAPPY_LIBRARY}") - message(FATAL_ERROR "Snappy library not found: ${SNAPPY_LIBRARY}") -endif() - -message(STATUS "========================================") -message(STATUS "Paimon-cpp Library Reuse Configuration") -message(STATUS "========================================") -message(STATUS "Reusing from Doris:") -message(STATUS " ✓ ZLIB, ZSTD, LZ4, Snappy") -if(PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " ✓ Arrow, Parquet, Arrow Dataset, Arrow Acero (Plan B)") -else() - message(STATUS " ✗ Arrow (building separately, symbol visibility=hidden)") -endif() -message(STATUS "") -message(STATUS "Building separately:") -if(NOT PAIMON_USE_EXTERNAL_ARROW) - message(STATUS " - Arrow, Protobuf, Thrift, ORC") -else() - message(STATUS " - Protobuf, Thrift, ORC") -endif() -message(STATUS " - glog, RapidJSON, TBB") -message(STATUS "========================================") diff --git a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch index 1847a50694f014..85ec1f8021cd9f 100644 --- a/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch @@ -1,11 +1,10 @@ --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,8 +215,9 @@ +@@ -215,7 +215,9 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && -- target_unit == ::arrow::TimeUnit::NANO) { +- if (arrow_properties.support_deprecated_int96_timestamps()) { + if (arrow_properties.force_write_int96_timestamps() || + (arrow_properties.support_deprecated_int96_timestamps() && + target_unit == ::arrow::TimeUnit::NANO)) { diff --git a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch deleted file mode 100644 index 583f85b4400bbf..00000000000000 --- a/thirdparty/patches/apache-arrow-24.0.0-paimon.patch +++ /dev/null @@ -1,226 +0,0 @@ ---- a/cpp/src/parquet/arrow/schema.cc -+++ b/cpp/src/parquet/arrow/schema.cc -@@ -215,7 +215,8 @@ - - // The user is explicitly asking for Impala int96 encoding, there is no - // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && -+ target_unit == ::arrow::TimeUnit::NANO) { - *physical_type = ParquetType::INT96; - return Status::OK(); - } ---- a/cpp/src/parquet/arrow/reader.cc -+++ b/cpp/src/parquet/arrow/reader.cc -@@ -1041,25 +1041,32 @@ - ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); - } - -- int64_t num_rows = 0; -+ std::vector num_rows; - for (int row_group : row_groups) { -- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); -+ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); - } - - using ::arrow::RecordBatchIterator; -+ size_t row_group_idx = 0; - - // NB: This lambda will be invoked outside the scope of this call to - // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. - // `this` is a non-owning pointer so we are relying on the parent FileReader outliving - // this RecordBatchReader. - ::arrow::Iterator batches = ::arrow::MakeFunctionIterator( -- [readers, batch_schema, num_rows, -+ [readers, batch_schema, num_rows, row_group_idx, - this]() mutable -> ::arrow::Result { - ::arrow::ChunkedArrayVector columns(readers.size()); - - // don't reserve more rows than necessary -- int64_t batch_size = std::min(properties().batch_size(), num_rows); -- num_rows -= batch_size; -+ int64_t batch_size = 0; -+ if (!num_rows.empty()) { -+ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); -+ num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && row_group_idx + 1 < num_rows.size()) { -+ ++row_group_idx; -+ } -+ } - - RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( - reader_properties_.use_threads(), static_cast(readers.size()), ---- a/cpp/src/parquet/arrow/writer.cc -+++ b/cpp/src/parquet/arrow/writer.cc -@@ -341,6 +341,14 @@ - return Status::OK(); - } - -+ int64_t GetBufferedSize() override { -+ if (row_group_writer_ == nullptr) { -+ return 0; -+ } -+ return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); -+ } -+ - Status Close() override { - if (!closed_) { - // Make idempotent -@@ -444,10 +452,14 @@ - - // Max number of rows allowed in a row group. - const int64_t max_row_group_length = this->properties().max_row_group_length(); -+ const int64_t max_row_group_size = this->properties().max_row_group_size(); - - // Initialize a new buffered row group writer if necessary. - if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || -- row_group_writer_->num_rows() >= max_row_group_length) { -+ row_group_writer_->num_rows() >= max_row_group_length || -+ row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= -+ max_row_group_size) { - RETURN_NOT_OK(NewBufferedRowGroup()); - } - ---- a/cpp/src/parquet/arrow/writer.h -+++ b/cpp/src/parquet/arrow/writer.h -@@ -124,6 +124,9 @@ - /// option in this case. - virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; - -+ /// \brief Return the buffered size in bytes. -+ virtual int64_t GetBufferedSize() = 0; -+ - /// \brief Write the footer and close the file. - virtual ::arrow::Status Close() = 0; - virtual ~FileWriter(); ---- a/cpp/src/parquet/properties.h -+++ b/cpp/src/parquet/properties.h -@@ -160,6 +160,7 @@ - static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; - static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; - static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; -+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; - static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; - static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; - static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -343,6 +344,7 @@ - dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), - write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), - max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), -+ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), - pagesize_(kDefaultDataPageSize), - max_rows_per_page_(kDefaultMaxRowsPerPage), - version_(ParquetVersion::PARQUET_2_6), -@@ -359,6 +361,7 @@ - dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), - write_batch_size_(properties.write_batch_size()), - max_row_group_length_(properties.max_row_group_length()), -+ max_row_group_size_(properties.max_row_group_size()), - pagesize_(properties.data_pagesize()), - max_rows_per_page_(properties.max_rows_per_page()), - version_(properties.version()), -@@ -468,6 +471,13 @@ - return this; - } - -+ /// Specify the max bytes size to put in a single row group. -+ /// Default 128MB. -+ Builder* max_row_group_size(int64_t max_row_group_size) { -+ max_row_group_size_ = max_row_group_size; -+ return this; -+ } -+ - /// Specify the data page size. - /// Default 1MB. - Builder* data_pagesize(int64_t pg_size) { -@@ -867,11 +877,12 @@ - - return std::shared_ptr(new WriterProperties( - pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, -- size_statistics_level_, std::move(file_encryption_properties_), -- default_column_properties_, column_properties, data_page_version_, -- store_decimal_as_integer_, std::move(sorting_columns_), -- content_defined_chunking_enabled_, content_defined_chunking_options_)); -+ max_row_group_size_, pagesize_, max_rows_per_page_, version_, created_by_, -+ page_checksum_enabled_, size_statistics_level_, -+ std::move(file_encryption_properties_), default_column_properties_, -+ column_properties, data_page_version_, store_decimal_as_integer_, -+ std::move(sorting_columns_), content_defined_chunking_enabled_, -+ content_defined_chunking_options_)); - } - - private: -@@ -881,6 +892,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetVersion::type version_; -@@ -917,6 +929,8 @@ - - inline int64_t max_row_group_length() const { return max_row_group_length_; } - -+ inline int64_t max_row_group_size() const { return max_row_group_size_; } -+ - inline int64_t data_pagesize() const { return pagesize_; } - - inline int64_t max_rows_per_page() const { return max_rows_per_page_; } -@@ -1046,9 +1060,10 @@ - private: - explicit WriterProperties( - MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, -- ParquetVersion::type version, const std::string& created_by, -- bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, -+ int64_t max_rows_per_page, ParquetVersion::type version, -+ const std::string& created_by, bool page_write_checksum_enabled, -+ SizeStatisticsLevel size_statistics_level, - std::shared_ptr file_encryption_properties, - const ColumnProperties& default_column_properties, - const std::unordered_map& column_properties, -@@ -1059,6 +1074,7 @@ - dictionary_pagesize_limit_(dictionary_pagesize_limit), - write_batch_size_(write_batch_size), - max_row_group_length_(max_row_group_length), -+ max_row_group_size_(max_row_group_size), - pagesize_(pagesize), - max_rows_per_page_(max_rows_per_page), - parquet_data_page_version_(data_page_version), -@@ -1078,6 +1094,7 @@ - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - int64_t max_rows_per_page_; - ParquetDataPageVersion parquet_data_page_version_; ---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1884,7 +1884,20 @@ - REQUIRED_VERSION - 0.11.0) - -- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -+ if(NOT Thrift_VERSION) -+ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -+ set(Thrift_VERSION "${thrift_PC_VERSION}") -+ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -+ set(Thrift_VERSION "${ThriftAlt_VERSION}") -+ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -+ set(Thrift_VERSION "${THRIFT_VERSION}") -+ endif() -+ endif() -+ if(NOT Thrift_VERSION) -+ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -+ endif() -+ -+ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") - list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) - list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) - list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch deleted file mode 100644 index d0e723a34d98ef..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/include/paimon/status.h b/include/paimon/status.h ---- a/include/paimon/status.h -+++ b/include/paimon/status.h -@@ -458,9 +458,4 @@ - } \ - } while (false) - --// This is an internal-use macro and should not be used in public headers. --#ifndef RETURN_NOT_OK --#define RETURN_NOT_OK(s) PAIMON_RETURN_NOT_OK(s) --#endif -- - } // namespace paimon -diff --git a/src/paimon/format/parquet/parquet_input_stream_impl.cpp b/src/paimon/format/parquet/parquet_input_stream_impl.cpp ---- a/src/paimon/format/parquet/parquet_input_stream_impl.cpp -+++ b/src/paimon/format/parquet/parquet_input_stream_impl.cpp -@@ -59,7 +59,7 @@ arrow::Result> ParquetInputStreamImpl::Read(int64 - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, Read(nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -@@ -78,7 +78,7 @@ arrow::Result> ParquetInputStreamImpl::ReadAt(int - arrow::AllocateResizableBuffer(nbytes, pool_.get())); - ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, ReadAt(position, nbytes, buffer->mutable_data())); - if (read_bytes < nbytes) { -- RETURN_NOT_OK(buffer->Resize(read_bytes)); -+ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); - } - return std::shared_ptr(std::move(buffer)); - } -diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp ---- a/src/paimon/format/parquet/file_reader_wrapper.cpp -+++ b/src/paimon/format/parquet/file_reader_wrapper.cpp -@@ -81,8 +81,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); - } -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- target_row_group_indices, target_column_indices_, &batch_reader_)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader_, file_reader_->GetRecordBatchReader(target_row_group_indices, -+ target_column_indices_)); - return Status::OK(); - } - } -@@ -152,9 +153,11 @@ Status FileReaderWrapper::PrepareForReading( - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - std::unique_ptr batch_reader; -- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( -- std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -- column_indices, &batch_reader)); -+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( -+ batch_reader, -+ file_reader_->GetRecordBatchReader( -+ std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), -+ column_indices)); - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch b/thirdparty/patches/paimon-cpp-arrow-24-compute.patch deleted file mode 100644 index 1b581b8f93ee92..00000000000000 --- a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -38,6 +38,8 @@ - set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH - "Include directory for external Arrow/Parquet headers") - set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "" CACHE FILEPATH -+ "Path to external libarrow_compute.a") - set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") - set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") - set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -@@ -1008,6 +1010,7 @@ - - foreach(_paimon_external_lib - IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_COMPUTE_LIB - PAIMON_EXTERNAL_ARROW_DATASET_LIB - PAIMON_EXTERNAL_ARROW_ACERO_LIB - PAIMON_EXTERNAL_PARQUET_LIB -@@ -1028,6 +1031,13 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -+ add_library(arrow_compute STATIC IMPORTED) -+ set_target_properties(arrow_compute -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_COMPUTE_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ - add_library(arrow_dataset STATIC IMPORTED) - set_target_properties(arrow_dataset - PROPERTIES IMPORTED_LOCATION -@@ -1055,7 +1065,9 @@ - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_INCLUDE_DIR}") - -- target_link_libraries(arrow_acero INTERFACE arrow) -+ target_link_libraries(arrow_compute INTERFACE arrow) -+ -+ target_link_libraries(arrow_acero INTERFACE arrow_compute) - - target_link_libraries(arrow_dataset INTERFACE arrow_acero) diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch deleted file mode 100644 index 213cc279f52e5c..00000000000000 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ /dev/null @@ -1,362 +0,0 @@ -diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake -index 74654a4..4065297 100644 ---- a/cmake_modules/BuildUtils.cmake -+++ b/cmake_modules/BuildUtils.cmake -@@ -55,12 +55,18 @@ function(add_paimon_lib LIB_NAME) - # Necessary to make static linking into other shared libraries work properly - set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) - if(ARG_DEPENDENCIES) -- # Avoid add_dependencies on non-existent targets (e.g. when building static only). -+ # In static-only builds, some dependency names are still declared as -+ # *_shared. Map them to *_static when the shared target is unavailable. - set(_paimon_objlib_deps) - foreach(_paimon_dep IN LISTS ARG_DEPENDENCIES) -- if(TARGET ${_paimon_dep}) -- list(APPEND _paimon_objlib_deps ${_paimon_dep}) -+ set(_paimon_mapped_dep "${_paimon_dep}") -+ if(NOT TARGET ${_paimon_mapped_dep} AND _paimon_dep MATCHES "_shared$") -+ string(REGEX REPLACE "_shared$" "_static" _paimon_mapped_dep "${_paimon_dep}") - endif() -+ if(TARGET ${_paimon_mapped_dep}) -+ list(APPEND _paimon_objlib_deps ${_paimon_mapped_dep}) -+ endif() -+ unset(_paimon_mapped_dep) - endforeach() - if(_paimon_objlib_deps) - add_dependencies(${LIB_NAME}_objlib ${_paimon_objlib_deps}) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 57f6c50..d8e3cd0 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -38,6 +38,8 @@ string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) - --set(CMAKE_CXX_STANDARD 17) -+if(NOT DEFINED CMAKE_CXX_STANDARD) -+ set(CMAKE_CXX_STANDARD 17) -+endif() - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -909,9 +909,6 @@ macro(build_orc) - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" -- "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" -- "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" - "-DLZ4_HOME=${ORC_LZ4_ROOT}" - "-DZSTD_HOME=${ORC_ZSTD_ROOT}" -@@ -923,6 +920,13 @@ macro(build_orc) - -DBUILD_TOOLS=OFF - -DBUILD_CPP_ENABLE_METRICS=ON) - -+ if(ORC_RPATH) -+ list(APPEND ORC_CMAKE_ARGS -+ "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}" -+ "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}") -+ endif() -+ - set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/orc.diff") - externalproject_add(orc_ep - URL ${ORC_SOURCE_URL} - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1024,6 +1024,14 @@ macro(build_arrow) - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" - -DARROW_DEPENDENCY_USE_SHARED=OFF -+ # Avoid forcing CONDA dependency mode when CONDA_PREFIX is present. -+ # AUTO keeps the normal "find system first, fallback to bundled" -+ # behavior and prevents accidental pickup of conda's thrift/zstd. -+ -DARROW_DEPENDENCY_SOURCE=AUTO -+ # Isolate from user/system CMake package registries to improve -+ # reproducibility in CI and local mixed environments. -+ -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF -+ -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF - -DARROW_BUILD_SHARED=OFF - -DARROW_BUILD_STATIC=ON - -DARROW_BUILD_TESTS=OFF - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -761,6 +761,9 @@ macro(build_protobuf) - get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}" - DIRECTORY) -+ set(THIRDPARTY_ZLIB_STATIC_LIB -+ "${THIRDPARTY_ZLIB_ROOT}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" -+ ) - - # Strip lto flags (which may be added by dh_auto_configure) - # See https://github.com/protocolbuffers/protobuf/issues/7092 -@@ -778,6 +781,10 @@ macro(build_protobuf) - "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}" - "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}" -+ "-DZLIB_INCLUDE_DIR=${THIRDPARTY_ZLIB_INCLUDE_DIR}" -+ "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_STATIC_LIB}" -+ "-DZLIB_LIBRARY_DEBUG=${THIRDPARTY_ZLIB_STATIC_LIB}" - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) - set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS}) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -34,6 +34,16 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" - "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") - -+option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF) -+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH -+ "Include directory for external Arrow/Parquet headers") -+set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") -+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") -+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") -+set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") -+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH -+ "Path to external libarrow_bundled_dependencies.a") -+ - macro(set_urls URLS) - set(${URLS} ${ARGN}) - endmacro() - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -961,5 +961,95 @@ macro(build_orc) - endmacro() - - macro(build_arrow) -- message(STATUS "Building Arrow from source") -+ if(PAIMON_USE_EXTERNAL_ARROW) -+ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include") -+ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") -+ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow" -+ "${ARROW_INCLUDE_DIR}/arrow") -+ endif() -+ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet") -+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink -+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" -+ "${ARROW_INCLUDE_DIR}/parquet") -+ endif() -+ -+ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR) -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON" -+ ) -+ endif() -+ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}") -+ message(FATAL_ERROR -+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}" -+ ) -+ endif() -+ -+ foreach(_paimon_external_lib -+ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB -+ PAIMON_EXTERNAL_ARROW_DATASET_LIB -+ PAIMON_EXTERNAL_ARROW_ACERO_LIB -+ PAIMON_EXTERNAL_PARQUET_LIB -+ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB) -+ if(NOT ${_paimon_external_lib}) -+ message(FATAL_ERROR -+ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON") -+ endif() -+ if(NOT EXISTS "${${_paimon_external_lib}}") -+ message(FATAL_ERROR -+ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}") -+ endif() -+ endforeach() -+ -+ add_library(arrow STATIC IMPORTED) -+ set_target_properties(arrow -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_dataset STATIC IMPORTED) -+ set_target_properties(arrow_dataset -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_acero STATIC IMPORTED) -+ set_target_properties(arrow_acero -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(parquet STATIC IMPORTED) -+ set_target_properties(parquet -+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ add_library(arrow_bundled_dependencies STATIC IMPORTED) -+ set_target_properties(arrow_bundled_dependencies -+ PROPERTIES IMPORTED_LOCATION -+ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}" -+ INTERFACE_INCLUDE_DIRECTORIES -+ "${ARROW_INCLUDE_DIR}") -+ -+ target_link_libraries(arrow_acero INTERFACE arrow) -+ -+ target_link_libraries(arrow_dataset INTERFACE arrow_acero) -+ -+ target_link_libraries(arrow -+ INTERFACE zstd -+ snappy -+ lz4 -+ zlib -+ arrow_bundled_dependencies) -+ -+ target_link_libraries(parquet -+ INTERFACE zstd snappy lz4 zlib arrow_bundled_dependencies -+ arrow_dataset) -+ else() -+ message(STATUS "Building Arrow from source") - - get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES) - get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY) - -diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake ---- a/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1121,6 +1121,7 @@ macro(build_arrow) - zlib - arrow_bundled_dependencies - arrow_dataset) -+ endif() - - endmacro(build_arrow) - -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -326,10 +326,10 @@ if(PAIMON_ENABLE_LUMINA) - include_directories("${CMAKE_SOURCE_DIR}/third_party/lumina/include") - endif() - -+include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) - include_directories(SYSTEM ${TBB_INCLUDE_DIR}) - --include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - add_compile_definitions("GLOG_USE_GLOG_EXPORT") - - set(THREADS_PREFER_PTHREAD_FLAG ON) - -diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp ---- a/src/paimon/common/logging/logging.cpp -+++ b/src/paimon/common/logging/logging.cpp -@@ -83,7 +83,7 @@ std::unique_ptr Logger::GetLogger(const std::string& path) { - } - std::unique_lock ulock(getRegistryLock()); - if (!google::IsGoogleLoggingInitialized()) { -- google::InitGoogleLogging(program_invocation_name); -+ google::InitGoogleLogging("paimon-cpp"); - } - return std::make_unique(); - } - -diff --git a/src/paimon/common/memory/memory_pool.cpp b/src/paimon/common/memory/memory_pool.cpp ---- a/src/paimon/common/memory/memory_pool.cpp -+++ b/src/paimon/common/memory/memory_pool.cpp -@@ -55,7 +55,7 @@ void* MemoryPoolImpl::Malloc(uint64_t size, uint64_t alignment) { - return memptr; - } - --void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, size_t alignment) { -+void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment) { - if (alignment == 0) { - void* memptr = ::realloc(p, new_size); - total_allocated_size.fetch_add(new_size - old_size); - -diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp ---- a/src/paimon/format/blob/blob_format_writer.cpp -+++ b/src/paimon/format/blob/blob_format_writer.cpp -@@ -138,7 +138,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - uint64_t total_read_length = 0; -- uint32_t read_len = std::min(file_length, tmp_buffer_->size()); -+ uint32_t read_len = -+ static_cast(std::min(file_length, tmp_buffer_->size())); - while (read_len > 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); - if (static_cast(actual_read_len) != read_len) { -@@ -149,7 +150,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - } - PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); - total_read_length += actual_read_len; -- read_len = std::min(file_length - total_read_length, tmp_buffer_->size()); -+ read_len = static_cast( -+ std::min(file_length - total_read_length, tmp_buffer_->size())); - } - - // write bin length - ---- a/cmake_modules/arrow.diff -+++ b/cmake_modules/arrow.diff -@@ -196,3 +196,29 @@ - int64_t pagesize_; - ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+index 9df922afa2..5c8b3d4d07 100644 -+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -+@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) -+ REQUIRED_VERSION -+ 0.11.0) -+ -+- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -++ if(NOT Thrift_VERSION) -++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -++ set(Thrift_VERSION "${thrift_PC_VERSION}") -++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -++ set(Thrift_VERSION "${ThriftAlt_VERSION}") -++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -++ set(Thrift_VERSION "${THRIFT_VERSION}") -++ endif() -++ endif() -++ if(NOT Thrift_VERSION) -++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -++ endif() -++ -++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") -+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) -+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) -+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) - -diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt -index 76dd03b..9317324 100644 ---- a/src/paimon/CMakeLists.txt -+++ b/src/paimon/CMakeLists.txt -@@ -281,6 +281,11 @@ add_paimon_lib(paimon - ${PAIMON_COMMON_SRCS} - ${PAIMON_CORE_SRCS} - DEPENDENCIES -+ # Codec headers come from ExternalProject install steps, so object -+ # compilation must not race those steps in clean parallel builds. -+ zstd -+ snappy -+ lz4 - arrow - tbb - glog diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh deleted file mode 100755 index f946be5c7631d3..00000000000000 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -create_fingerprint_fixture() { - local destination="$1" - - mkdir -p "${destination}/patches" - cp "${ROOT}/arrow-paimon-vars.sh" "${destination}/arrow-paimon-vars.sh" - cp "${ROOT}/paimon-cpp-cache.cmake" "${destination}/paimon-cpp-cache.cmake" - cp "${ROOT}"/patches/apache-arrow-24.0.0-*.patch "${destination}/patches/" - cp "${ROOT}"/patches/paimon-cpp-*.patch "${destination}/patches/" -} - -fingerprint_from_fixture() { - local fixture="$1" - local component="$2" - local result_variable="$3" - local fingerprint - - fingerprint="$( - set -e - # shellcheck source=/dev/null - . "${fixture}/arrow-paimon-vars.sh" - "${component}_build_fingerprint" - )" - printf -v "${result_variable}" '%s' "${fingerprint}" -} - -exercise_semantic_fingerprints() { - local first_fixture="${tmpdir}/fingerprint-first" - local second_fixture="${tmpdir}/fingerprint-second" - local first_arrow - local first_paimon - local second_arrow - local second_paimon - local changed_arrow - local changed_paimon - - create_fingerprint_fixture "${first_fixture}" - create_fingerprint_fixture "${second_fixture}" - - printf '%s\n' first-env >"${first_fixture}/env.sh" - printf '%s\n' second-env >"${second_fixture}/env.sh" - printf '%s\n' first-vars >"${first_fixture}/vars.sh" - printf '%s\n' second-vars >"${second_fixture}/vars.sh" - printf '%s\n' first-download >"${first_fixture}/download-thirdparty.sh" - printf '%s\n' second-download >"${second_fixture}/download-thirdparty.sh" - printf '%s\n' first-build >"${first_fixture}/build-thirdparty.sh" - printf '%s\n' second-build >"${second_fixture}/build-thirdparty.sh" - printf '%s\n' '# release-branch-only comment' >>"${second_fixture}/arrow-paimon-vars.sh" - - fingerprint_from_fixture "${first_fixture}" arrow first_arrow - fingerprint_from_fixture "${first_fixture}" paimon first_paimon - [[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Arrow legacy marker migration target is stale" - [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the Paimon legacy marker migration target is stale" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon second_paimon - [[ "${first_arrow}" == "${second_arrow}" ]] || - fail "unrelated branch scripts changed the Arrow fingerprint" - [[ "${first_paimon}" == "${second_paimon}" ]] || - fail "unrelated branch scripts changed the Paimon fingerprint" - - printf '%s\n' semantic-change \ - >>"${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - fingerprint_from_fixture "${second_fixture}" arrow changed_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow patch change did not change the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "an Arrow patch change did not change the Paimon fingerprint" - - cp "${first_fixture}/patches/apache-arrow-24.0.0-lzo.patch" \ - "${second_fixture}/patches/apache-arrow-24.0.0-lzo.patch" - printf '%s\n' semantic-change >>"${second_fixture}/paimon-cpp-cache.cmake" - fingerprint_from_fixture "${second_fixture}" arrow second_arrow - fingerprint_from_fixture "${second_fixture}" paimon changed_paimon - [[ "${second_arrow}" == "${first_arrow}" ]] || - fail "a Paimon-only cache change changed the Arrow fingerprint" - [[ "${changed_paimon}" != "${first_paimon}" ]] || - fail "a Paimon cache change did not change the Paimon fingerprint" - - changed_arrow="$( - set -e - # shellcheck source=/dev/null - . "${first_fixture}/arrow-paimon-vars.sh" - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_build_fingerprint - )" - [[ "${changed_arrow}" != "${first_arrow}" ]] || - fail "an Arrow build-schema change did not change its fingerprint" -} - -# shellcheck source=../arrow-paimon-vars.sh -. "${ROOT}/arrow-paimon-vars.sh" -exercise_semantic_fingerprints - -harness="${tmpdir}/harness" -mkdir -p "${harness}/src" "${harness}/patches" "${harness}/installed" -cp "${ROOT}/download-thirdparty.sh" "${harness}/download-thirdparty.sh" - -create_patch() { - local patch_file="$1" - local source_file="$2" - { - printf '%s\n' "--- a/${source_file}" - printf '%s\n' "+++ b/${source_file}" - printf '%s\n' '@@ -1 +1 @@' - printf '%s\n' '-original' - printf '%s\n' '+patched' - } >"${patch_file}" -} - -create_archive() { - local source_name="$1" - local archive_name="$2" - local prefix="$3" - local index - - mkdir -p "${harness}/src/${source_name}" - for index in 1 2 3; do - printf '%s\n' original >"${harness}/src/${source_name}/${prefix}-${index}.txt" - done - tar -czf "${harness}/src/${archive_name}" -C "${harness}/src" "${source_name}" - rm -rf "${harness}/src/${source_name}" -} - -arrow_source="arrow-apache-arrow-24.0.0" -arrow_archive="apache-arrow-24.0.0.tar.gz" -paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" -paimon_archive="paimon-cpp-0a4f4e2.tar.gz" - -create_archive "${arrow_source}" "${arrow_archive}" arrow -create_archive "${paimon_source}" "${paimon_archive}" paimon - -arrow_patches=( - apache-arrow-24.0.0-paimon.patch - apache-arrow-24.0.0-force-write-int96-timestamps.patch - apache-arrow-24.0.0-lzo.patch -) -paimon_patches=( - paimon-cpp-buildutils-static-deps.patch - paimon-cpp-arrow-24-compatibility.patch - paimon-cpp-arrow-24-compute.patch -) - -for index in 0 1 2; do - create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" - create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" -done - -arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" -paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" -{ - printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" - printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" - printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' - printf 'ARROW_NAME="%s"\n' "${arrow_archive}" - printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" - printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" - printf '%s\n' 'ARROW_DOWNLOAD="unused"' - printf 'PAIMON_CPP_NAME="%s"\n' "${paimon_archive}" - printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" - printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" - printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' - printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' -} >"${harness}/vars.sh" - -exercise_interrupted_patch_set() { - local package="$1" - local source_name="$2" - local archive_name="$3" - local prefix="$4" - shift 4 - local patches=("$@") - local applied - local index - - for applied in 1 2 3; do - rm -rf "${harness}/src/${source_name}" - tar -xzf "${harness}/src/${archive_name}" -C "${harness}/src" - for ((index = 0; index < applied; ++index)); do - ( - cd "${harness}/src/${source_name}" - patch -s -p1 <"${harness}/patches/${patches[${index}]}" - ) - done - - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - - for index in 1 2 3; do - [[ "$(<"${harness}/src/${source_name}/${prefix}-${index}.txt")" == "patched" ]] || - fail "${package} did not recover after interruption boundary ${applied}" - done - [[ "$(<"${harness}/src/${source_name}/patched_mark_arrow_paimon_fingerprint")" == "test-fingerprint" ]] || - fail "${package} fingerprint marker is missing" - - touch "${harness}/src/${source_name}/idempotence-sentinel" - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" "${package}" >/dev/null - [[ -f "${harness}/src/${source_name}/idempotence-sentinel" ]] || - fail "${package} reset a completely patched source tree" - done -} - -exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow \ - "${arrow_patches[@]}" -exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ - "${paimon_patches[@]}" - -exercise_generic_recovery_dispatch() { - local generic="${tmpdir}/generic-recovery" - local thirdparty_dir="${generic}/thirdparty" - local external_thirdparty_dir="${generic}/external-thirdparty" - local args_file="${generic}/build-args.txt" - local output_file="${generic}/build-output.txt" - local external_builder_called="${generic}/external-builder-called" - local fake_mvn="${generic}/fake-mvn" - local non_native_status=79 - local non_native_target - local status - local flag - local parallel - local clean - local package1 - local package2 - local extra - - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ - "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ - "${generic}/be/output" - cp "${ROOT}/../build.sh" "${generic}/build.sh" - cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" - printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" - # shellcheck disable=SC2016 - printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" - chmod +x "${fake_mvn}" - { - printf '%s\n' '#!/usr/bin/env bash' - printf 'exit %q\n' "${non_native_status}" - } >"${generic}/generated-source.sh" - chmod +x "${generic}/generated-source.sh" - { - printf '%s\n' 'DORIS_BUILD_PROFILE=0' - printf '%s\n' 'TARGET_SYSTEM=Linux' - printf 'MVN_CMD=%q\n' "${fake_mvn}" - } >"${generic}/env.sh" - { - printf '%s\n' '#!/usr/bin/env bash' - # RECOVERY_ARGS_FILE is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'printf "%s\n" "$*" >"${RECOVERY_ARGS_FILE:?}"' - # shellcheck disable=SC2016 - printf '%s\n' 'exit "${RECOVERY_EXIT_STATUS:-73}"' - } >"${thirdparty_dir}/build-thirdparty.sh" - - DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --clean >"${output_file}" 2>&1 || - fail "bare clean failed before reaching its clean-only exit" - [[ ! -e "${args_file}" ]] || fail "bare clean invoked the thirdparty builder" - [[ ! -d "${generic}/be/build_Release" && ! -d "${generic}/be/output" ]] || - fail "bare clean did not remove BE build outputs" - - for non_native_target in --fe --hive-udf; do - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" "${non_native_target}" >"${output_file}" 2>&1; then - fail "${non_native_target} did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "${non_native_target} failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "${non_native_target} invoked the native Arrow/Paimon builder" - done - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then - fail "--fe --clean did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "--fe --clean failed before the generated-source sentinel" - [[ ! -e "${args_file}" ]] || - fail "--fe --clean invoked the native Arrow/Paimon builder" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic stale-prebuilt recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be --clean >"${output_file}" 2>&1; then - fail "generic clean recovery did not invoke the focused builder" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || - fail "generic clean recovery dispatched the wrong build package set" - - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - RECOVERY_EXIT_STATUS=0 bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted artifacts that failed current-checkout validation" - fi - grep -Fq "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs" \ - "${output_file}" || fail "generic recovery did not validate artifacts after its builder returned" - - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an invalid install-only thirdparty prefix" - fi - grep -Fq "is an install-only or incomplete prefix" "${output_file}" || - fail "generic recovery did not explain how to refresh an install-only prebuilt" - - { - printf '%s\n' '#!/usr/bin/env bash' - # EXTERNAL_BUILDER_CALLED is expanded when the generated builder runs. - # shellcheck disable=SC2016 - printf '%s\n' 'touch "${EXTERNAL_BUILDER_CALLED:?}"' - } >"${external_thirdparty_dir}/build-thirdparty.sh" - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - EXTERNAL_BUILDER_CALLED="${external_builder_called}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "generic recovery accepted an external thirdparty source tree" - fi - grep -Fq "Cannot rebuild thirdparty libraries with an external source tree" "${output_file}" || - fail "generic recovery did not reject an external thirdparty source tree" - [[ ! -e "${external_builder_called}" ]] || - fail "generic recovery invoked an external thirdparty builder" -} - -exercise_generic_recovery_dispatch - -# A Paimon-only build may publish only its own fingerprint. It must not make a -# stale Arrow installation pass the shared prebuilt validation. -prebuilt="${tmpdir}/prebuilt" -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" -printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" -for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" -done - -prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" -[[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || - fail "focused recovery dispatches a bundled source package as a build target" -[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || - fail "focused recovery does not download the complete Arrow source closure" - -# A legacy prebuilt may have the old combined marker but no component markers. -# Generic build.sh consumers must reject it before importing Arrow Compute. -arrow_paimon_build_fingerprint >"${prebuilt}/arrow-paimon-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "legacy combined marker certified an unversioned component closure" -fi - -printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/arrow-build-fingerprint.txt" -printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${prebuilt}/paimon-build-fingerprint.txt" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "the complete shared prebuilt was rejected during fingerprint migration" -if ( - ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" - arrow_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Arrow marker survived a semantic fingerprint change" -fi -if ( - PAIMON_BUILD_SCHEMA_VERSION="${PAIMON_BUILD_SCHEMA_VERSION}-changed" - paimon_prebuilt_valid "${prebuilt}" -) >/dev/null 2>&1; then - fail "the legacy Paimon marker survived a semantic fingerprint change" -fi - -publish_arrow_prebuilt_marker "${prebuilt}" -publish_paimon_prebuilt_marker "${prebuilt}" -rm "${prebuilt}/lib64/libarrow_compute.a" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "prebuilt validation accepted a missing Arrow Compute archive" -fi -touch "${prebuilt}/lib64/libarrow_compute.a" - -printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon-only marker update certified a stale Arrow build" -fi -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "Paimon build accepted a stale installed Arrow" -fi - -publish_arrow_prebuilt_marker "${prebuilt}" -require_arrow_prebuilt_for_paimon "${prebuilt}" || - fail "Paimon build rejected the selected installed Arrow" -arrow_paimon_prebuilt_valid "${prebuilt}" || fail "matching component markers were rejected" - -invalidate_paimon_prebuilt_marker "${prebuilt}" -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Paimon rebuild left its old marker valid" -fi -publish_paimon_prebuilt_marker "${prebuilt}" - -invalidate_arrow_prebuilt_marker "${prebuilt}" -if require_arrow_prebuilt_for_paimon "${prebuilt}" >/dev/null 2>&1; then - fail "an interrupted Arrow rebuild left its old marker valid" -fi -publish_arrow_prebuilt_marker "${prebuilt}" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "republished component markers were rejected" - -echo "PASS" diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh deleted file mode 100755 index 4d3b92458b8c26..00000000000000 --- a/thirdparty/test/paimon-codec-dependency-test.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -tmpdir="$(mktemp -d)" -trap 'rm -rf "${tmpdir}"' EXIT - -fixture="${tmpdir}/paimon-codec-dependency" -focused_patch="${fixture}/codec-dependency.patch" -mkdir -p "${fixture}/src/paimon" -{ - printf '%s\n' 'add_paimon_lib(paimon' - printf '%s\n' ' SOURCES' - printf '%s\n' ' ${PAIMON_COMMON_SRCS}' - printf '%s\n' ' ${PAIMON_CORE_SRCS}' - printf '%s\n' ' DEPENDENCIES' - printf '%s\n' ' arrow' - printf '%s\n' ' tbb' - printf '%s\n' ' glog' - printf '%s\n' ' fmt' - printf '%s\n' ' roaring_bitmap' - printf '%s\n' ' xxhash' - printf '%s\n' ' Threads::Threads' - printf '%s\n' ' RapidJSON' - printf '%s\n' ' STATIC_LINK_LIBS' - printf '%s\n' ' arrow)' -} >"${fixture}/src/paimon/CMakeLists.txt" - -awk ' - /^diff --git / { - if (found) { - exit - } - if ($0 == "diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt") { - found = 1 - } - } - found { print } -' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}" -[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is missing" -( - cd "${fixture}" - patch -s -p1 <"${focused_patch}" -) || fail "the Paimon codec dependency patch did not apply" - -dependencies="$(sed -n '/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \ - "${fixture}/src/paimon/CMakeLists.txt")" -grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the ZSTD headers" -grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the Snappy headers" -grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" || - fail "paimon_objlib does not wait for the LZ4 headers" - -echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 84f4a73edaabf9..fa7643d62c33bb 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -50,10 +50,6 @@ export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" # export REPOSITORY_URL= DORIS_THIRDPARTY_REPOSITORY_URL="${DORIS_THIRDPARTY_REPOSITORY_URL:-https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/thirdparty/juicefs}" -DORIS_THIRDPARTY_VARS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=arrow-paimon-vars.sh -. "${DORIS_THIRDPARTY_VARS_DIR}/arrow-paimon-vars.sh" - ##################################################### # Download url, filename and unpaced filename # of all thirdparties @@ -145,7 +141,6 @@ LZO2_SOURCE=lzo-2.10 LZO2_MD5SUM="39d3f3f9c55c87b1e5d6888e1420f4b5" # rapidjson -# Updated to match paimon-cpp version (commit 232389d4f1012dddec4ef84861face2d2ba85709) RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/232389d4f1012dddec4ef84861face2d2ba85709.tar.gz" RAPIDJSON_NAME=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz RAPIDJSON_SOURCE=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709 @@ -239,6 +234,12 @@ ZSTD_NAME=zstd-1.5.7.tar.gz ZSTD_SOURCE=zstd-1.5.7 ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48" +# brotli +BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" +BROTLI_NAME="brotli-1.0.9.tar.gz" +BROTLI_SOURCE="brotli-1.0.9" +BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" + # flatbuffers FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz" FLATBUFFERS_NAME=flatbuffers-23.5.26.tar.gz @@ -258,6 +259,12 @@ GRPC_NAME="grpc-v1.54.3.tar.gz" GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" +# arrow +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz" +ARROW_NAME="apache-arrow-24.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-24.0.0" +ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" + # arrow-adbc # One source tree, three artifacts: the driver manager (statically linked into # doris_be), the JNI bridge (loaded by the FE adbc connector) and the SQLite @@ -455,6 +462,13 @@ BENCHMARK_NAME=benchmark-v1.8.0.tar.gz BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" +# xsimd +# for arrow-24.0.0, if arrow upgrade, this version may also need to be changed +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" +XSIMD_NAME="14.0.0.tar.gz" +XSIMD_SOURCE=xsimd-14.0.0 +XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" + # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" SIMDJSON_NAME=simdjson-3.11.6.tar.gz @@ -679,7 +693,6 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' - 'PAIMON_CPP' 'LANCE_C' )