[chore](paimon) remove paimon-cpp dependency

.github/workflows/be-ut-mac.yml

@@ -75,9 +75,8 @@ jobs:
           )
           brew install "${cellars[@]}" || true
 
-          # Paimon's bundled dependencies still require pre-CMake-4 policy
-          # compatibility. Match the supported version used by the full
-          # thirdparty jobs instead of relying on Homebrew's latest CMake.
+          # Match the supported version used by the full thirdparty jobs instead
+          # of relying on Homebrew's latest CMake.
           brew unlink cmake || true
           wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
           tar -xzf cmake-3.25.3-macos-universal.tar.gz
@@ -101,21 +100,20 @@ jobs:
           fi
           tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz
 
-          # Rebuild the Arrow/Paimon stack when the shared prebuilt predates the
-          # selected Arrow version/component closure.
-          # shellcheck source=thirdparty/arrow-paimon-vars.sh
-          . ./arrow-paimon-vars.sh
-          arrow_paimon_prebuilt_is_valid=false
-          if arrow_paimon_prebuilt_valid installed; then
-            arrow_paimon_prebuilt_is_valid=true
+          # Rebuild Arrow when the shared prebuilt predates the selected source closure.
+          # shellcheck source=thirdparty/arrow-vars.sh
+          . ./arrow-vars.sh
+          arrow_prebuilt_is_valid=false
+          if arrow_prebuilt_valid installed; then
+            arrow_prebuilt_is_valid=true
           fi
-          if [[ "${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then
+          if [[ "${arrow_prebuilt_is_valid}" != "true" ]]; then
             curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \
               -o doris-thirdparty-source.tgz
             tar -zxvf doris-thirdparty-source.tgz
-            ./download-thirdparty.sh arrow paimon_cpp xsimd brotli
+            ./download-thirdparty.sh arrow xsimd brotli
             export MACOSX_DEPLOYMENT_TARGET=12.0
-            ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp
+            ./build-thirdparty.sh -j "$(nproc)" arrow
           fi
           popd
 

.github/workflows/build-thirdparty.yml

@@ -32,7 +32,7 @@ jobs:
     if: github.event_name == 'pull_request'
     outputs:
       thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
-      arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }}
+      arrow_lifecycle_changes: ${{ steps.filter.outputs.arrow_lifecycle_changes }}
     steps:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
@@ -48,36 +48,32 @@ jobs:
             thirdparty_changes:
               - 'thirdparty/**'
               - 'env.sh'
-            arrow_paimon_lifecycle_changes:
+            arrow_lifecycle_changes:
               - 'build.sh'
               - 'env.sh'
-              - 'thirdparty/arrow-paimon-vars.sh'
+              - 'thirdparty/arrow-vars.sh'
               - 'thirdparty/vars.sh'
               - 'thirdparty/build-thirdparty.sh'
               - 'thirdparty/download-thirdparty.sh'
-              - 'thirdparty/paimon-cpp-cache.cmake'
               - 'thirdparty/patches/apache-arrow-*.patch'
-              - 'thirdparty/patches/paimon-cpp-*.patch'
               - 'regression-test/pipeline/external/conf/fe.conf'
-              - 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
               - 'thirdparty/test/adbc-jni-config-test.sh'
-              - 'thirdparty/test/paimon-codec-dependency-test.sh'
+              - 'thirdparty/test/arrow-lifecycle-test.sh'
               - '.github/workflows/build-thirdparty.yml'
 
-  arrow_paimon_lifecycle_test:
-    name: Arrow/Paimon Lifecycle Test
+  arrow_lifecycle_test:
+    name: Arrow Lifecycle Test
     needs: changes
-    if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }}
+    if: ${{ needs.changes.outputs.arrow_lifecycle_changes == 'true' }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
 
       - name: Test focused thirdparty lifecycle
         run: |
-          thirdparty/test/arrow-paimon-lifecycle-test.sh
+          thirdparty/test/arrow-lifecycle-test.sh
           thirdparty/test/adbc-jni-config-test.sh
-          thirdparty/test/paimon-codec-dependency-test.sh
 
   script_test:
     name: Thirdparty Script Test

be/CMakeLists.txt

@@ -199,8 +199,6 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}"
 option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF)
 message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
 
-option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
-set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
 option(ENABLE_TDE "Enable TDE feature module" OFF)
 set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
 option(ENABLE_TLS "Enable TLS feature module" OFF)
@@ -218,14 +216,6 @@ if (ENABLE_VARIANT_NESTED_GROUP AND "${VARIANT_NESTED_GROUP_MODULE_DIR}" STREQUA
     message(FATAL_ERROR "ENABLE_VARIANT_NESTED_GROUP requires VARIANT_NESTED_GROUP_MODULE_DIR")
 endif()
 
-# Allow env to override when reconfiguring (avoid picking /usr/local).
-if (DEFINED ENV{ENABLE_PAIMON_CPP})
-    set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
-endif()
-if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
-    set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
-endif()
-
 set(CMAKE_SKIP_RPATH TRUE)
 set(Boost_USE_STATIC_LIBS ON)
 set(Boost_USE_STATIC_RUNTIME ON)
@@ -696,10 +686,6 @@ set(COMMON_THIRDPARTY
     ${COMMON_THIRDPARTY}
 )
 
-if (ENABLE_PAIMON_CPP)
-    message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode")
-endif()
-
 if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
     add_library(hadoop_hdfs STATIC IMPORTED)
     set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
@@ -759,19 +745,6 @@ if (BUILD_BENCHMARK)
     )
 endif()
 
-set(PAIMON_FACTORY_REGISTRY_LIBS)
-if (ENABLE_PAIMON_CPP)
-    # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
-    # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
-    # thirdparty.cmake.  paimon-cpp reuses the same Arrow (no paimon_deps).
-    # No dual-stack selection needed — single Arrow for everything.
-
-    # paimon_parquet_file_format depends on Arrow Dataset symbols.
-    # Force-link it with --whole-archive so its factory registration runs.
-    set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format)
-    list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS})
-endif()
-
 set(DORIS_DEPENDENCIES
     ${DORIS_DEPENDENCIES}
     ${WL_START_GROUP}
@@ -789,29 +762,6 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static)
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static)
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib)
 
-if (ENABLE_PAIMON_CPP)
-    if (PAIMON_FACTORY_REGISTRY_LIBS)
-        if (APPLE)
-            foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS})
-                set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-                    -Wl,-force_load,$<TARGET_FILE:${lib}>)
-            endforeach()
-        else()
-            set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-                -Wl,--whole-archive
-                ${PAIMON_FACTORY_REGISTRY_LIBS}
-                -Wl,--no-whole-archive)
-        endif()
-    endif()
-
-    # paimon-cpp internal dependencies (renamed with _paimon suffix)
-    # These must come after paimon libraries to resolve symbols.
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon)
-endif()
-
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
 
 # Add all external dependencies. They should come after the palo libs.

be/cmake/thirdparty.cmake

@@ -189,18 +189,3 @@ add_thirdparty(icudata LIB64)
 
 
 add_thirdparty(pugixml LIB64)
-
-if (ENABLE_PAIMON_CPP)
-    add_thirdparty(paimon LIB64)
-    add_thirdparty(paimon_parquet_file_format LIB64)
-    add_thirdparty(paimon_orc_file_format LIB64)
-    add_thirdparty(paimon_blob_file_format LIB64)
-    add_thirdparty(paimon_local_file_system LIB64)
-    add_thirdparty(paimon_file_index LIB64)
-    add_thirdparty(paimon_global_index LIB64)
-
-    add_thirdparty(roaring_bitmap_paimon LIB64)
-    add_thirdparty(xxhash_paimon LIB64)
-    add_thirdparty(fmt_paimon LIB64)
-    add_thirdparty(tbb_paimon LIB64)
-endif()

be/src/common/config.cpp

@@ -1721,40 +1721,10 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB
 /** Iceberg sink configurations **/
 DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB
 
-// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem.
-// Each entry uses the format "<scheme>=<file_type>", and file_type must be one of:
-// local, hdfs, s3, http, broker.
-DEFINE_Strings(paimon_file_system_scheme_mappings,
-               "file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs,"
-               "s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3,"
-               "abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http,"
-               "ofs=broker,gfs=broker");
-DEFINE_Validator(paimon_file_system_scheme_mappings,
-                 ([](const std::vector<std::string>& mappings) -> bool {
-                     doris::StringCaseUnorderedSet seen_schemes;
-                     static const doris::StringCaseUnorderedSet supported_types = {
-                             "local", "hdfs", "s3", "http", "broker"};
-                     for (const auto& raw_entry : mappings) {
-                         std::string_view entry = doris::trim(raw_entry);
-                         size_t separator = entry.find('=');
-                         if (separator == std::string_view::npos) {
-                             return false;
-                         }
-                         std::string scheme = std::string(doris::trim(entry.substr(0, separator)));
-                         std::string file_type =
-                                 std::string(doris::trim(entry.substr(separator + 1)));
-                         if (scheme.empty() || file_type.empty()) {
-                             return false;
-                         }
-                         if (supported_types.find(file_type) == supported_types.end()) {
-                             return false;
-                         }
-                         if (!seen_schemes.insert(scheme).second) {
-                             return false;
-                         }
-                     }
-                     return true;
-                 }));
+/** Paimon sink configurations **/
+DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB
+DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes,
+                 [](int64_t bytes) -> bool { return bytes > 0; });
 
 DEFINE_mInt32(thrift_client_open_num_tries, "1");
 

be/src/common/config.h

@@ -1787,8 +1787,9 @@ DECLARE_mInt64(hive_sink_max_file_size);
 /** Iceberg sink configurations **/
 DECLARE_mInt64(iceberg_sink_max_file_size);
 
-/** Paimon file system configurations **/
-DECLARE_Strings(paimon_file_system_scheme_mappings);
+/** Paimon sink configurations **/
+// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer.
+DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes);
 
 // Number of open tries, default 1 means only try to open once.
 // Retry the Open num_retries time waiting 100 milliseconds between retries.

be/src/exec/scan/file_scanner.cpp

@@ -75,9 +75,7 @@
 #include "format/table/iceberg_sys_table_jni_reader.h"
 #include "format/table/jdbc_jni_reader.h"
 #include "format/table/max_compute_jni_reader.h"
-#include "format/table/paimon_cpp_reader.h"
 #include "format/table/paimon_jni_reader.h"
-#include "format/table/paimon_predicate_converter.h"
 #include "format/table/paimon_reader.h"
 #include "format/table/partition_column_filler.h"
 #include "format/table/remote_doris_reader.h"
@@ -1084,50 +1082,11 @@ Status FileScanner::_get_next_reader() {
                 _cur_reader = std::move(mc_reader);
             } else if (range.__isset.table_format_params &&
                        range.table_format_params.table_format_type == "paimon") {
-                const auto& paimon_params = range.table_format_params.paimon_params;
-                bool use_paimon_cpp_reader = false;
-                if (paimon_params.__isset.reader_type) {
-                    switch (paimon_params.reader_type) {
-                    case TPaimonReaderType::PAIMON_CPP:
-                        use_paimon_cpp_reader = true;
-                        break;
-                    case TPaimonReaderType::PAIMON_JNI:
-                        break;
-                    case TPaimonReaderType::PAIMON_NATIVE:
-                        return Status::InternalError(
-                                "invalid PAIMON_NATIVE reader_type for paimon FORMAT_JNI split, "
-                                "possibly caused by FE/BE protocol mismatch");
-                    default:
-                        return Status::InternalError(
-                                "unknown paimon reader_type for paimon FORMAT_JNI split, possibly "
-                                "caused by FE/BE protocol mismatch");
-                    }
-                } else {
-                    // TODO: Remove this fallback after all FE versions set TPaimonReaderType.
-                    use_paimon_cpp_reader =
-                            _state->query_options().__isset.enable_paimon_cpp_reader &&
-                            _state->query_options().enable_paimon_cpp_reader;
-                }
-                if (use_paimon_cpp_reader) {
-                    auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state,
-                                                                     _profile, range, _params);
-                    if (!_is_load && !_push_down_conjuncts.empty()) {
-                        PaimonPredicateConverter predicate_converter(_file_slot_descs, _state);
-                        auto predicate = predicate_converter.build(_push_down_conjuncts);
-                        if (predicate) {
-                            cpp_reader->set_predicate(std::move(predicate));
-                        }
-                    }
-                    init_status =
-                            static_cast<GenericReader*>(cpp_reader.get())->init_reader(&jni_ctx);
-                    _cur_reader = std::move(cpp_reader);
-                } else {
-                    auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state,
-                                                                        _profile, range, _params);
-                    init_status =
-                            static_cast<GenericReader*>(paimon_reader.get())->init_reader(&jni_ctx);
-                    _cur_reader = std::move(paimon_reader);
-                }
+                auto paimon_reader = PaimonJniReader::create_unique(_file_slot_descs, _state,
+                                                                    _profile, range, _params);
+                init_status =
+                        static_cast<GenericReader*>(paimon_reader.get())->init_reader(&jni_ctx);
+                _cur_reader = std::move(paimon_reader);
             } else if (range.__isset.table_format_params &&
                        range.table_format_params.table_format_type == "hudi") {
                 auto hudi_reader = HudiJniReader::create_unique(

be/src/exec/scan/file_scanner_v2.cpp

@@ -142,12 +142,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) {
         }
         const auto& params = range.table_format_params.paimon_params;
         if (params.__isset.reader_type) {
-            if (params.reader_type == TPaimonReaderType::PAIMON_JNI) {
-                return params.__isset.paimon_split;
-            }
-            // V2 cannot pass a logical DataSplit through a raw native child without silently
-            // dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback.
-            return false;
+            return params.reader_type == TPaimonReaderType::PAIMON_JNI &&
+                   params.__isset.paimon_split;
         }
         if (params.__isset.paimon_split) {
             // Before reader_type was added, an encoded split unambiguously selected the Java

be/src/format/generic_reader.h

@@ -178,7 +178,7 @@ class GenericReader : public ProfileCollector {
     /// cannot be accidentally overridden.
     Status init_reader(ReaderInitContext* ctx) {
         // Apply push_down_agg_type early so _open_file_reader and _do_init_reader
-        // can use it (e.g., PaimonCppReader skips full init on COUNT pushdown).
+        // can use it before format-specific initialization.
         // on_after_init_reader may reset this (e.g., Iceberg with equality deletes).
         set_push_down_agg_type(ctx->push_down_agg_type);