Skip to content

branch-4.1: [chore](paimon) remove paimon-cpp dependency #67378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
suxiaogang223 wants to merge 1 commit into
base: branch-4.1
Choose a base branch
from
Open

branch-4.1: [chore](paimon) remove paimon-cpp dependency #67378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 10 additions & 19 deletions .github/workflows/build-thirdparty.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
github.event.issue.user.login == 'github-actions[bot]')
outputs:
thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }}
focused_test_changes: ${{ steps.filter.outputs.focused_test_changes }}
steps:
- name: Checkout ${{ github.ref }}
uses: actions/checkout@v4
Expand All @@ -55,36 +55,27 @@ jobs:
thirdparty_changes:
- 'thirdparty/**'
- 'env.sh'
arrow_paimon_lifecycle_changes:
- 'build.sh'
- 'env.sh'
- 'thirdparty/arrow-paimon-vars.sh'
- 'thirdparty/vars.sh'
- 'thirdparty/build-thirdparty.sh'
- 'thirdparty/download-thirdparty.sh'
- 'thirdparty/paimon-cpp-cache.cmake'
- 'thirdparty/patches/apache-arrow-*.patch'
- 'thirdparty/patches/paimon-cpp-*.patch'
focused_test_changes:
- 'regression-test/pipeline/external/conf/fe.conf'
- 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
- 'thirdparty/test/adbc-jni-config-test.sh'
- 'thirdparty/test/paimon-codec-dependency-test.sh'
- '.github/workflows/build-thirdparty.yml'

arrow_paimon_lifecycle_test:
name: Arrow/Paimon Lifecycle Test
script_test:
name: Thirdparty Script Test
needs: changes
if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }}
if: ${{ needs.changes.outputs.thirdparty_changes == 'true' || needs.changes.outputs.focused_test_changes == 'true' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout ${{ github.ref }}
uses: actions/checkout@v4

- name: Test focused thirdparty lifecycle
- name: Test download-thirdparty.sh
run: |
bash thirdparty/test/download-thirdparty-fallback-test.sh

- name: Test thirdparty configuration
run: |
thirdparty/test/arrow-paimon-lifecycle-test.sh
thirdparty/test/adbc-jni-config-test.sh
thirdparty/test/paimon-codec-dependency-test.sh

build_linux:
name: Build Third Party Libraries (Linux)
Expand Down
50 changes: 0 additions & 50 deletions be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,6 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}"
option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF)
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")

option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
option(ENABLE_TDE "Enable TDE feature module" OFF)
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
option(ENABLE_TLS "Enable TLS feature module" OFF)
Expand All @@ -166,14 +164,6 @@ if (ENABLE_VARIANT_NESTED_GROUP AND "${VARIANT_NESTED_GROUP_MODULE_DIR}" STREQUA
message(FATAL_ERROR "ENABLE_VARIANT_NESTED_GROUP requires VARIANT_NESTED_GROUP_MODULE_DIR")
endif()

# Allow env to override when reconfiguring (avoid picking /usr/local).
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
endif()

set(CMAKE_SKIP_RPATH TRUE)
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
Expand Down Expand Up @@ -597,10 +587,6 @@ set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
)

if (ENABLE_PAIMON_CPP)
message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode")
endif()

if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
add_library(hadoop_hdfs STATIC IMPORTED)
set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
Expand Down Expand Up @@ -660,19 +646,6 @@ if (BUILD_BENCHMARK)
)
endif()

set(PAIMON_FACTORY_REGISTRY_LIBS)
if (ENABLE_PAIMON_CPP)
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
# No dual-stack selection needed — single Arrow for everything.

# paimon_parquet_file_format depends on Arrow Dataset symbols.
# Force-link it with --whole-archive so its factory registration runs.
set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format)
list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS})
endif()

set(DORIS_DEPENDENCIES
${DORIS_DEPENDENCIES}
${WL_START_GROUP}
Expand All @@ -690,29 +663,6 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib)

if (ENABLE_PAIMON_CPP)
if (PAIMON_FACTORY_REGISTRY_LIBS)
if (APPLE)
foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS})
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-Wl,-force_load,$<TARGET_FILE:${lib}>)
endforeach()
else()
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-Wl,--whole-archive
${PAIMON_FACTORY_REGISTRY_LIBS}
-Wl,--no-whole-archive)
endif()
endif()

# paimon-cpp internal dependencies (renamed with _paimon suffix)
# These must come after paimon libraries to resolve symbols.
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon)
endif()

set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})

# Add all external dependencies. They should come after the palo libs.
Expand Down
15 changes: 0 additions & 15 deletions be/cmake/thirdparty.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,18 +187,3 @@ add_thirdparty(icudata LIB64)


add_thirdparty(pugixml LIB64)

if (ENABLE_PAIMON_CPP)
add_thirdparty(paimon LIB64)
add_thirdparty(paimon_parquet_file_format LIB64)
add_thirdparty(paimon_orc_file_format LIB64)
add_thirdparty(paimon_blob_file_format LIB64)
add_thirdparty(paimon_local_file_system LIB64)
add_thirdparty(paimon_file_index LIB64)
add_thirdparty(paimon_global_index LIB64)

add_thirdparty(roaring_bitmap_paimon LIB64)
add_thirdparty(xxhash_paimon LIB64)
add_thirdparty(fmt_paimon LIB64)
add_thirdparty(tbb_paimon LIB64)
endif()
35 changes: 0 additions & 35 deletions be/src/common/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1636,41 +1636,6 @@ DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB
DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes,
[](int64_t bytes) -> bool { return bytes > 0; });

// URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem.
// Each entry uses the format "<scheme>=<file_type>", and file_type must be one of:
// local, hdfs, s3, http, broker.
DEFINE_Strings(paimon_file_system_scheme_mappings,
"file=local,hdfs=hdfs,viewfs=hdfs,local=hdfs,jfs=hdfs,"
"s3=s3,s3a=s3,s3n=s3,oss=s3,obs=s3,cos=s3,cosn=s3,gs=s3,"
"abfs=s3,abfss=s3,wasb=s3,wasbs=s3,http=http,https=http,"
"ofs=broker,gfs=broker");
DEFINE_Validator(paimon_file_system_scheme_mappings,
([](const std::vector<std::string>& mappings) -> bool {
doris::StringCaseUnorderedSet seen_schemes;
static const doris::StringCaseUnorderedSet supported_types = {
"local", "hdfs", "s3", "http", "broker"};
for (const auto& raw_entry : mappings) {
std::string_view entry = doris::trim(raw_entry);
size_t separator = entry.find('=');
if (separator == std::string_view::npos) {
return false;
}
std::string scheme = std::string(doris::trim(entry.substr(0, separator)));
std::string file_type =
std::string(doris::trim(entry.substr(separator + 1)));
if (scheme.empty() || file_type.empty()) {
return false;
}
if (supported_types.find(file_type) == supported_types.end()) {
return false;
}
if (!seen_schemes.insert(scheme).second) {
return false;
}
}
return true;
}));

DEFINE_mInt32(thrift_client_open_num_tries, "1");

DEFINE_Bool(enable_index_compaction, "false");
Expand Down
3 changes: 0 additions & 3 deletions be/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1698,9 +1698,6 @@ DECLARE_mInt64(iceberg_sink_max_file_size);
// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer.
DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes);

/** Paimon file system configurations **/
DECLARE_Strings(paimon_file_system_scheme_mappings);

// Number of open tries, default 1 means only try to open once.
// Retry the Open num_retries time waiting 100 milliseconds between retries.
DECLARE_mInt32(thrift_client_open_num_tries);
Expand Down
24 changes: 3 additions & 21 deletions be/src/exec/scan/file_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@
#include "format/table/iceberg_sys_table_jni_reader.h"
#include "format/table/lakesoul_jni_reader.h"
#include "format/table/max_compute_jni_reader.h"
#include "format/table/paimon_cpp_reader.h"
#include "format/table/paimon_jni_reader.h"
#include "format/table/paimon_predicate_converter.h"
#include "format/table/paimon_reader.h"
#include "format/table/partition_column_filler.h"
#include "format/table/remote_doris_reader.h"
Expand Down Expand Up @@ -1111,25 +1109,9 @@ Status FileScanner::_get_next_reader() {
_cur_reader = std::move(mc_reader);
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "paimon") {
if (_state->query_options().__isset.enable_paimon_cpp_reader &&
_state->query_options().enable_paimon_cpp_reader) {
auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state,
_profile, range, _params);
cpp_reader->set_push_down_agg_type(_get_push_down_agg_type());
if (!_is_load && !_push_down_conjuncts.empty()) {
PaimonPredicateConverter predicate_converter(_file_slot_descs, _state);
auto predicate = predicate_converter.build(_push_down_conjuncts);
if (predicate) {
cpp_reader->set_predicate(std::move(predicate));
}
}
init_status = cpp_reader->init_reader();
_cur_reader = std::move(cpp_reader);
} else {
_cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
range, _params);
init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader();
}
_cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
range, _params);
init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader();
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "hudi") {
_cur_reader = HudiJniReader::create_unique(*_params,
Expand Down
8 changes: 2 additions & 6 deletions be/src/exec/scan/file_scanner_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,12 +145,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) {
}
const auto& params = range.table_format_params.paimon_params;
if (params.__isset.reader_type) {
if (params.reader_type == TPaimonReaderType::PAIMON_JNI) {
return params.__isset.paimon_split;
}
// V2 cannot pass a logical DataSplit through a raw native child without silently
// dropping its multi-file semantics, so PAIMON_CPP must remain on the V1 fallback.
return false;
return params.reader_type == TPaimonReaderType::PAIMON_JNI &&
params.__isset.paimon_split;
}
if (params.__isset.paimon_split) {
// Before reader_type was added, an encoded split unambiguously selected the Java
Expand Down
Loading
Loading