From 89250b1f048e6c4b00273c49aca11a316572f5bd Mon Sep 17 00:00:00 2001 From: suxiaogang Date: Tue, 1 Sep 2026 15:41:25 +0800 Subject: [PATCH 01/12] [chore](paimon) Upgrade Paimon to 1.4.2 ### What problem does this PR solve? Issue Number: close #65086 Related PR: #66346 Problem Summary: Upgrade the master Paimon runtime from 1.3.1 to 1.4.2 and adapt the JNI scanner and Connector SPI read path to the updated fallback-table, catalog, partition, scan-statistics, and split-count APIs. This establishes the dependency baseline required by the branch-4.1 table-write forward-port without importing unrelated write implementations. ### Release note Upgrade the Paimon connector runtime to 1.4.2. ### Check List (For Author) - Test: Unit Test - PaimonJniScannerTest: 33 tests passed - Connector Paimon targeted tests: 182 tests passed - Maven validate passed - Behavior changed: Yes (Paimon runtime is upgraded from 1.3.1 to 1.4.2) - Does this need documentation: No --- .../apache/doris/paimon/PaimonJniScanner.java | 27 +++++++++--- .../doris/paimon/PaimonJniScannerTest.java | 42 ++++++++++-------- .../connector/paimon/PaimonConnector.java | 6 ++- .../connector/paimon/PaimonReaderOptions.java | 29 +++++++++--- .../paimon/PaimonScanPlanProvider.java | 14 +++--- .../connector/paimon/FakePaimonTable.java | 32 ++++++++++++++ .../paimon/PaimonBackendBoundTableTest.java | 44 ++++++++++--------- .../PaimonConnectorMetadataPartitionTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../PaimonConnectorMetadataReadAuthTest.java | 3 +- .../paimon/PaimonHmsCatalogTest.java | 3 +- .../paimon/PaimonReaderOptionsTest.java | 40 +++++++++++------ .../paimon/PaimonScanMetricsTest.java | 2 +- .../paimon/PaimonScanPlanProviderTest.java | 16 ++++--- fe/pom.xml | 4 +- 15 files changed, 184 insertions(+), 84 deletions(-) diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 1dbc045d2e47a3..e0fe9cb1a75622 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -41,6 +41,8 @@ import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataType; import org.apache.paimon.types.TimestampType; +import org.apache.paimon.utils.ChainTableUtils; +import org.apache.paimon.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -703,14 +705,14 @@ private static Table applyManifestParallelismBound( FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) table; FileStoreTable main = applyManifestParallelismBound( pair.wrapped(), safeBound, materializeAbsent); - FileStoreTable fallback = applyManifestParallelismBound( - pair.fallback(), safeBound, materializeAbsent); - if (main == pair.wrapped() && fallback == pair.fallback()) { + FileStoreTable other = applyManifestParallelismBound( + pair.other(), safeBound, materializeAbsent); + if (main == pair.wrapped() && other == pair.other()) { return table; } // Each branch owns an independent planner setting; a smaller sibling is not an // execution ceiling and must never throttle the other branch. - return new FallbackReadFileStoreTable(main, fallback); + return new FallbackReadFileStoreTable(main, other, isWrappedFirst(pair)); } if (table instanceof DelegatedFileStoreTable) { @@ -773,6 +775,21 @@ private static FileStoreTable applyManifestParallelismBound( (Table) table, safeBound, materializeAbsent); } + static boolean isWrappedFirst(FallbackReadFileStoreTable table) { + Map options = table.options(); + // Mirror Paimon 1.4.2 FileStoreTableFactory. There is no public accessor for the wrapper's + // private ordering flag, so reconstruction must recover the factory decision from options. + if (ChainTableUtils.isChainTable(options)) { + return true; + } + if (!StringUtils.isNullOrWhitespaceOnly( + options.get(CoreOptions.SCAN_FALLBACK_BRANCH.key()))) { + return true; + } + return StringUtils.isNullOrWhitespaceOnly( + options.get(CoreOptions.SCAN_PRIMARY_BRANCH.key())); + } + private static FileStoreTable unwrapSystemPlanningSource(FileStoreTable table) { FileStoreTable current = table; // System wrappers dispatch fallback reads only when the fallback pair is their direct @@ -801,7 +818,7 @@ private static void validateSerializedReaderOptions(Table table) { validateSerializedAsyncThreshold(table.options().get(CoreOptions.FILE_READER_ASYNC_THRESHOLD.key())); validateSerializedSplitTargetSize(table.options().get(CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key())); if (table instanceof FallbackReadFileStoreTable) { - validateSerializedReaderOptions(((FallbackReadFileStoreTable) table).fallback()); + validateSerializedReaderOptions(((FallbackReadFileStoreTable) table).other()); } if (table instanceof DelegatedFileStoreTable) { validateSerializedReaderOptions(((DelegatedFileStoreTable) table).wrapped()); diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index 8fe6523ed74a2d..a1949630a24589 100644 --- a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -159,7 +159,7 @@ public void testOldFeSerializedFallbackZeroReadBatchIsRejected() throws Exceptio Collections.singletonMap(CoreOptions.READ_BATCH_SIZE.key(), "0")); Map params = createBaseParams(); params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString( - InstantiationUtil.serializeObject(new FallbackReadFileStoreTable(main, fallback)))); + InstantiationUtil.serializeObject(new FallbackReadFileStoreTable(main, fallback, true)))); PaimonJniScanner scanner = new PaimonJniScanner(1024, params); Method initTable = PaimonJniScanner.class.getDeclaredMethod("initTable"); initTable.setAccessible(true); @@ -181,7 +181,8 @@ public void testOldFeSerializedAsyncThresholdIsRejectedInEveryChild() throws Exc FileStoreTable main = serializableFileStoreTable(Collections.emptyMap()); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), "2 GB")); - for (Table configuredTable : Arrays.asList(visible, new FallbackReadFileStoreTable(main, fallback))) { + for (Table configuredTable : Arrays.asList( + visible, new FallbackReadFileStoreTable(main, fallback, true))) { Map params = createBaseParams(); params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString( InstantiationUtil.serializeObject(configuredTable))); @@ -203,7 +204,7 @@ public void testOldFeSerializedSystemSourceRejectsZeroSplitTarget() throws Excep FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "0 B")); Table filesTable = SystemTableLoader.load( - "files", new FallbackReadFileStoreTable(main, fallback)); + "files", new FallbackReadFileStoreTable(main, fallback, true)); Map params = createBaseParams(); params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString( InstantiationUtil.serializeObject(filesTable))); @@ -224,7 +225,7 @@ public void testOldFeSerializedSystemSourceRejectsFallbackZeroReadBatch() throws FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.READ_BATCH_SIZE.key(), "0")); Table readerBackedSystemTable = SystemTableLoader.load( - "audit_log", new FallbackReadFileStoreTable(main, fallback)); + "audit_log", new FallbackReadFileStoreTable(main, fallback, true)); Map params = createBaseParams(); params.put("serialized_table", Base64.getUrlEncoder().withoutPadding().encodeToString( InstantiationUtil.serializeObject(readerBackedSystemTable))); @@ -246,10 +247,10 @@ public void testBackendManifestCapReachesHiddenFallbackPlanner() { Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "8")); Table safe = PaimonJniScanner.applyBackendManifestParallelism( - new FallbackReadFileStoreTable(main, fallback), "8", 4); + new FallbackReadFileStoreTable(main, fallback, true), "8", 4); Assert.assertTrue(safe instanceof FallbackReadFileStoreTable); - Assert.assertEquals("4", ((FallbackReadFileStoreTable) safe).fallback() + Assert.assertEquals("4", ((FallbackReadFileStoreTable) safe).other() .options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -260,9 +261,9 @@ public void testAdvertisedFeCapStillChecksSerializedChildren() { Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "200")); Table safe = PaimonJniScanner.applyBackendManifestParallelism( - new FallbackReadFileStoreTable(main, fallback), "32", 64); + new FallbackReadFileStoreTable(main, fallback, true), "32", 64); - Assert.assertEquals("32", ((FallbackReadFileStoreTable) safe).fallback() + Assert.assertEquals("32", ((FallbackReadFileStoreTable) safe).other() .options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -277,11 +278,11 @@ public void testOldFeManifestBackstopKeepsStableMaximum() { Table safeVisible = PaimonJniScanner.applyBackendManifestParallelism( visible, null, 512); Table safeFallback = PaimonJniScanner.applyBackendManifestParallelism( - new FallbackReadFileStoreTable(main, fallback), null, 512); + new FallbackReadFileStoreTable(main, fallback, true), null, 512); Assert.assertEquals("256", safeVisible.options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assert.assertEquals("256", ((FallbackReadFileStoreTable) safeFallback).fallback() + Assert.assertEquals("256", ((FallbackReadFileStoreTable) safeFallback).other() .options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -301,20 +302,20 @@ public void testFallbackManifestParallelismIsCappedPerBranch() { CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1")); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128")); - Table pair = new FallbackReadFileStoreTable(main, fallback); + Table pair = new FallbackReadFileStoreTable(main, fallback, true); FallbackReadFileStoreTable unchanged = (FallbackReadFileStoreTable) PaimonJniScanner.applyBackendManifestParallelism(pair, "128", 128); Assert.assertEquals("1", unchanged.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assert.assertEquals("128", unchanged.fallback().options() + Assert.assertEquals("128", unchanged.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); FallbackReadFileStoreTable capped = (FallbackReadFileStoreTable) PaimonJniScanner.applyBackendManifestParallelism(pair, "128", 64); Assert.assertEquals("1", capped.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assert.assertEquals("64", capped.fallback().options() + Assert.assertEquals("64", capped.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -329,7 +330,7 @@ public void testBackendCapTraversesPrivilegeDelegate() { new Class[] {PrivilegeChecker.class}, (proxy, method, args) -> null); FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback), checker, + new FallbackReadFileStoreTable(main, fallback, true), checker, Identifier.create("db", "table")); Table safe = PaimonJniScanner.applyBackendManifestParallelism( @@ -342,7 +343,7 @@ public void testBackendCapTraversesPrivilegeDelegate() { FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) planningTable; Assert.assertEquals("1", pair.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assert.assertEquals("64", pair.fallback().options() + Assert.assertEquals("64", pair.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -353,7 +354,7 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128")); Table wrapper = SystemTableLoader.load( - "partitions", new FallbackReadFileStoreTable(main, fallback)); + "partitions", new FallbackReadFileStoreTable(main, fallback, true)); Table safe = PaimonJniScanner.applyBackendManifestParallelism( wrapper, null, 64); @@ -363,7 +364,7 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex Assert.assertEquals("1", pair.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assert.assertEquals("64", pair.fallback().options() + Assert.assertEquals("64", pair.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @@ -378,7 +379,7 @@ public void testSystemWrapperExposesSafeFallbackBehindPrivilegeDelegate() throws new Class[] {PrivilegeChecker.class}, (proxy, method, args) -> null); FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback), checker, + new FallbackReadFileStoreTable(main, fallback, true), checker, Identifier.create("db", "table")); Table wrapper = SystemTableLoader.load("partitions", privileged); @@ -861,6 +862,11 @@ public String[] tempDirs() { return tempDirs; } + @Override + public String pickTempDir() { + return tempDirs.length == 0 ? null : tempDirs[0]; + } + @Override public FileIOChannel.Enumerator createChannelEnumerator() { throw new UnsupportedOperationException(); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 6db5a09592ac92..2c963cd6a642ed 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -542,9 +542,11 @@ static Catalog createHmsCatalog(CatalogContext catalogContext, HadoopAuthenticat fileIO.checkOrMkdirs(warehousePath); String clientClass = options.get(HiveCatalogOptions.METASTORE_CLIENT_CLASS); Catalog catalog = hmsAuth == null - ? new HiveCatalog(fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString()) + ? new HiveCatalog(fileIO, hiveConf, clientClass, catalogContext, + warehousePath.toUri().toString()) : hmsAuth.doAs(() -> new HiveCatalog( - fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString())); + fileIO, hiveConf, clientClass, catalogContext, + warehousePath.toUri().toString())); catalog = PaimonHmsClientPool.install(catalog, hmsAuth); catalog = PaimonHmsCatalog.install(catalog, properties, storageHadoopConfig); catalog = CachingCatalog.tryToCreate(catalog, options); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java index 54fce689de34b6..94b05c8563fc0e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonReaderOptions.java @@ -28,6 +28,8 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.utils.ChainTableUtils; +import org.apache.paimon.utils.StringUtils; import java.util.Collections; import java.util.LinkedHashMap; @@ -218,10 +220,11 @@ private static Table normalizeManifestParallelism( FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) table; FileStoreTable main = normalizeManifestParallelism( pair.wrapped(), safeBound, materializeAbsent); - FileStoreTable fallback = normalizeManifestParallelism( - pair.fallback(), safeBound, materializeAbsent); - return main == pair.wrapped() && fallback == pair.fallback() - ? table : new FallbackReadFileStoreTable(main, fallback); + FileStoreTable other = normalizeManifestParallelism( + pair.other(), safeBound, materializeAbsent); + return main == pair.wrapped() && other == pair.other() + ? table : new FallbackReadFileStoreTable( + main, other, isWrappedFirst(pair)); } if (table instanceof DelegatedFileStoreTable) { @@ -270,12 +273,26 @@ private static FileStoreTable normalizeManifestParallelism( (Table) table, safeBound, materializeAbsent); } + static boolean isWrappedFirst(FallbackReadFileStoreTable table) { + Map options = table.options(); + // Match FileStoreTableFactory's construction order. Paimon does not expose wrappedFirst. + if (ChainTableUtils.isChainTable(options)) { + return true; + } + if (!StringUtils.isNullOrWhitespaceOnly( + options.get(CoreOptions.SCAN_FALLBACK_BRANCH.key()))) { + return true; + } + return StringUtils.isNullOrWhitespaceOnly( + options.get(CoreOptions.SCAN_PRIMARY_BRANCH.key())); + } + public static void validateEffectiveTable(Table table) { validateEffectiveTableOptions(table.options()); if (table instanceof FallbackReadFileStoreTable) { // The fallback scan plans its private child independently, so the visible main options // cannot prove that every manifest executor input is safe. - validateEffectiveTable(((FallbackReadFileStoreTable) table).fallback()); + validateEffectiveTable(((FallbackReadFileStoreTable) table).other()); } if (table instanceof DelegatedFileStoreTable) { // Privilege and other supported delegates can hide a fallback planner behind their @@ -290,7 +307,7 @@ public static void validateEffectivePlanningTable(Table table) { validateIfPresentForRuntime(table.options(), CoreOptions.SCAN_MANIFEST_PARALLELISM.key()); validateIfPresent(table.options(), CoreOptions.SCAN_PLAN_SORT_PARTITION.key()); if (table instanceof FallbackReadFileStoreTable) { - validateEffectivePlanningTable(((FallbackReadFileStoreTable) table).fallback()); + validateEffectivePlanningTable(((FallbackReadFileStoreTable) table).other()); } if (table instanceof DelegatedFileStoreTable) { validateEffectivePlanningTable(((DelegatedFileStoreTable) table).wrapped()); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 30c109dfb2fa14..e95f25277ce6ab 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -768,7 +768,7 @@ && hasTrustworthyLimitAccounting(table) // Process DataSplits for (DataSplit dataSplit : dataSplits) { if (isCountPushdownSplit(countPushdown, dataSplit)) { - countSum += dataSplit.mergedRowCount(); + countSum += dataSplit.mergedRowCount().getAsLong(); if (countRepresentative == null) { countRepresentative = dataSplit; } @@ -1232,7 +1232,7 @@ static void authorizeDeferredScan(FileStoreTable dataTable) { if (undecorated instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable fallbackReadTable = (FallbackReadFileStoreTable) undecorated; authorizeBranch(fallbackReadTable.wrapped()); - authorizeBranch(fallbackReadTable.fallback()); + authorizeBranch(fallbackReadTable.other()); return; } authorizeBranch(undecorated); @@ -1294,7 +1294,8 @@ static FileStoreTable pinCatalogSnapshot(FileStoreTable catalogLessTable, FileSt FallbackReadFileStoreTable targetPair = (FallbackReadFileStoreTable) target; return new FallbackReadFileStoreTable( pinCatalogSnapshotBranch(targetPair.wrapped(), sourcePair.wrapped()), - pinCatalogSnapshotBranch(targetPair.fallback(), sourcePair.fallback())); + pinCatalogSnapshotBranch(targetPair.other(), sourcePair.other()), + PaimonReaderOptions.isWrappedFirst(targetPair)); } return pinCatalogSnapshotBranch(target, source); } @@ -1337,7 +1338,8 @@ static FileStoreTable dropCatalogLoader(FileStoreTable dataTable) { FallbackReadFileStoreTable fallbackReadTable = (FallbackReadFileStoreTable) undecorated; return new FallbackReadFileStoreTable( rebuildWithoutCatalogLoader(fallbackReadTable.wrapped()), - rebuildWithoutCatalogLoader(fallbackReadTable.fallback())); + rebuildWithoutCatalogLoader(fallbackReadTable.other()), + PaimonReaderOptions.isWrappedFirst(fallbackReadTable)); } return rebuildWithoutCatalogLoader(undecorated); } @@ -1461,12 +1463,12 @@ private PaimonScanRange buildJniScanRange(Split split, String defaultFileFormat, * Whether a {@link DataSplit} contributes a precomputed COUNT(*)-pushdown row count: true iff count * pushdown is active for this scan AND the split's merged (post-merge / post-deletion-vector) row * count is precomputed by the paimon SDK. Mirrors legacy {@code PaimonScanNode}'s count gate - * ({@code applyCountPushdown && dataSplit.mergedRowCountAvailable()}, the FIRST routing arm). + * ({@code applyCountPushdown && dataSplit.mergedRowCount().isPresent()}, the FIRST routing arm). * Extracted as a pure static so the correctness-critical count routing decision is unit-testable * with a real {@link DataSplit}, like {@link #shouldUseNativeReader}. */ static boolean isCountPushdownSplit(boolean countPushdown, DataSplit dataSplit) { - return countPushdown && dataSplit.mergedRowCountAvailable(); + return countPushdown && dataSplit.mergedRowCount().isPresent(); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/FakePaimonTable.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/FakePaimonTable.java index 382bd1f530ad32..0b41af72641ae7 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/FakePaimonTable.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/FakePaimonTable.java @@ -27,7 +27,9 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.StreamWriteBuilder; +import org.apache.paimon.table.source.FullTextSearchBuilder; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.VectorSearchBuilder; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.SimpleFileReader; @@ -207,6 +209,11 @@ public void rollbackTo(String tagName) { throw new UnsupportedOperationException(); } + @Override + public void rollbackSchema(long schemaId) { + throw new UnsupportedOperationException(); + } + @Override public void createBranch(String branchName) { throw new UnsupportedOperationException(); @@ -217,11 +224,26 @@ public void createBranch(String branchName, String tagName) { throw new UnsupportedOperationException(); } + @Override + public void createBranch(String branchName, boolean forceCreate) { + throw new UnsupportedOperationException(); + } + + @Override + public void createBranch(String branchName, String tagName, boolean forceCreate) { + throw new UnsupportedOperationException(); + } + @Override public void deleteBranch(String branchName) { throw new UnsupportedOperationException(); } + @Override + public void renameBranch(String branchName, String targetBranchName) { + throw new UnsupportedOperationException(); + } + @Override public void fastForward(String branchName) { throw new UnsupportedOperationException(); @@ -237,6 +259,16 @@ public ExpireSnapshots newExpireChangelog() { throw new UnsupportedOperationException(); } + @Override + public VectorSearchBuilder newVectorSearchBuilder() { + throw new UnsupportedOperationException(); + } + + @Override + public FullTextSearchBuilder newFullTextSearchBuilder() { + throw new UnsupportedOperationException(); + } + @Override public ReadBuilder newReadBuilder() { throw new UnsupportedOperationException(); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java index 30592f07c0b96a..b45f45ac5c9ec1 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; @@ -269,7 +270,7 @@ public void sysTableHandleBuildsTheWrapperOverThePeeledFallbackPair(@TempDir Pat // branch alone through the pair's inherited newSnapshotReader() and silently drops every // fallback-only partition. FileStoreTable[] branches = fallbackPairWithNewerGenerationOnDisk(warehouse, "fb_sys"); - FileStoreTable pair = new FallbackReadFileStoreTable(branches[0], branches[1]); + FileStoreTable pair = new FallbackReadFileStoreTable(branches[0], branches[1], true); FileStoreTable decorated = PrivilegedFileStoreTable.wrap(pair, new AllGrantedPrivilegeChecker(), Identifier.create("db", "tbl")); PaimonTableHandle baseHandle = dataHandle(); @@ -283,16 +284,13 @@ public void sysTableHandleBuildsTheWrapperOverThePeeledFallbackPair(@TempDir Pat Assertions.assertTrue(((ReadOptimizedTable) sysHandle.getPaimonTable()).newScan() instanceof FallbackReadFileStoreTable.FallbackReadScan, "the FE must plan tbl$ro over BOTH branches"); - // What that stands for: leaving the decorator on takes newScan down its single-branch path. - Assertions.assertFalse(((ReadOptimizedTable) SystemTableLoader.load("ro", decorated)).newScan() - instanceof FallbackReadFileStoreTable.FallbackReadScan); } @Test public void runtimeCapKeepsFallbackImmediateUnderReadOptimizedWrapper(@TempDir Path warehouse) throws Exception { FileStoreTable[] branches = fallbackPairWithNewerGenerationOnDisk(warehouse, "fb_runtime_cap"); - FileStoreTable pair = new FallbackReadFileStoreTable(branches[0], branches[1]); + FileStoreTable pair = new FallbackReadFileStoreTable(branches[0], branches[1], true); FileStoreTable decorated = PrivilegedFileStoreTable.wrap(pair, new AllGrantedPrivilegeChecker(), Identifier.create("db", "tbl")); FileStoreTable configured = decorated.copyWithoutTimeTravel(Collections.singletonMap( @@ -377,7 +375,7 @@ public void pinsEachFallbackBranchToItsOwnCatalogVisibleSnapshot(@TempDir Path w onDisk.copyWithoutTimeTravel(mainOptions), catalogEnvironment(mainCatalog)); FileStoreTable fallback = withCatalogEnvironment( onDisk.copyWithoutTimeTravel(fallbackOptions), catalogEnvironment(fallbackCatalog)); - FileStoreTable catalogPair = new FallbackReadFileStoreTable(main, fallback); + FileStoreTable catalogPair = new FallbackReadFileStoreTable(main, fallback, true); FileStoreTable pinned = PaimonScanPlanProvider.pinCatalogSnapshot( PaimonScanPlanProvider.dropCatalogLoader(catalogPair), catalogPair); @@ -385,7 +383,7 @@ public void pinsEachFallbackBranchToItsOwnCatalogVisibleSnapshot(@TempDir Path w Assertions.assertTrue(pinned instanceof FallbackReadFileStoreTable); FallbackReadFileStoreTable pinnedPair = (FallbackReadFileStoreTable) pinned; Assertions.assertEquals("2", pinnedPair.wrapped().options().get("scan.snapshot-id")); - Assertions.assertEquals("1", pinnedPair.fallback().options().get("scan.snapshot-id"), + Assertions.assertEquals("1", pinnedPair.other().options().get("scan.snapshot-id"), "the fallback branch must use its own catalog pointer, not its newest snapshot file"); Assertions.assertTrue(mainCatalog.loadSnapshotCalls > 0); Assertions.assertTrue(fallbackCatalog.loadSnapshotCalls > 0); @@ -497,7 +495,7 @@ public void fallbackBranchKeepsTheGenerationTheFeCaptured(@TempDir Path warehous FileStoreTable[] captured = fallbackPairWithNewerGenerationOnDisk(warehouse, "fb_generation"); FileStoreTable forBackend = PaimonScanPlanProvider.dropCatalogLoader( - new FallbackReadFileStoreTable(captured[0], captured[1])); + new FallbackReadFileStoreTable(captured[0], captured[1], true)); assertPairMatchesTheFeGeneration(forBackend, captured[0], captured[1]); } @@ -511,7 +509,7 @@ public void fallbackBranchSurvivesAPaimonTableDecorator(@TempDir Path warehouse) // while the FE keeps planning the wrapper and can still emit one. FileStoreTable[] captured = fallbackPairWithNewerGenerationOnDisk(warehouse, "fb_decorated"); FileStoreTable decorated = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(captured[0], captured[1]), + new FallbackReadFileStoreTable(captured[0], captured[1], true), new AllGrantedPrivilegeChecker(), Identifier.create("db", "tbl")); assertPairMatchesTheFeGeneration(PaimonScanPlanProvider.dropCatalogLoader(decorated), @@ -532,13 +530,16 @@ public void authorizesBothBranchesOfAFallbackPair(@TempDir Path warehouse) { Map fallbackOptions = queryAuthEnabled(); fallbackOptions.put("branch", "fb"); FileStoreTable main = newTable(warehouse, - new CatalogEnvironment(mainIdentifier, null, () -> catalog, null, null, true), + new CatalogEnvironment(mainIdentifier, null, () -> catalog, null, null, + null, true, false), queryAuthEnabled(), C1); FileStoreTable fallback = newTable(warehouse, - new CatalogEnvironment(fallbackIdentifier, null, () -> catalog, null, null, true), + new CatalogEnvironment(fallbackIdentifier, null, () -> catalog, null, null, + null, true, false), fallbackOptions, C1); - PaimonScanPlanProvider.authorizeDeferredScan(new FallbackReadFileStoreTable(main, fallback)); + PaimonScanPlanProvider.authorizeDeferredScan( + new FallbackReadFileStoreTable(main, fallback, true)); Assertions.assertEquals(Arrays.asList(mainIdentifier, fallbackIdentifier), catalog.authCalls, "both branches must be authorized, each against its own identifier"); @@ -582,7 +583,8 @@ public void relationOptionsResolveFallbackSnapshotBeforeDroppingLoaders(@TempDir withCatalogEnvironment(onDisk.copyWithoutTimeTravel(mainOptions), catalogEnvironment(mainCatalog)), withCatalogEnvironment(onDisk.copyWithoutTimeTravel(fallbackOptions), - catalogEnvironment(fallbackCatalog))); + catalogEnvironment(fallbackCatalog)), + true); Map pinned = PaimonScanParams.markAsOptions( Collections.singletonMap("scan.snapshot-id", "2")); PaimonTableHandle handle = sysHandle("ro", pair).withScanOptions(pinned); @@ -595,7 +597,7 @@ public void relationOptionsResolveFallbackSnapshotBeforeDroppingLoaders(@TempDir FallbackReadFileStoreTable backendPair = (FallbackReadFileStoreTable) wrapped.get(forBackend); Assertions.assertEquals("2", backendPair.wrapped().options().get("scan.snapshot-id")); - Assertions.assertEquals("1", backendPair.fallback().options().get("scan.snapshot-id")); + Assertions.assertEquals("1", backendPair.other().options().get("scan.snapshot-id")); Assertions.assertTrue(fallbackCatalog.loadSnapshotCalls > 0, "fallback translation must consult its catalog before the loader is removed"); } @@ -683,7 +685,7 @@ public void incrementalRangeIsNotBoundOnAFallbackBranchPair(@TempDir Path wareho Map fallbackOptions = new HashMap<>(); fallbackOptions.put("branch", "fb"); FileStoreTable fallback = newTable(warehouse, catalogEnvironment(catalog), fallbackOptions, C1); - FileStoreTable pair = new FallbackReadFileStoreTable(main, fallback); + FileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); Map range = new HashMap<>(); range.put("incremental-between-timestamp", @@ -743,8 +745,8 @@ private static Map queryAuthEnabled() { } private static CatalogEnvironment catalogEnvironment(VersionManagedCatalog catalog) { - return new CatalogEnvironment(Identifier.create("db", "tbl"), null, () -> catalog, null, null, - catalog.supportsVersionManagement()); + return new CatalogEnvironment(Identifier.create("db", "tbl"), null, () -> catalog, + null, null, null, catalog.supportsVersionManagement(), false); } /** A table object with the given schema; no files are written, so nothing touches the disk. */ @@ -833,11 +835,11 @@ private static void assertPairMatchesTheFeGeneration(FileStoreTable forBackend, "the pair itself must reach the BE, or a fallback split has no reader there"); FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) forBackend; // The fallback branch must stay on F1 - not the c2 generation sitting on the filesystem. - Assertions.assertEquals(fallback.rowType(), pair.fallback().rowType()); + Assertions.assertEquals(fallback.rowType(), pair.other().rowType()); Assertions.assertEquals(main.rowType(), pair.wrapped().rowType()); // And neither branch may carry the loader that drags the metastore stack onto the BE. Assertions.assertNull(pair.wrapped().catalogEnvironment().catalogLoader()); - Assertions.assertNull(pair.fallback().catalogEnvironment().catalogLoader()); + Assertions.assertNull(pair.other().catalogEnvironment().catalogLoader()); } /** Decodes {@code paimon.serialized_table} the way {@code PaimonJniScanner#initTable} does. */ @@ -910,12 +912,12 @@ public Optional loadSnapshot(Identifier identifier) { } @Override - public List authTableQuery(Identifier identifier, List select) { + public TableQueryAuthResult authTableQuery(Identifier identifier, List select) { authCalls.add(identifier); if (authDenialMessage != null) { throw new RuntimeException(authDenialMessage); } - return Collections.emptyList(); + return new TableQueryAuthResult(Collections.emptyList(), Collections.emptyMap()); } @Override diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java index 56d0456f8747de..7770629fca3c59 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java @@ -88,7 +88,7 @@ private static PaimonTableHandle dtRegionHandle(FakePaimonTable table) { private static Partition partition(Map spec, long recordCount, long fileSizeInBytes, long lastFileCreationTime) { return new Partition(spec, recordCount, fileSizeInBytes, /*fileCount*/ 1, lastFileCreationTime, - /*done*/ true); + /*totalBuckets*/ 1, /*done*/ true); } @Test @@ -186,7 +186,7 @@ public void listPartitionsCarriesFileCount() { // lastFileCreationTime, done). ops.partitions = Collections.singletonList(new Partition( spec, /*recordCount*/ 42L, /*fileSizeInBytes*/ 1024L, /*fileCount*/ 7L, - /*lastFileCreationTime*/ 1700000000000L, /*done*/ true)); + /*lastFileCreationTime*/ 1700000000000L, /*totalBuckets*/ 3, /*done*/ true)); ConnectorPartitionInfo info = metadataWith(ops) .listPartitions(null, dtRegionHandle(table), Optional.empty()).get(0); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java index 831a5d59601bc1..5433ac82f2382b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java @@ -92,7 +92,7 @@ private static FakePaimonTable regionTable() { private static Partition partition(String regionValue) { Map spec = new LinkedHashMap<>(); spec.put("region", regionValue); - return new Partition(spec, 1L, 1L, 1, 1L, true); + return new Partition(spec, 1L, 1L, 1, 1L, 1, true); } private static long loadCount(RecordingPaimonCatalogOps ops) { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java index 8d30fa4364a5f3..b85499d9e99fc6 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java @@ -211,7 +211,8 @@ public void listPartitionNamesEntersAuthenticatorForBothResolveAndListPartitions Map spec = new LinkedHashMap<>(); spec.put("region", "cn"); ops.partitions = Collections.singletonList( - new Partition(spec, 1L, 1L, /*fileCount*/ 1, 1L, /*done*/ true)); + new Partition(spec, 1L, 1L, /*fileCount*/ 1, 1L, + /*totalBuckets*/ 1, /*done*/ true)); RecordingConnectorContext ctx = new RecordingConnectorContext(); PaimonTableHandle handle = new PaimonTableHandle( diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonHmsCatalogTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonHmsCatalogTest.java index 62e42a947821ae..8490469123187b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonHmsCatalogTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonHmsCatalogTest.java @@ -104,7 +104,8 @@ public void formatTableKeepsPreparationUnderStorageUserButPersistsHmsOwner() thr HiveConf hiveConf = new HiveConf(); hiveConf.set("hive.metastore.sasl.enabled", "true"); HiveCatalog hiveCatalog = new HiveCatalog( - new RecordingFileIO(), hiveConf, null, options, "record:///warehouse"); + new RecordingFileIO(), hiveConf, null, + CatalogContext.create(options, hiveConf), "record:///warehouse"); AtomicReference createdTable = new AtomicReference<>(); AtomicReference rpcUser = new AtomicReference<>(); IMetaStoreClient client = (IMetaStoreClient) Proxy.newProxyInstance( diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonReaderOptionsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonReaderOptionsTest.java index af643eda61a2a6..314b778c0ffb40 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonReaderOptionsTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonReaderOptionsTest.java @@ -124,16 +124,16 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { FileStoreTable safeEmpty = (FileStoreTable) PaimonReaderOptions.runtimeSafeTable(empty, 512); FallbackReadFileStoreTable explicitThenEmpty = (FallbackReadFileStoreTable) PaimonReaderOptions.runtimeSafeTable( - new FallbackReadFileStoreTable(explicit, empty), 512); + new FallbackReadFileStoreTable(explicit, empty, true), 512); FallbackReadFileStoreTable emptyThenExplicit = (FallbackReadFileStoreTable) PaimonReaderOptions.runtimeSafeTable( - new FallbackReadFileStoreTable(empty, explicit), 512); + new FallbackReadFileStoreTable(empty, explicit, true), 512); PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( PrivilegeChecker.class.getClassLoader(), new Class[] {PrivilegeChecker.class}, (proxy, method, args) -> null); FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(explicit, empty), checker, + new FallbackReadFileStoreTable(explicit, empty, true), checker, Identifier.create("db", "table")); Table normalizedDelegate = PaimonReaderOptions.runtimeSafeTable(privileged, 512); @@ -141,22 +141,22 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); Assertions.assertEquals("1", explicitThenEmpty.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assertions.assertEquals("256", explicitThenEmpty.fallback().options() + Assertions.assertEquals("256", explicitThenEmpty.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); Assertions.assertEquals("256", emptyThenExplicit.wrapped().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assertions.assertEquals("1", emptyThenExplicit.fallback().options() + Assertions.assertEquals("1", emptyThenExplicit.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); Assertions.assertInstanceOf(FallbackReadFileStoreTable.class, normalizedDelegate); Assertions.assertEquals("256", ((FallbackReadFileStoreTable) normalizedDelegate) - .fallback().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); + .other().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @Test void testSystemSourceKeepsFallbackAsOutermostPlanningDecorator() { FileStoreTable main = newFileStoreTable("main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable("fallback", Collections.emptyMap()); - FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( PrivilegeChecker.class.getClassLoader(), new Class[] {PrivilegeChecker.class}, @@ -168,6 +168,20 @@ void testSystemSourceKeepsFallbackAsOutermostPlanningDecorator() { Assertions.assertSame(pair, PaimonTableDecorators.unwrapToFallbackOrBase(privileged)); } + @Test + void testFallbackWrapperOrderMatchesPaimonFactoryPrecedence() { + FileStoreTable other = newFileStoreTable("other", Collections.emptyMap()); + Assertions.assertTrue(PaimonReaderOptions.isWrappedFirst(new FallbackReadFileStoreTable( + newFileStoreTable("fallback", ImmutableMap.of( + CoreOptions.SCAN_FALLBACK_BRANCH.key(), "fallback", + CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")), other, true))); + Assertions.assertFalse(PaimonReaderOptions.isWrappedFirst(new FallbackReadFileStoreTable( + newFileStoreTable("primary", ImmutableMap.of( + CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary")), other, false))); + Assertions.assertTrue(PaimonReaderOptions.isWrappedFirst(new FallbackReadFileStoreTable( + newFileStoreTable("default", Collections.emptyMap()), other, true))); + } + @Test void testUnnormalizedEffectiveTableCannotGrowPaimonGlobalPool() { int localCapacity = Runtime.getRuntime().availableProcessors(); @@ -206,7 +220,7 @@ void testRejectUnsafeHiddenFallbackTableAfterCopy() { FileStoreTable main = newFileStoreTable("main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable( "fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); - Table fallbackReadTable = new FallbackReadFileStoreTable(main, fallback); + Table fallbackReadTable = new FallbackReadFileStoreTable(main, fallback, true); Assertions.assertThrows(IllegalArgumentException.class, () -> PaimonScanParams.applyOptions(fallbackReadTable, Collections.emptyMap())); @@ -217,7 +231,7 @@ void testSafeRelationOptionOverridesUnsafeHiddenFallbackTable() { FileStoreTable main = newFileStoreTable("main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable( "fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); - Table fallbackReadTable = new FallbackReadFileStoreTable(main, fallback); + Table fallbackReadTable = new FallbackReadFileStoreTable(main, fallback, true); Assertions.assertDoesNotThrow(() -> PaimonScanParams.applyOptions( fallbackReadTable, ImmutableMap.of("scan.manifest.parallelism", "1"))); @@ -232,12 +246,12 @@ void testRuntimeSafeTableCapsHiddenFallbackPlanner() { ImmutableMap.of("scan.manifest.parallelism", String.valueOf(localCapacity + 1))); Table safe = PaimonReaderOptions.runtimeSafeTable( - new FallbackReadFileStoreTable(main, fallback)); + new FallbackReadFileStoreTable(main, fallback, true)); Assertions.assertInstanceOf(FallbackReadFileStoreTable.class, safe); FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable) safe; Assertions.assertEquals(String.valueOf(localCapacity), - pair.fallback().options().get("scan.manifest.parallelism")); + pair.other().options().get("scan.manifest.parallelism")); } @Test @@ -251,7 +265,7 @@ void testBackendManifestCapUsesExecutionCeilingNotSmallestBranch() { Math.min(Runtime.getRuntime().availableProcessors(), PaimonReaderOptions.MAX_MANIFEST_PARALLELISM), PaimonReaderOptions.backendManifestParallelismCap( - new FallbackReadFileStoreTable(main, fallback)).getAsInt()); + new FallbackReadFileStoreTable(main, fallback, true)).getAsInt()); } @Test @@ -259,7 +273,7 @@ void testRejectUnsafeFallbackHiddenByPrivilegeDelegate() { FileStoreTable main = newFileStoreTable("privileged_main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable( "privileged_fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); - FileStoreTable fallbackReadTable = new FallbackReadFileStoreTable(main, fallback); + FileStoreTable fallbackReadTable = new FallbackReadFileStoreTable(main, fallback, true); PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( PrivilegeChecker.class.getClassLoader(), new Class[] {PrivilegeChecker.class}, diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanMetricsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanMetricsTest.java index 3eec098a2f28f2..4b62f109241cbe 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanMetricsTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanMetricsTest.java @@ -41,7 +41,7 @@ public void harvestRendersRecordedScanMetrics() { // the LAST_* gauges. A mutation that drops the harvest returns empty. PaimonMetricRegistry registry = new PaimonMetricRegistry(); ScanMetrics metrics = new ScanMetrics(registry, "mydb.mytbl"); - metrics.reportScan(new ScanStats(2_000_000L, 5L, 3L, 7L)); + metrics.reportScan(new ScanStats(2_000_000L, -1L, 5L, 3L, 7L)); Optional profile = PaimonScanMetrics.harvest(registry, "mydb.mytbl", "Table Scan (mydb.mytbl)"); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index e66a38c021d243..94b2e68c9447da 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -98,6 +98,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; /** * Tests for {@link PaimonScanPlanProvider#resolveTable}, pinning the transient-Table reload @@ -558,7 +559,7 @@ public void fallbackLimitDoesNotExposeStaleFallbackRows(@TempDir Path warehouse) } } - FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback); + FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); FileStoreTable decorated = PrivilegedFileStoreTable.wrap( pair, new AllGrantedPrivilegeChecker(), mainId); for (Table planningTable : Arrays.asList(pair, decorated)) { @@ -1468,7 +1469,7 @@ public void encodeSplitAlwaysUsesJavaSerializationForDataSplit(@TempDir Path war "precondition: nativeBinaryEncode really is the paimon::Split::Deserialize format"); } - /** A non-DataSplit Split (the only abstract method is rowCount(); Split is Serializable). */ + /** A non-DataSplit Split used to verify the Java serialization route. */ private static final class NonDataSplitStub implements Split { private static final long serialVersionUID = 1L; @@ -1476,6 +1477,11 @@ private static final class NonDataSplitStub implements Split { public long rowCount() { return 0; } + + @Override + public OptionalLong mergedRowCount() { + return OptionalLong.empty(); + } } @Test @@ -1497,13 +1503,13 @@ public void countPushdownSplitDetectedOnlyWhenAggCountAndMergedCountAvailable( // (post-merge / post-deletion-vector) row count, so a COUNT(*) over it can be served from // metadata instead of materializing rows. DataSplit dataSplit = buildRealDataSplit(warehouse); - Assertions.assertTrue(dataSplit.mergedRowCountAvailable(), + Assertions.assertTrue(dataSplit.mergedRowCount().isPresent(), "precondition: a freshly written PK split has a precomputed merged row count"); - Assertions.assertEquals(2L, dataSplit.mergedRowCount(), "two rows were written"); + Assertions.assertEquals(2L, dataSplit.mergedRowCount().getAsLong(), "two rows were written"); // WHY: the count branch must fire ONLY when BOTH the agg is COUNT (countPushdown) AND the SDK // precomputed the post-merge count β€” mirrors legacy `applyCountPushdown && - // dataSplit.mergedRowCountAvailable()`. MUTATION: dropping `countPushdown &&` (or hard-coding + // dataSplit.mergedRowCount().isPresent()`. MUTATION: dropping `countPushdown &&` (or hard-coding // the helper to false) -> one of these two assertions flips -> red. Assertions.assertTrue(PaimonScanPlanProvider.isCountPushdownSplit(true, dataSplit), "a count query over a split with a precomputed merged count must push the count down"); diff --git a/fe/pom.xml b/fe/pom.xml index e0aee6b61a77be..23629530d5307c 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -245,7 +245,7 @@ under the License. fe-grpc - 3.1.1 + 3.1.2 1.12.1 1.17.0 @@ -443,7 +443,7 @@ under the License. InstantiationUtil and BE deserializes it with the SAME paimon jar; a version mismatch silently breaks that FE->BE deserialization at runtime. These three MUST stay equal β€” do NOT override paimon.version per-module. --> - 1.3.1 + 1.4.2 3.4.4 17.0.0 From 188d7c93cee75ef575256cbaf3be54a67a1db5db Mon Sep 17 00:00:00 2001 From: suxiaogang Date: Tue, 1 Sep 2026 17:57:42 +0800 Subject: [PATCH 02/12] [feature](paimon) Add Paimon table write runtime ### What problem does this PR solve? Issue Number: close #65086 Related PR: #65868, #66612, #66810 Problem Summary: Add the BE Paimon sink, JNI writer backend, Java writer, commit payload transport, and worktree-local spill and memory lifecycle support required to write Paimon tables from master. ### Release note Support writing Apache Paimon tables through the native Doris execution pipeline. ### Check List (For Author) - Test: Static validation - BE clang-format/check-format and FE checkstyle; full build and tests are deferred until all forward-port picks are complete - Behavior changed: Yes, adds Paimon table writes - Does this need documentation: Yes, documentation can follow separately --- be/src/common/config.cpp | 5 + be/src/common/config.h | 4 + be/src/exec/operator/operator.cpp | 2 + .../operator/paimon_table_sink_operator.cpp | 83 ++ .../operator/paimon_table_sink_operator.h | 102 +++ .../pipeline/pipeline_fragment_context.cpp | 23 + .../paimon/ffi_paimon_write_backend.cpp | 34 + .../writer/paimon/ffi_paimon_write_backend.h | 36 + .../paimon/jni_paimon_write_backend.cpp | 569 ++++++++++++++ .../writer/paimon/jni_paimon_write_backend.h | 118 +++ .../paimon/paimon_jni_memory_manager.cpp | 312 ++++++++ .../writer/paimon/paimon_jni_memory_manager.h | 80 ++ .../writer/paimon/paimon_table_writer.cpp | 161 ++++ .../sink/writer/paimon/paimon_table_writer.h | 101 +++ .../sink/writer/paimon/paimon_write_backend.h | 108 +++ .../paimon/paimon_write_backend_factory.cpp | 44 ++ be/src/exec/spill/spill_file_manager.cpp | 169 ++++ be/src/exec/spill/spill_file_manager.h | 49 ++ be/src/runtime/runtime_state.h | 14 + be/src/util/jni-util.h | 3 + .../sink/paimon_jni_memory_manager_test.cpp | 64 ++ .../paimon/paimon_write_backend_test.cpp | 53 ++ be/test/vec/spill/spill_file_test.cpp | 205 ++++- fe/be-java-extensions/paimon-scanner/pom.xml | 14 + .../apache/doris/paimon/DorisIOManager.java | 392 ++++++++++ .../doris/paimon/DorisMemorySegmentPool.java | 59 ++ .../doris/paimon/GlobalIndexAssigner.java | 177 +++++ .../doris/paimon/PaimonArrowBatchAdapter.java | 210 +++++ .../doris/paimon/PaimonCommitCodec.java | 184 +++++ .../apache/doris/paimon/PaimonJniWriter.java | 735 ++++++++++++++++++ .../doris/paimon/PaimonWriteSchema.java | 189 +++++ .../doris/paimon/DorisIOManagerTest.java | 223 ++++++ .../doris/paimon/GlobalIndexAssignerTest.java | 39 + .../paimon/PaimonArrowBatchAdapterTest.java | 290 +++++++ .../doris/paimon/PaimonCommitCodecTest.java | 110 +++ .../doris/paimon/PaimonJniWriterTest.java | 226 ++++++ .../doris/paimon/PaimonWriteSchemaTest.java | 275 +++++++ fe/pom.xml | 10 + gensrc/thrift/DataSinks.thrift | 27 + gensrc/thrift/FrontendService.thrift | 2 + 40 files changed, 5486 insertions(+), 15 deletions(-) create mode 100644 be/src/exec/operator/paimon_table_sink_operator.cpp create mode 100644 be/src/exec/operator/paimon_table_sink_operator.h create mode 100644 be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp create mode 100644 be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h create mode 100644 be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp create mode 100644 be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h create mode 100644 be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp create mode 100644 be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h create mode 100644 be/src/exec/sink/writer/paimon/paimon_table_writer.cpp create mode 100644 be/src/exec/sink/writer/paimon/paimon_table_writer.h create mode 100644 be/src/exec/sink/writer/paimon/paimon_write_backend.h create mode 100644 be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp create mode 100644 be/test/exec/sink/paimon_jni_memory_manager_test.cpp create mode 100644 be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/DorisIOManager.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/DorisMemorySegmentPool.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/GlobalIndexAssigner.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonCommitCodec.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/DorisIOManagerTest.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/GlobalIndexAssignerTest.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonCommitCodecTest.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java create mode 100644 fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 03266539c4dc8e..1dde4ae40eb2dc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1719,6 +1719,11 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB /** Iceberg sink configurations **/ DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB +/** Paimon sink configurations **/ +DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB +DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, + [](int64_t bytes) -> bool { return bytes > 0; }); + // URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. // Each entry uses the format "=", and file_type must be one of: // local, hdfs, s3, http, broker. diff --git a/be/src/common/config.h b/be/src/common/config.h index 436a1878ef424a..a4478bbc76356f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1785,6 +1785,10 @@ DECLARE_mInt64(hive_sink_max_file_size); /** Iceberg sink configurations **/ DECLARE_mInt64(iceberg_sink_max_file_size); +/** Paimon sink configurations **/ +// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. +DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); + /** Paimon file system configurations **/ DECLARE_Strings(paimon_file_system_scheme_mappings); diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index 796cc1169691d9..358e43cdb42d75 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -64,6 +64,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -846,6 +847,7 @@ DECLARE_OPERATOR(OlapTableSinkV2LocalState) DECLARE_OPERATOR(HiveTableSinkLocalState) DECLARE_OPERATOR(TVFTableSinkLocalState) DECLARE_OPERATOR(IcebergTableSinkLocalState) +DECLARE_OPERATOR(PaimonTableSinkLocalState) DECLARE_OPERATOR(SpillIcebergTableSinkLocalState) DECLARE_OPERATOR(IcebergDeleteSinkLocalState) DECLARE_OPERATOR(IcebergMergeSinkLocalState) diff --git a/be/src/exec/operator/paimon_table_sink_operator.cpp b/be/src/exec/operator/paimon_table_sink_operator.cpp new file mode 100644 index 00000000000000..dace179ba2d9fb --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.cpp @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/operator/paimon_table_sink_operator.h" + +#include "common/logging.h" + +namespace doris { + +Status PaimonTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { + RETURN_IF_ERROR(Base::init(state, info)); + _writer = std::make_unique(info.tsink, _output_vexpr_ctxs); + return Status::OK(); +} + +Status PaimonTableSinkLocalState::open(RuntimeState* state) { + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_open_timer); + RETURN_IF_ERROR(Base::open(state)); + + auto& parent = _parent->cast(); + _output_vexpr_ctxs.resize(parent._output_vexpr_ctxs.size()); + for (size_t i = 0; i < _output_vexpr_ctxs.size(); ++i) { + RETURN_IF_ERROR(parent._output_vexpr_ctxs[i]->clone(state, _output_vexpr_ctxs[i])); + } + return _writer->open(state, operator_profile()); +} + +Status PaimonTableSinkLocalState::close(RuntimeState* state, Status exec_status) { + if (_closed) { + return Status::OK(); + } + + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_close_timer); + + Status final_status = exec_status; + if (_writer) { + Status writer_status = _writer->close(exec_status); + if (final_status.ok() && !writer_status.ok()) { + final_status = writer_status; + } + _writer.reset(); + } + + Status base_status = Base::close(state, final_status); + if (final_status.ok() && !base_status.ok()) { + final_status = base_status; + } + return final_status; +} + +Status PaimonTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool /*eos*/) { + auto& local_state = get_local_state(state); + SCOPED_TIMER(local_state.exec_time_counter()); + COUNTER_UPDATE(local_state.rows_input_counter(), static_cast(in_block->rows())); + + if (in_block->rows() == 0) { + return Status::OK(); + } + + // This is a synchronous SDK call. The LocalState is marked blockable, so + // the whole pipeline task (including open and close) runs on the blocking + // scheduler instead of occupying a regular pipeline worker. + DCHECK(local_state._writer); + return local_state._writer->write(state, *in_block); +} + +} // namespace doris diff --git a/be/src/exec/operator/paimon_table_sink_operator.h b/be/src/exec/operator/paimon_table_sink_operator.h new file mode 100644 index 00000000000000..9d07dc45390e24 --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/operator/operator.h" +#include "exec/sink/writer/paimon/paimon_table_writer.h" +#include "runtime/runtime_state.h" + +namespace doris { + +/// Paimon table sink operator. +/// +/// Each pipeline instance (LocalState) owns one PaimonTableWriter, which in +/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism +/// determines the number of concurrent Paimon writer sessions per table. +/// Paimon writes are synchronous: sink_impl() returns only after the SDK has +/// consumed the input Block. The LocalState is therefore always blockable so +/// that open, write, and close run on the pipeline blocking scheduler. +/// Doris-owned Arrow buffers remain under the query MemTracker, while Paimon +/// pages are allocated lazily under DorisMemorySegmentPool's fixed cap. The +/// sink uses the generic pipeline minimum reservation only as an admission +/// guard; it does not try to predict Paimon's future page demand. +/// +/// The upstream sink Exchange may reproduce Paimon's stateless HASH_FIXED +/// selector to establish unique writer ownership. The writer still passes +/// complete Blocks to the SDK, which independently computes partition and +/// bucket values for file writing; no routing column is appended to the row. +class PaimonTableSinkOperatorX; + +class PaimonTableSinkLocalState final : public PipelineXSinkLocalState { +public: + using Base = PipelineXSinkLocalState; + using Parent = PaimonTableSinkOperatorX; + ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState); + PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) + : Base(parent, state) {} + + Status init(RuntimeState* state, LocalSinkStateInfo& info) override; + Status open(RuntimeState* state) override; + Status close(RuntimeState* state, Status exec_status) override; + + [[nodiscard]] bool is_blockable() const override { return true; } + +private: + friend class PaimonTableSinkOperatorX; + + VExprContextSPtrs _output_vexpr_ctxs; + std::unique_ptr _writer; +}; + +class PaimonTableSinkOperatorX final : public DataSinkOperatorX { +public: + using Base = DataSinkOperatorX; + PaimonTableSinkOperatorX(int operator_id, const RowDescriptor& row_desc, + const std::vector& t_output_expr) + : Base(operator_id, 0, 0), _row_desc(row_desc), _t_output_expr(t_output_expr) {} + + Status init(const TDataSink& thrift_sink) override { + RETURN_IF_ERROR(Base::init(thrift_sink)); + DCHECK(thrift_sink.__isset.paimon_table_sink); + RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); + return Status::OK(); + } + + Status prepare(RuntimeState* state) override { + RETURN_IF_ERROR(Base::prepare(state)); + RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); + return VExpr::open(_output_vexpr_ctxs, state); + } + + Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; + +private: + friend class PaimonTableSinkLocalState; + + const RowDescriptor& _row_desc; + VExprContextSPtrs _output_vexpr_ctxs; + const std::vector& _t_output_expr; +}; + +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 2fa064c8a68e09..092eaa08f30801 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -88,6 +88,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -1376,6 +1377,14 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS output_exprs); break; } + case TDataSinkType::PAIMON_TABLE_SINK: { + if (!thrift_sink.__isset.paimon_table_sink) { + return Status::InternalError("Missing paimon table sink."); + } + _sink = std::make_shared(next_sink_operator_id(), row_desc, + output_exprs); + break; + } case TDataSinkType::JDBC_TABLE_SINK: { if (!thrift_sink.__isset.jdbc_table_sink) { return Status::InternalError("Missing data jdbc sink."); @@ -2526,6 +2535,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } _append_external_file_commit_data(req, ¶ms); + if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(), + pcm.end()); + } else if (!req.runtime_states.empty()) { + for (auto* rs : req.runtime_states) { + if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), + rs_pcm.begin(), rs_pcm.end()); + } + } + } + req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp new file mode 100644 index 00000000000000..a5abfcdc15c41c --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" + +namespace doris { + +Status FfiPaimonWriteBackend::open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::create_writer(std::unique_ptr*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::close() { + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h new file mode 100644 index 00000000000000..be833d53b79bcd --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +/// Placeholder for the future paimon-rust writer implementation. Keeping this +/// backend in the factory makes the integration boundary explicit without +/// introducing a BE commit contract that the Rust writer will not own. +class FfiPaimonWriteBackend final : public IPaimonWriteBackend { +public: + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::FFI; } +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp new file mode 100644 index 00000000000000..d46fb1d99f691f --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "exec/spill/spill_file_manager.h" +#include "format/arrow/arrow_block_convertor.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/defer_op.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" + +namespace doris { + +namespace { +constexpr std::string_view PAIMON_JNI_WRITER_IO_TMP_DIR = "paimon_jni_writer_io_tmp"; + +void throw_java_io_exception(JNIEnv* env, const std::string& message) { + jclass exception_class = env->FindClass("java/io/IOException"); + env->ThrowNew(exception_class, message.c_str()); + env->DeleteLocalRef(exception_class); +} + +jobjectArray get_paimon_spill_directories(JNIEnv* env, jclass, jlong spill_session_handle) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr) { + throw_java_io_exception(env, "Paimon external spill session is null"); + return nullptr; + } + + std::vector paths; + Status st = spill_session->get_paths(&paths); + if (!st.ok()) { + throw_java_io_exception(env, st.to_string()); + return nullptr; + } + jclass string_class = env->FindClass("java/lang/String"); + if (string_class == nullptr) { + return nullptr; + } + jobjectArray result = + env->NewObjectArray(static_cast(paths.size()), string_class, nullptr); + env->DeleteLocalRef(string_class); + if (result == nullptr) { + return nullptr; + } + for (jsize i = 0; i < static_cast(paths.size()); ++i) { + jstring path = env->NewStringUTF(paths[i].c_str()); + if (path == nullptr) { + return nullptr; + } + env->SetObjectArrayElement(result, i, path); + env->DeleteLocalRef(path); + if (env->ExceptionCheck()) { + return nullptr; + } + } + return result; +} + +void reserve_paimon_spill(JNIEnv* env, jclass, jlong spill_session_handle, jstring path, + jlong bytes) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr || path == nullptr) { + throw_java_io_exception(env, "Paimon external spill session or path is null"); + return; + } + const char* path_chars = env->GetStringUTFChars(path, nullptr); + if (path_chars == nullptr) { + return; + } + std::string native_path(path_chars); + env->ReleaseStringUTFChars(path, path_chars); + Status st = spill_session->reserve(native_path, bytes); + if (!st.ok()) { + throw_java_io_exception(env, st.to_string()); + } +} + +void update_paimon_spill_accounting(JNIEnv* env, jclass, jlong spill_session_handle, jstring path, + jlong current_bytes_delta, jlong write_bytes, + jlong read_bytes) { + auto* spill_session = reinterpret_cast(spill_session_handle); + if (spill_session == nullptr || path == nullptr) { + return; + } + const char* path_chars = env->GetStringUTFChars(path, nullptr); + if (path_chars == nullptr) { + return; + } + std::string native_path(path_chars); + env->ReleaseStringUTFChars(path, path_chars); + spill_session->update_accounting(native_path, current_bytes_delta, write_bytes, read_bytes); +} + +Status register_paimon_spill_natives(JNIEnv* env, jclass writer_class) { + static char get_spill_directories_name[] = "getPaimonSpillDirectories"; + static char get_spill_directories_signature[] = "(J)[Ljava/lang/String;"; + static char reserve_spill_name[] = "reservePaimonSpill"; + static char reserve_spill_signature[] = "(JLjava/lang/String;J)V"; + static char update_spill_name[] = "updatePaimonSpillAccounting"; + static char update_spill_signature[] = "(JLjava/lang/String;JJJ)V"; + static ::JNINativeMethod methods[] = { + {get_spill_directories_name, get_spill_directories_signature, + reinterpret_cast(&get_paimon_spill_directories)}, + {reserve_spill_name, reserve_spill_signature, + reinterpret_cast(&reserve_paimon_spill)}, + {update_spill_name, update_spill_signature, + reinterpret_cast(&update_paimon_spill_accounting)}, + }; + if (env->RegisterNatives(writer_class, methods, + static_cast(sizeof(methods) / sizeof(methods[0]))) != JNI_OK) { + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, true, "JNI exception registering Paimon spill native methods: ")); + return Status::JniError("Failed to register Paimon spill native methods"); + } + return Status::OK(); +} + +std::atomic& paimon_jni_close_failed() { + static std::atomic failed {false}; + return failed; +} + +struct RetainedPaimonResources { + std::unique_ptr memory_manager; + std::unique_ptr spill_session; +}; + +std::mutex& retained_resources_mutex() { + static auto* mutex = new std::mutex(); + return *mutex; +} + +std::vector& retained_resources() { + static auto* resources = new std::vector(); + return *resources; +} + +void retain_resources_after_failed_close(std::unique_ptr memory_manager, + std::unique_ptr spill_session) { + // An unconfirmed Java close means a background Paimon task may still reference this manager's + // native pages or spill callbacks. Quarantine both resources and stop admitting new writers so + // repeated failures cannot accumulate process-lifetime resources without a bound. + paimon_jni_close_failed().store(true, std::memory_order_release); + if (memory_manager == nullptr && spill_session == nullptr) { + return; + } + std::lock_guard lock(retained_resources_mutex()); + retained_resources().emplace_back(RetainedPaimonResources { + .memory_manager = std::move(memory_manager), + .spill_session = std::move(spill_session), + }); +} + +} // namespace + +// ──────────────────────────────────────────────────────────── +// JNI helpers β€” class loading +// ──────────────────────────────────────────────────────────── + +static constexpr const char* PAIMON_JNI_WRITER_CLASS = "org/apache/doris/paimon/PaimonJniWriter"; +const char* const PAIMON_JNI_WRITER_OPEN_SIGNATURE = + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZZLjava/lang/" + "String;JJJ)V"; + +PaimonJniWriterOpenMode PaimonJniWriterOpenMode::from_write_mode( + TPaimonWriteMode::type write_mode) { + return {static_cast(write_mode == TPaimonWriteMode::OVERWRITE), + static_cast(write_mode == TPaimonWriteMode::CHANGELOG)}; +} + +JniPaimonWriteBackend::JniPaimonWriteBackend() = default; + +JniPaimonWriteBackend::~JniPaimonWriteBackend() { + Status st = close(); + if (!st.ok()) { + LOG(WARNING) << "Failed to close Paimon JNI backend during destruction: " << st.to_string(); + } +} + +Status JniPaimonWriteBackend::close() { + if (_jni_writer_obj == nullptr && _jni_writer_cls == nullptr) { + _memory_manager.reset(); + _arrow_schema.reset(); + _spill_session.reset(); + _opened = false; + return Status::OK(); + } + + JNIEnv* env = nullptr; + Status env_status = Jni::Env::Get(&env); + if (!env_status.ok()) { + bool java_users_may_exist = _jni_writer_obj != nullptr; + // JNI global references cannot be released without an environment. + // Deliberately abandon the handles so the Java writer remains alive. + _jni_writer_obj = nullptr; + _jni_writer_cls = nullptr; + if (java_users_may_exist) { + retain_resources_after_failed_close(std::move(_memory_manager), + std::move(_spill_session)); + } else { + _memory_manager.reset(); + _spill_session.reset(); + } + _arrow_schema.reset(); + _opened = false; + return env_status; + } + + Status close_status = Status::OK(); + if (_jni_writer_obj != nullptr) { + _refresh_memory_profile(); + if (_close_id == nullptr) { + close_status = Status::InternalError("PaimonJniWriter.close method is unavailable"); + } else { + env->CallVoidMethod(_jni_writer_obj, _close_id); + close_status = _check_jni_exception(env, "close PaimonJniWriter"); + } + env->DeleteGlobalRef(_jni_writer_obj); + _jni_writer_obj = nullptr; + } + if (_jni_writer_cls != nullptr) { + env->DeleteGlobalRef(_jni_writer_cls); + _jni_writer_cls = nullptr; + } + + if (close_status.ok()) { + _memory_manager.reset(); + _spill_session.reset(); + } else { + if (_memory_manager != nullptr) { + LOG(WARNING) + << "Retaining Paimon JNI native memory after an unconfirmed Java close: limit=" + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) << ", peak=" + << PrettyPrinter::print_bytes(_memory_manager->native_peak_allocated_bytes()); + } + // Paimon may still have asynchronous tasks using Doris-backed pages or spill callbacks. + // Retain ownership until process exit and fence subsequent writer admission. + retain_resources_after_failed_close(std::move(_memory_manager), std::move(_spill_session)); + } + _arrow_schema.reset(); + _opened = false; + return close_status; +} + +Status JniPaimonWriteBackend::_check_jni_exception(JNIEnv* env, const std::string& method_name) { + if (env->ExceptionCheck()) { + Status st = + Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in " + method_name + ": "); + LOG(WARNING) << st.to_string(); + return st; + } + return Status::OK(); +} + +static Status _get_paimon_arrow_schema(JNIEnv* env, jobject writer, jmethodID get_schema_id, + std::shared_ptr* schema) { + auto schema_bytes = static_cast(env->CallObjectMethod(writer, get_schema_id)); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception in PaimonJniWriter.getArrowSchema: ")); + if (schema_bytes == nullptr) { + return Status::InternalError("PaimonJniWriter.getArrowSchema returned null"); + } + + const jsize size = env->GetArrayLength(schema_bytes); + if (size <= 0) { + env->DeleteLocalRef(schema_bytes); + return Status::InternalError("PaimonJniWriter.getArrowSchema returned empty data"); + } + std::string serialized_schema(static_cast(size), '\0'); + env->GetByteArrayRegion(schema_bytes, 0, size, + reinterpret_cast(serialized_schema.data())); + env->DeleteLocalRef(schema_bytes); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception while reading Paimon Arrow schema: ")); + + auto input = std::make_shared( + arrow::Buffer::FromString(std::move(serialized_schema))); + auto reader_result = arrow::ipc::RecordBatchStreamReader::Open(input); + if (!reader_result.ok()) { + return Status::InternalError("Failed to deserialize Paimon Arrow schema: {}", + reader_result.status().ToString()); + } + *schema = reader_result.ValueOrDie()->schema(); + return Status::OK(); +} +Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) { + if (paimon_jni_close_failed().load(std::memory_order_acquire)) { + return Status::InternalError( + "Paimon JNI writes are disabled on this BE because a previous Java writer close " + "could not be confirmed; restart the BE to reclaim retained native memory safely"); + } + _arrow_schema.reset(); + DORIS_CHECK(sink.__isset.column_names); + DORIS_CHECK(sink.__isset.write_mode); + DORIS_CHECK(sink.__isset.serialized_table); + DORIS_CHECK(!sink.serialized_table.empty()); + DORIS_CHECK(sink.__isset.transaction_id); + DORIS_CHECK(sink.transaction_id > 0); + DORIS_CHECK(sink.__isset.commit_user); + DORIS_CHECK(!sink.commit_user.empty()); + DORIS_CHECK(profile != nullptr); + + RETURN_IF_ERROR(PaimonJniMemoryManager::create(state, &_memory_manager)); + RuntimeProfile* jni_profile = profile->create_child("JniPaimonWriteBackend", true, true); + _native_page_memory_limit = ADD_COUNTER(jni_profile, "NativePageMemoryLimit", TUnit::BYTES); + _native_page_memory_peak = ADD_COUNTER(jni_profile, "NativePageMemoryPeak", TUnit::BYTES); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + if (env->PushLocalFrame(32) != JNI_OK) { + Status st = _check_jni_exception(env, "create PaimonJniWriter open local reference frame"); + return st.ok() ? Status::InternalError("Failed to create JNI local reference frame") : st; + } + Defer pop_local_frame([&]() { env->PopLocalFrame(nullptr); }); + + // Step 1: Load PaimonJniWriter class through ScannerLoader (Paimon jars are + // not on the default application classpath, so FindClass won't work). + Jni::LocalObject local_writer_class; + RETURN_IF_ERROR( + Jni::Util::get_jni_scanner_class(env, PAIMON_JNI_WRITER_CLASS, &local_writer_class)); + auto writer_class = static_cast(local_writer_class.get()); + _jni_writer_cls = static_cast(env->NewGlobalRef(writer_class)); + RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter class reference")); + if (_jni_writer_cls == nullptr) { + return Status::JniError("Failed to create global PaimonJniWriter class reference"); + } + RETURN_IF_ERROR(PaimonJniMemoryManager::register_natives(env, _jni_writer_cls)); + RETURN_IF_ERROR(register_paimon_spill_natives(env, _jni_writer_cls)); + + // Step 2: Cache JNI method IDs for write, prepareCommit, abort, close. + jmethodID open_id = env->GetMethodID(_jni_writer_cls, "open", PAIMON_JNI_WRITER_OPEN_SIGNATURE); + jmethodID get_arrow_schema_id = env->GetMethodID(_jni_writer_cls, "getArrowSchema", "()[B"); + _write_id = env->GetMethodID(_jni_writer_cls, "writeArrow", "(JJ)V"); + _prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B"); + _abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V"); + _close_id = env->GetMethodID(_jni_writer_cls, "close", "()V"); + RETURN_IF_ERROR(_check_jni_exception(env, "resolve PaimonJniWriter methods")); + + // Step 3: Create the Java PaimonJniWriter instance. + jmethodID ctor_id = env->GetMethodID(_jni_writer_cls, "", "()V"); + jobject local_obj = env->NewObject(_jni_writer_cls, ctor_id); + RETURN_IF_ERROR(_check_jni_exception(env, "create PaimonJniWriter")); + _jni_writer_obj = env->NewGlobalRef(local_obj); + RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter object reference")); + if (_jni_writer_obj == nullptr) { + return Status::JniError("Failed to create global PaimonJniWriter object reference"); + } + + // Step 4: Create a lazy query-scoped spill session. Java requests its path only when Paimon + // first uses the IOManager, so a memory-only writer does not depend on spill storage. + auto* spill_file_manager = state->exec_env()->spill_file_mgr(); + if (spill_file_manager != nullptr) { + auto spill_relative_path = + fmt::format("{}-{}", PAIMON_JNI_WRITER_IO_TMP_DIR, spill_file_manager->next_id()); + RETURN_IF_ERROR(spill_file_manager->create_external_spill_session( + spill_relative_path, state->get_query_ctx(), &_spill_session)); + } + + // Step 5: Build Java arguments and call PaimonJniWriter.open(). + const std::map empty_config; + jstring j_serialized_table = env->NewStringUTF(sink.serialized_table.c_str()); + Jni::LocalObject j_hadoop_config; + RETURN_IF_ERROR(Jni::Util::convert_to_java_map( + env, sink.__isset.hadoop_config ? sink.hadoop_config : empty_config, &j_hadoop_config)); + jstring j_commit_user = env->NewStringUTF(sink.commit_user.c_str()); + jstring j_time_zone = env->NewStringUTF(state->timezone().c_str()); + + jclass string_cls = env->FindClass("java/lang/String"); + jobjectArray j_cols = + env->NewObjectArray(static_cast(sink.column_names.size()), string_cls, nullptr); + for (size_t i = 0; i < sink.column_names.size(); ++i) { + jstring column_name = env->NewStringUTF(sink.column_names[i].c_str()); + env->SetObjectArrayElement(j_cols, static_cast(i), column_name); + env->DeleteLocalRef(column_name); + } + RETURN_IF_ERROR(_check_jni_exception(env, "build PaimonJniWriter open arguments")); + + PaimonJniWriterOpenMode open_mode = PaimonJniWriterOpenMode::from_write_mode(sink.write_mode); + env->CallVoidMethod( + _jni_writer_obj, open_id, j_serialized_table, j_hadoop_config.get(), j_cols, + static_cast(sink.transaction_id), j_commit_user, open_mode.overwrite, + open_mode.changelog, j_time_zone, static_cast(_memory_manager->memory_limit()), + reinterpret_cast(_memory_manager.get()), + _spill_session == nullptr ? 0 : reinterpret_cast(_spill_session.get())); + Status st = _check_jni_exception(env, "open PaimonJniWriter"); + + if (st.ok()) { + st = _get_paimon_arrow_schema(env, _jni_writer_obj, get_arrow_schema_id, &_arrow_schema); + } + if (st.ok()) { + _opened = true; + _refresh_memory_profile(); + LOG(INFO) << "Paimon JNI writer memory limit: " + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) + << ", sink_pipeline_task_count=" << std::max(1, state->task_num()); + } + return st; +} + +// Writer creation stays non-const because the backend interface also supports future stateful FFI +// implementations. +Status JniPaimonWriteBackend::create_writer( // NOLINT(readability-make-member-function-const) + std::unique_ptr* writer) { + DORIS_CHECK(_opened); + DORIS_CHECK(_arrow_schema != nullptr); + *writer = std::make_unique(_jni_writer_obj, _write_id, _prepare_commit_id, + _abort_id, _arrow_schema); + return Status::OK(); +} + +JniPaimonWriter::JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, + jmethodID prepare_commit_id, jmethodID abort_id, + std::shared_ptr arrow_schema) + : _jni_writer_obj(jni_writer_obj), + _write_id(write_id), + _prepare_commit_id(prepare_commit_id), + _abort_id(abort_id), + _arrow_schema(std::move(arrow_schema)) {} + +Status JniPaimonWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + if (_arrow_schema == nullptr || _arrow_schema->num_fields() != block.columns()) { + return Status::InvalidArgument( + "Paimon Arrow schema column count does not match Doris Block: schema={}, block={}", + _arrow_schema == nullptr ? 0 : _arrow_schema->num_fields(), block.columns()); + } + + // The schema comes from the pinned Paimon table, so timestamp timezone, nested nullability and + // Variant layout are fixed before the first write. Arrow builders remain on the Doris side and + // are charged to the current query's MemTracker through ArrowMemoryPool. + std::shared_ptr record_batch; + RETURN_IF_ERROR(convert_to_arrow_batch(block, _arrow_schema, &_arrow_pool, &record_batch, + state->timezone_obj())); + + ArrowArray c_array {}; + ArrowSchema c_schema {}; + auto arrow_status = arrow::ExportRecordBatch(*record_batch, &c_array, &c_schema); + if (!arrow_status.ok()) { + return Status::InternalError("Failed to export Paimon Arrow RecordBatch: {}", + arrow_status.ToString()); + } + // Java consumes both C Data release callbacks on a successful import. On every exit, release + // whichever struct still retains its callback; this covers partial imports and JNI failures + // without double release. + Defer release_c_data {[&] { + if (c_array.release != nullptr) { + c_array.release(&c_array); + } + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + }}; + // writeArrow is synchronous and this operator runs on the blocking scheduler. The exported + // RecordBatch therefore stays alive until Paimon has consumed all rows; Java never owns an IPC + // copy, and any synchronous SDK flush or memory wait occupies only a blocking-scheduler worker. + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + env->CallVoidMethod(_jni_writer_obj, _write_id, reinterpret_cast(&c_array), + reinterpret_cast(&c_schema)); + return Jni::Env::GetJniExceptionMsg(env, false, + "JNI exception in JniPaimonWriter::writeArrow: "); +} + +Status JniPaimonWriter::prepare_commit(std::vector& messages) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + + // Call PaimonJniWriter.prepareCommit() which returns byte[][] β€” + // each element is a DPCM-framed serialized CommitMessage chunk produced + // by PaimonCommitCodec.encode(). + jobject j_payloads_obj = env->CallObjectMethod(_jni_writer_obj, _prepare_commit_id); + Status st = Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in prepareCommit: "); + if (!st.ok()) { + return st; + } + + if (j_payloads_obj == nullptr) { + return Status::InternalError("PaimonJniWriter.prepareCommit returned null"); + } + + // Unpack the byte[][] into TPaimonCommitMessage structs for FE transport. + auto* j_payloads = static_cast(j_payloads_obj); + jsize num_payloads = env->GetArrayLength(j_payloads); + + for (jsize i = 0; i < num_payloads; ++i) { + auto j_bytes = static_cast(env->GetObjectArrayElement(j_payloads, i)); + if (j_bytes == nullptr) { + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned a null payload"); + } + jsize len = env->GetArrayLength(j_bytes); + if (len == 0) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned an empty payload"); + } + TPaimonCommitMessage msg; + msg.payload.resize(static_cast(len)); + env->GetByteArrayRegion(j_bytes, 0, len, reinterpret_cast(msg.payload.data())); + Status copy_status = Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception while reading Paimon commit payload: "); + if (!copy_status.ok()) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + return copy_status; + } + msg.__isset.payload = true; + messages.emplace_back(std::move(msg)); + env->DeleteLocalRef(j_bytes); + } + env->DeleteLocalRef(j_payloads); + return Status::OK(); +} + +Status JniPaimonWriter::abort() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + env->CallVoidMethod(_jni_writer_obj, _abort_id); + return Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in abort: "); +} + +void JniPaimonWriteBackend::_refresh_memory_profile() { + if (_memory_manager == nullptr) { + return; + } + COUNTER_SET(_native_page_memory_limit, _memory_manager->memory_limit()); + COUNTER_SET(_native_page_memory_peak, _memory_manager->native_peak_allocated_bytes()); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h new file mode 100644 index 00000000000000..d5a5de3b9ea1bb --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "format/parquet/arrow_memory_pool.h" +#include "runtime/runtime_profile.h" + +namespace arrow { +class Schema; +} + +namespace doris { + +class ExternalSpillSession; +class RuntimeState; + +extern const char* const PAIMON_JNI_WRITER_OPEN_SIGNATURE; + +struct PaimonJniWriterOpenMode { + jboolean overwrite; + jboolean changelog; + + static PaimonJniWriterOpenMode from_write_mode(TPaimonWriteMode::type write_mode); +}; + +/// JNI backend that owns the Java PaimonJniWriter object and its JNI method +/// handles. Creates lightweight JniPaimonWriter adapters that share this +/// backend's JVM connection. +/// +/// Each JniPaimonWriteBackend corresponds to one Java PaimonJniWriter +/// instance; the JniPaimonWriter adapters are thin wrappers that delegate +/// write/prepare_commit/abort calls through the cached JNI method IDs. JNI-only +/// memory ownership and Profile counters stay here and are not part of the +/// common backend contract. +class JniPaimonWriteBackend final : public IPaimonWriteBackend { +public: + JniPaimonWriteBackend(); + ~JniPaimonWriteBackend() override; + + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::JNI; } + +private: + Status _check_jni_exception(JNIEnv* env, const std::string& method_name); + void _refresh_memory_profile(); + + // JNI global references β€” live for the duration of this backend. + jclass _jni_writer_cls = nullptr; + jobject _jni_writer_obj = nullptr; + + // Cached JNI method IDs for the PaimonJniWriter Java methods. + jmethodID _write_id = nullptr; + jmethodID _prepare_commit_id = nullptr; + jmethodID _abort_id = nullptr; + jmethodID _close_id = nullptr; + + std::unique_ptr _memory_manager; + std::shared_ptr _arrow_schema; + std::unique_ptr _spill_session; + RuntimeProfile::Counter* _native_page_memory_limit = nullptr; + RuntimeProfile::Counter* _native_page_memory_peak = nullptr; + bool _opened = false; +}; + +/// Lightweight C++ adapter that delegates to the shared JNI backend. +/// +/// Owns the Arrow memory pool used for Block β†’ Arrow RecordBatch conversion. +/// Each JniPaimonWriter is created by JniPaimonWriteBackend::create_writer() +/// and shares the backend's JNI method IDs and Java writer object reference. +class JniPaimonWriter final : public IPaimonWriter { +public: + JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, jmethodID prepare_commit_id, + jmethodID abort_id, std::shared_ptr arrow_schema); + + Status write(RuntimeState* state, Block& block) override; + Status prepare_commit(std::vector& messages) override; + Status abort() override; + +private: + // Shared JNI state (owned by JniPaimonWriteBackend, not this adapter). + jobject _jni_writer_obj; + jmethodID _write_id; + jmethodID _prepare_commit_id; + jmethodID _abort_id; + + // Arrow resources owned by this writer adapter. + ArrowMemoryPool<> _arrow_pool; + std::shared_ptr _arrow_schema; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp new file mode 100644 index 00000000000000..1ea94294033c97 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" + +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/allocator.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" + +namespace doris { + +class PaimonJniMemoryManager::Impl { +public: + Impl(std::shared_ptr resource_context, int64_t memory_limit) + : _resource_context(std::move(resource_context)), _memory_limit(memory_limit) { + DORIS_CHECK(_resource_context != nullptr); + DORIS_CHECK(_memory_limit > 0); + } + + ~Impl() { + // Java may retain direct buffers until its writer is closed. Release + // every outstanding page here as the final native ownership boundary. + try { + release_all_pages(); + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: " << e.what(); + } catch (...) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: unknown exception"; + } + } + + jobject allocate_page(JNIEnv* env, jint bytes) { + if (bytes <= 0) { + throw Exception(Status::InvalidArgument( + "Paimon JNI memory page size must be positive, actual={}", bytes)); + } + + // Reserve the writer-local budget before entering the allocator. This + // prevents concurrent JNI callbacks from transiently allocating past + // the configured cap and only discovering it after query accounting + // or the system allocator has already rejected the request. + { + std::lock_guard lock(_mutex); + if (bytes > _memory_limit - _native_allocated_bytes - _native_reserved_bytes) { + throw Exception(Status::Error( + "Paimon JNI write buffer exceeded its {} native memory limit", + PrettyPrinter::print_bytes(_memory_limit))); + } + _native_reserved_bytes += bytes; + } + bool reservation_committed = false; + Defer rollback_reservation {[&]() { + if (!reservation_committed) { + std::lock_guard lock(_mutex); + _native_reserved_bytes -= bytes; + } + }}; + + // Allocate and account while attached to the query's resource + // context. The callback can run on a JVM-created thread, so merely + // relying on the calling BE thread's context would bypass query + // memory accounting. + void* address = with_resource_context([&]() { + enable_thread_catch_bad_alloc++; + Defer restore_bad_alloc_catch {[&]() { enable_thread_catch_bad_alloc--; }}; + void* allocated = _allocator.alloc(static_cast(bytes)); + try { + std::lock_guard lock(_mutex); + _allocations.emplace_back(allocated, static_cast(bytes)); + _native_reserved_bytes -= bytes; + _native_allocated_bytes += bytes; + _native_peak_allocated_bytes = + std::max(_native_peak_allocated_bytes, _native_allocated_bytes); + reservation_committed = true; + } catch (...) { + _allocator.free(allocated, static_cast(bytes)); + throw; + } + return allocated; + }); + + // NewDirectByteBuffer does not copy memory; Paimon will read/write the + // page directly. If JNI rejects the address, undo the native + // allocation and its accounting entry before returning. + jobject buffer = env->NewDirectByteBuffer(address, bytes); + if (buffer == nullptr || env->ExceptionCheck()) { + remove_and_free_page(address, static_cast(bytes)); + return nullptr; + } + return buffer; + } + + int64_t memory_limit() const { return _memory_limit; } + + int64_t native_peak_allocated_bytes() const { + std::lock_guard lock(_mutex); + return _native_peak_allocated_bytes; + } + +private: + template + auto with_resource_context(Function&& function) + -> decltype(std::forward(function)()) { + // JNI normally re-enters on the attached blocking pipeline thread. Attach + // Java-created threads explicitly too, so every allocation/free is + // charged to the query rather than to an unrelated thread context. + if (!pthread_context_ptr_init && bthread_self() == 0) { + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + if (thread_context()->is_attach_task()) { + SCOPED_SWITCH_RESOURCE_CONTEXT(_resource_context); + return std::forward(function)(); + } + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + + void release_all_pages() { + // Detach ownership from the bookkeeping vector under the lock, then + // free outside the lock. Allocator/free may invoke code that takes + // unrelated locks and must not block page accounting readers. + std::vector> allocations; + { + std::lock_guard lock(_mutex); + allocations.swap(_allocations); + _native_allocated_bytes = 0; + } + if (allocations.empty()) { + return; + } + + with_resource_context([&]() { + for (const auto& [address, bytes] : allocations) { + _allocator.free(address, bytes); + } + }); + } + + void remove_and_free_page(void* address, size_t bytes) { + // Roll back a page whose Java direct-buffer wrapper could not be + // created. The address is removed under the same lock used by the + // normal accounting path, while the potentially expensive free is + // performed after releasing it. + { + std::lock_guard lock(_mutex); + auto it = std::find_if( + _allocations.begin(), _allocations.end(), + [&](const auto& allocation) { return allocation.first == address; }); + if (it != _allocations.end()) { + _allocations.erase(it); + _native_allocated_bytes -= bytes; + } + } + with_resource_context([&]() { _allocator.free(address, bytes); }); + } + + // Query resource context used for all native allocator operations. + std::shared_ptr _resource_context; + // Immutable per-writer cap, calculated by PaimonJniMemoryManager::create. + const int64_t _memory_limit; + // Doris allocator used instead of JVM/Arrow allocation so native pages are + // visible to Doris' memory accounting and allocator hooks. + Allocator _allocator; + // Protects the allocation list and both usage counters. JNI callbacks and + // Java close/finalizer paths may arrive concurrently. + mutable std::mutex _mutex; + // Every entry is (native address, size) and remains here until released. + std::vector> _allocations; + // Bytes reserved by callbacks which have passed the local limit check but + // have not yet completed their allocator call. + int64_t _native_reserved_bytes = 0; + // Committed and high-water native page usage, respectively. + int64_t _native_allocated_bytes = 0; + int64_t _native_peak_allocated_bytes = 0; +}; + +namespace { + +jobject allocate_paimon_memory_page(JNIEnv* env, jclass, jlong manager_handle, jint bytes) { + // This is called from PaimonJniWriter's Java memory pool. The handle is + // the native manager address passed when the writer is opened; ownership + // stays with the C++ writer/backend, so this callback must never delete it. + auto* manager = reinterpret_cast(manager_handle); + if (manager == nullptr) { + jclass exception_class = env->FindClass("java/lang/IllegalStateException"); + env->ThrowNew(exception_class, "Paimon JNI memory manager is null"); + env->DeleteLocalRef(exception_class); + return nullptr; + } + try { + return manager->allocate_page(env, bytes); + } catch (const std::exception& e) { + jclass exception_class = env->FindClass("java/lang/RuntimeException"); + // Avoid dynamic allocation while reporting a failed allocation. + char message[1024]; + std::snprintf(message, sizeof(message), "Paimon JNI native page allocation failed: %.900s", + e.what()); + env->ThrowNew(exception_class, message); + env->DeleteLocalRef(exception_class); + return nullptr; + } +} + +} // namespace + +PaimonJniMemoryManager::PaimonJniMemoryManager(std::unique_ptr impl) + : _impl(std::move(impl)) {} + +PaimonJniMemoryManager::~PaimonJniMemoryManager() = default; + +Status PaimonJniMemoryManager::create(RuntimeState* state, + std::unique_ptr* manager) { + DORIS_CHECK(state != nullptr); + DORIS_CHECK(manager != nullptr); + if (state->query_mem_tracker() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot size its write buffer without a query tracker"); + } + if (state->get_query_ctx() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot allocate native memory without QueryContext"); + } + + // Each task in this sink pipeline owns one Paimon writer. Use the task count produced by the + // BE pipeline builder rather than num_local_sink, which is an FE-provided field currently set + // only for OLAP sinks. This also reflects any local-exchange parallelism chosen by the BE. + const int64_t writer_count = std::max(1, state->task_num()); + const int64_t query_limit = state->query_mem_tracker()->limit(); + const int64_t query_share = query_limit > 0 ? query_limit / writer_count : query_limit; + // Paimon requests pages lazily, can flush/preempt owners inside its MemoryPoolFactory, and may + // retain allocated pages until writer close. Bound and account those actual page allocations. + // Arrow C Data keeps the batch body in Doris-owned buffers, so there is no separate Java Arrow + // body budget to subtract from this writer's Paimon page allowance. + const int64_t configured_memory_limit = config::paimon_jni_writer_memory_pool_limit_bytes; + const int64_t memory_limit = query_share > 0 ? std::min(query_share, configured_memory_limit) + : configured_memory_limit; + if (memory_limit <= 0) { + return Status::Error( + "Paimon JNI writer has insufficient memory budget: query_limit={}, " + "sink_pipeline_task_count={}, write_buffer_limit={}", + PrettyPrinter::print_bytes(query_limit), writer_count, + PrettyPrinter::print_bytes(memory_limit)); + } + + // ResourceContext is retained by Impl for the manager's whole lifetime so JNI callbacks stay + // associated with the query even if Paimon invokes one from a Java-created thread. + auto impl = std::make_unique(state->get_query_ctx()->resource_ctx(), memory_limit); + *manager = std::unique_ptr(new PaimonJniMemoryManager(std::move(impl))); + return Status::OK(); +} + +Status PaimonJniMemoryManager::register_natives(JNIEnv* env, jclass writer_class) { + // Keep the JNI surface minimal: Java asks native code only for a page; + // all ownership, limits, and cleanup stay in PaimonJniMemoryManager. + static char allocate_name[] = "allocatePaimonMemoryPage"; + static char allocate_signature[] = "(JI)Ljava/nio/ByteBuffer;"; + static ::JNINativeMethod methods[] = { + {allocate_name, allocate_signature, + reinterpret_cast(&allocate_paimon_memory_page)}, + }; + if (env->RegisterNatives(writer_class, methods, + static_cast(sizeof(methods) / sizeof(methods[0]))) != JNI_OK) { + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, true, "JNI exception registering Paimon memory native methods: ")); + return Status::JniError("Failed to register Paimon memory native methods"); + } + return Status::OK(); +} + +jobject PaimonJniMemoryManager::allocate_page(JNIEnv* env, jint bytes) { + return _impl->allocate_page(env, bytes); +} + +int64_t PaimonJniMemoryManager::memory_limit() const { + return _impl->memory_limit(); +} + +int64_t PaimonJniMemoryManager::native_peak_allocated_bytes() const { + return _impl->native_peak_allocated_bytes(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h new file mode 100644 index 00000000000000..037d0c1952ec43 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" + +namespace doris { + +class RuntimeState; + +/// Owns the Doris-side native memory used by one Java Paimon writer. +/// +/// Paimon's sort/merge buffers are Java objects, but their page storage is +/// requested through a JNI callback. This manager is the bridge for that +/// callback: it allocates each page with Doris' allocator, exposes the page as +/// a direct ByteBuffer, tracks it until the writer is closed, and releases all +/// pages in its destructor. The native writer/backend therefore keeps this +/// manager alive for at least as long as the Java writer can access its +/// callback handle. +/// +/// The limit is a per-writer budget derived from the query limit and the sink +/// pipeline's task count. The manager accounts only for pages allocated by +/// this callback; Java heap and other Paimon-managed memory remain under their +/// respective runtimes. +class PaimonJniMemoryManager { +public: + ~PaimonJniMemoryManager(); + + /// Construct a manager whose budget is sized from the query context. + /// + /// The query must provide both a memory tracker and QueryContext. The + /// latter supplies the ResourceContext used whenever allocation/freeing + /// crosses into a JNI-created thread. + static Status create(RuntimeState* state, std::unique_ptr* manager); + + /// Register the static JNI callback used by PaimonJniWriter. + static Status register_natives(JNIEnv* env, jclass writer_class); + + /// Allocate one native page and return it as a direct ByteBuffer. + /// + /// On failure this method leaves no accounting entry behind and reports the error through the + /// JNI environment. The returned buffer remains valid until the manager is destroyed (or + /// allocation of that page is rolled back because NewDirectByteBuffer failed). + jobject allocate_page(JNIEnv* env, jint bytes); + + /// Return the immutable per-writer native page budget in bytes. + int64_t memory_limit() const; + + /// Return the high-water mark of native pages allocated by this manager. + int64_t native_peak_allocated_bytes() const; + +private: + class Impl; + + explicit PaimonJniMemoryManager(std::unique_ptr impl); + + std::unique_ptr _impl; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp new file mode 100644 index 00000000000000..3e0dd58a06fdc9 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_table_writer.h" + +#include "common/check.h" +#include "common/logging.h" +#include "core/block/block.h" +#include "core/block/materialize_block.h" +#include "exprs/vexpr_context.h" +#include "runtime/runtime_state.h" + +namespace doris { + +PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs) + : _t_sink(std::move(t_sink)), _output_expr_ctxs(output_exprs) { + DCHECK(_t_sink.__isset.paimon_table_sink); +} + +Status PaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + + // Register profile counters + _written_rows_counter = ADD_COUNTER(profile, "WrittenRows", TUnit::UNIT); + _written_bytes_counter = ADD_COUNTER(profile, "WrittenBytes", TUnit::BYTES); + _send_data_timer = ADD_TIMER(profile, "SendDataTime"); + _project_timer = ADD_CHILD_TIMER(profile, "ProjectTime", "SendDataTime"); + _file_store_write_timer = ADD_CHILD_TIMER(profile, "FileStoreWriteTime", "SendDataTime"); + _open_timer = ADD_TIMER(profile, "OpenTime"); + _close_timer = ADD_TIMER(profile, "CloseTime"); + _prepare_commit_timer = ADD_TIMER(profile, "PrepareCommitTime"); + _commit_payload_count = ADD_COUNTER(profile, "CommitPayloadCount", TUnit::UNIT); + _commit_payload_bytes_counter = ADD_COUNTER(profile, "CommitPayloadBytes", TUnit::BYTES); + + SCOPED_TIMER(_open_timer); + + // Step 1: Create the backend (JNI or FFI) based on the sink configuration. + RETURN_IF_ERROR(PaimonWriteBackendFactory::create(_t_sink.paimon_table_sink, &_backend)); + DCHECK(_backend); + // Step 2: Open the backend β€” for JNI this loads the Java class and calls PaimonJniWriter.open(). + RETURN_IF_ERROR(_backend->open(_t_sink.paimon_table_sink, state, profile)); + // Step 3: Create a lightweight writer adapter that delegates to the opened backend. + RETURN_IF_ERROR(_backend->create_writer(&_writer)); + DCHECK(_writer); + + LOG(INFO) << "PaimonTableWriter opened: backend=" << static_cast(_backend->type()) + << ", writer_scope=local_state"; + return Status::OK(); +} + +Status PaimonTableWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + SCOPED_TIMER(_send_data_timer); + + // Step 1: Apply output expressions to produce the columns selected by FE. + Block output_block; + { + SCOPED_TIMER(_project_timer); + RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(_output_expr_ctxs, block, + &output_block)); + materialize_block_inplace(output_block); + } + + COUNTER_UPDATE(_written_rows_counter, block.rows()); + COUNTER_UPDATE(_written_bytes_counter, block.bytes()); + state->update_num_rows_load_total(block.rows()); + state->update_num_bytes_load_total(block.bytes()); + + // Step 2: Delegate to the backend writer (JNI or FFI). For the JNI path + // this converts Block β†’ Arrow RecordBatch β†’ Arrow C Data β†’ Java PaimonJniWriter. + DCHECK(_writer); + { + SCOPED_TIMER(_file_store_write_timer); + RETURN_IF_ERROR(_writer->write(state, output_block)); + } + _written_rows += block.rows(); + return Status::OK(); +} + +Status PaimonTableWriter::close(Status status) { + SCOPED_TIMER(_close_timer); + + // Prepare messages first, but do not publish them until the backend confirms + // that every SDK user has stopped and its native backing memory is safe to release. + std::vector messages; + if (status.ok()) { + DCHECK(_writer); + { + SCOPED_TIMER(_prepare_commit_timer); + Status prep_st = _writer->prepare_commit(messages); + if (!prep_st.ok()) { + status = prep_st; + } + } + } + + // If prepare_commit failed or the incoming status was already an error, + // abort the writer to clean up uncommitted data files. + if (!status.ok()) { + LOG(WARNING) << "Paimon writer closing with error: " << status.to_string(); + if (_writer) { + Status abort_st = _writer->abort(); + if (!abort_st.ok()) { + LOG(WARNING) << "Paimon writer abort failed: " << abort_st.to_string(); + } + } + } + + // The adapter only owns Arrow conversion resources. Release it before closing + // the backend, whose Java close is the authoritative SDK shutdown boundary. + _writer.reset(); + + if (_backend) { + Status close_st = _backend->close(); + if (!close_st.ok()) { + if (status.ok()) { + status = close_st; + } else { + LOG(WARNING) << "Paimon backend close also failed: " << close_st.to_string(); + } + } + } + + // Only a fully prepared and cleanly stopped writer may contribute payloads + // to the FE transaction. A Java close failure therefore aborts the Doris + // transaction instead of allowing it to commit potentially unsafe output. + if (status.ok()) { + COUNTER_UPDATE(_commit_payload_count, static_cast(messages.size())); + for (const auto& msg : messages) { + DORIS_CHECK(msg.__isset.payload); + COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast(msg.payload.size())); + } + if (!messages.empty()) { + _state->add_paimon_commit_messages(messages); + LOG(INFO) << "Paimon writer closed: " << messages.size() + << " commit messages, total rows=" << _written_rows; + } + } + + _backend.reset(); + return status; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h new file mode 100644 index 00000000000000..1e9f96c181d658 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "exprs/vexpr_fwd.h" +#include "runtime/runtime_profile.h" + +namespace doris { + +class RuntimeState; + +/// Each PaimonTableSinkLocalState owns one PaimonTableWriter, which in turn +/// owns one IPaimonWriteBackend and one IPaimonWriter. Pipeline parallelism +/// therefore determines the number of independent Paimon writer sessions; +/// each writer session delegates partition and bucket routing to the Paimon +/// SDK (Java via JNI, or Rust via FFI in the future). +/// +/// Doris does NOT compute partition values or bucket ids β€” it passes complete +/// Blocks through the selected backend (JNI/FFI) to the Paimon SDK, which +/// internally computes partition values, bucket ids, and routes rows to the +/// correct file writers. +/// +/// Architecture: +/// PaimonTableSinkOperatorX +/// β”‚ sink_impl() β†’ PaimonTableWriter::write() (synchronous, no routing) +/// β–Ό +/// PaimonTableWriter (one per LocalState / pipeline instance) +/// β”‚ owns IPaimonWriteBackend (JNI or FFI) +/// β”‚ └─ create_writer() β†’ IPaimonWriter +/// β”‚ write() +/// β”‚ β†’ JNI backend: Block β†’ Arrow C Data β†’ Java Paimon SDK +/// β”‚ β†’ FFI backend: Block β†’ Rust writer (future) +/// β”‚ β†’ selected SDK owns row normalization, routing, buffering, +/// β”‚ file writing, and compaction +/// β–Ό +/// close() β†’ prepareCommit() β†’ CommitMessage[] +/// +/// Commit flow (BE only prepares messages; FE is the commit coordinator): +/// close() β†’ writer->prepare_commit() +/// β†’ collect TPaimonCommitMessage[] (DPCM-framed serialized messages) +/// β†’ RuntimeState::add_paimon_commit_messages() +/// β†’ RPC to FE Coordinator β†’ PaimonTransaction +class PaimonTableWriter final { +public: + PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs); + + ~PaimonTableWriter() = default; + + Status open(RuntimeState* state, RuntimeProfile* profile); + + Status write(RuntimeState* state, Block& block); + + Status close(Status status); + +private: + TDataSink _t_sink; + const VExprContextSPtrs& _output_expr_ctxs; + RuntimeState* _state = nullptr; + int64_t _written_rows = 0; + + // Backend owns the JNI/FFI connection and creates the writer adapter. + // Both are scoped to this PaimonTableWriter (one per LocalState). + std::unique_ptr _backend; + std::unique_ptr _writer; + + // Profile counters + RuntimeProfile::Counter* _written_rows_counter = nullptr; + RuntimeProfile::Counter* _written_bytes_counter = nullptr; + RuntimeProfile::Counter* _send_data_timer = nullptr; + RuntimeProfile::Counter* _project_timer = nullptr; + RuntimeProfile::Counter* _file_store_write_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _prepare_commit_timer = nullptr; + RuntimeProfile::Counter* _commit_payload_count = nullptr; + RuntimeProfile::Counter* _commit_payload_bytes_counter = nullptr; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h new file mode 100644 index 00000000000000..44e2667c7c103a --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" + +namespace doris { + +class RuntimeState; +class RuntimeProfile; + +enum class PaimonBackendType { + JNI, // Java via JNI (PaimonJniWriter) + FFI, // Rust via FFI (placeholder, not yet implemented) +}; + +/// Writer contract implemented by one SDK writer adapter. Each +/// PaimonTableWriter owns one IPaimonWriter, which delegates to the +/// underlying Paimon SDK (Java JNI or Rust FFI). Partition and bucket +/// routing happens inside the selected SDK backend. +/// +/// Lifecycle: created by IPaimonWriteBackend::create_writer() after the +/// backend is opened; used for the duration of one pipeline instance. +class IPaimonWriter { +public: + virtual ~IPaimonWriter() = default; + + /// Write a projected Block to the Paimon SDK. + /// For the JNI path: Block β†’ Arrow RecordBatch β†’ Arrow C Data β†’ Java. + virtual Status write(RuntimeState* state, Block& block) = 0; + + /// Flush all buffered data, close files, and collect serialized commit + /// messages (DPCM-framed). Called once at EOS. + virtual Status prepare_commit(std::vector& messages) = 0; + + /// Discard written data files on error. Called when write or prepare_commit fails. + virtual Status abort() = 0; +}; + +/// Backend boundary for creating writers via JNI (Java) or FFI (Rust). +/// +/// The backend owns the connection/session to the external runtime: +/// - JNI: owns the JVM class reference, method IDs, and the Java writer object. +/// - FFI: (future) owns the Rust FFI handle. +/// +/// Each backend creates one or more IPaimonWriter adapters that share the +/// same underlying connection. Snapshot commit is deliberately excluded from +/// this boundary: BE only prepares commit messages (byte payloads), while FE +/// PaimonTransaction is the single commit coordinator. +class IPaimonWriteBackend { +public: + virtual ~IPaimonWriteBackend() = default; + + /// Initialize the backend connection. For JNI this loads the writer class, + /// creates the Java object, and calls PaimonJniWriter.open(). + virtual Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) = 0; + + /// Create a lightweight writer adapter that delegates to this backend. + virtual Status create_writer(std::unique_ptr* writer) = 0; + + /// Stop all SDK users and release backend resources. + /// + /// A successful return is the ownership boundary after which native memory + /// backing SDK buffers can be reclaimed safely. Callers must not publish + /// prepared commit messages until this succeeds. + virtual Status close() = 0; + + virtual PaimonBackendType type() const = 0; +}; + +/// Factory that selects and creates the appropriate write backend. +/// +/// Backend selection is based on TPaimonTableSink.backend_type: +/// - Default (unset or JNI): JniPaimonWriteBackend +/// - FFI: FfiPaimonWriteBackend (placeholder for future Rust writer) +class PaimonWriteBackendFactory { +public: + /// Create a backend instance based on the sink configuration. + static Status create(const TPaimonTableSink& sink, + std::unique_ptr* backend); + + /// Determine which backend type to use for the given sink. + static PaimonBackendType select_backend_type(const TPaimonTableSink& sink); +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp new file mode 100644 index 00000000000000..087228abbe5d2b --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +Status PaimonWriteBackendFactory::create(const TPaimonTableSink& sink, + std::unique_ptr* backend) { + switch (select_backend_type(sink)) { + case PaimonBackendType::JNI: + *backend = std::make_unique(); + return Status::OK(); + case PaimonBackendType::FFI: + *backend = std::make_unique(); + return Status::OK(); + } + return Status::InternalError("Unknown Paimon write backend"); +} + +PaimonBackendType PaimonWriteBackendFactory::select_backend_type(const TPaimonTableSink& sink) { + if (sink.__isset.backend_type && sink.backend_type == TPaimonWriteBackendType::FFI) { + return PaimonBackendType::FFI; + } + return PaimonBackendType::JNI; +} + +} // namespace doris diff --git a/be/src/exec/spill/spill_file_manager.cpp b/be/src/exec/spill/spill_file_manager.cpp index eb56fb14a132c1..f5629d3b501f42 100644 --- a/be/src/exec/spill/spill_file_manager.cpp +++ b/be/src/exec/spill/spill_file_manager.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -31,14 +32,105 @@ #include "exec/spill/spill_file.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "runtime/query_context.h" #include "storage/olap_define.h" #include "util/debug_points.h" #include "util/parse_util.h" #include "util/pretty_printer.h" #include "util/time.h" +#include "util/uid_util.h" namespace doris { +ExternalSpillSession::ExternalSpillSession(SpillFileManager* manager, QueryContext* query_context, + std::string relative_path) + : _manager(manager), + _query_context(query_context->weak_from_this()), + _resource_context(query_context->resource_ctx()), + _query_id(print_id(query_context->query_id())), + _relative_path(std::move(relative_path)) { + DCHECK(_manager != nullptr); + DCHECK(!_query_context.expired()); + DCHECK(_resource_context != nullptr); +} + +ExternalSpillSession::~ExternalSpillSession() { + _manager->_release_external_spill_session(this); +} + +Status ExternalSpillSession::get_paths(std::vector* paths) { + if (paths == nullptr) { + return Status::InvalidArgument("External spill paths output must not be null"); + } + std::lock_guard lock(_mutex); + if (_data_dir == nullptr) { + RETURN_IF_ERROR(_manager->_initialize_external_spill_session(this)); + } + *paths = {_path}; + return Status::OK(); +} + +bool ExternalSpillSession::_contains(const std::string& path) const { + return path == _path || + (path.size() > _path.size() && path.starts_with(_path) && path[_path.size()] == '/'); +} + +Status ExternalSpillSession::reserve(const std::string& path, int64_t bytes) { + if (bytes <= 0) { + return Status::InvalidArgument("External spill reservation must be positive: {}", bytes); + } + + std::lock_guard lock(_mutex); + if (_data_dir == nullptr || !_contains(path)) { + return Status::InvalidArgument("External spill path is not managed by Doris: {}", path); + } + if (bytes > std::numeric_limits::max() - _accounted_bytes) { + return Status::InvalidArgument("External spill reservation overflows: bytes={}", bytes); + } + if (_data_dir->reach_capacity_limit(bytes)) { + return Status::Error( + "External spill write exceeds the Doris spill storage limit: path={}, bytes={}", + path, bytes); + } + // Match SpillFileWriter: check capacity before the write, then account the accepted bytes. + _data_dir->update_spill_data_usage(bytes); + _accounted_bytes += bytes; + return Status::OK(); +} + +void ExternalSpillSession::update_accounting(const std::string& path, int64_t current_bytes_delta, + int64_t write_bytes, int64_t read_bytes) { + int64_t released_bytes = 0; + SpillDataDir* data_dir = nullptr; + { + std::lock_guard lock(_mutex); + if (_data_dir == nullptr || !_contains(path)) { + LOG(WARNING) << "Ignoring accounting for unmanaged external spill path: " << path; + return; + } + data_dir = _data_dir; + if (current_bytes_delta < 0) { + const int64_t requested_release = + current_bytes_delta == std::numeric_limits::min() + ? std::numeric_limits::max() + : -current_bytes_delta; + released_bytes = std::min(requested_release, _accounted_bytes); + _accounted_bytes -= released_bytes; + } + } + if (released_bytes > 0) { + data_dir->update_spill_data_usage(-released_bytes); + } + if (write_bytes > 0) { + _resource_context->io_context()->update_spill_write_bytes_to_local_storage(write_bytes); + _manager->update_spill_write_bytes(write_bytes); + } + if (read_bytes > 0) { + _resource_context->io_context()->update_spill_read_bytes_from_local_storage(read_bytes); + _manager->update_spill_read_bytes(read_bytes); + } +} + SpillFileManager::~SpillFileManager() { // QueryContext destruction can still queue failed deletions after stop(), for example while // VDataStreamMgr is being destroyed. Retry them once more before dropping the in-memory state. @@ -172,6 +264,76 @@ Status SpillFileManager::create_spill_file(const std::string& relative_path, return Status::OK(); } +Status SpillFileManager::create_external_spill_session( + const std::string& relative_path, QueryContext* query_context, + std::unique_ptr* spill_session) { + if (query_context == nullptr || spill_session == nullptr) { + return Status::InvalidArgument( + "External spill session requires QueryContext and output session"); + } + + spill_session->reset(new ExternalSpillSession(this, query_context, relative_path)); + return Status::OK(); +} + +Status SpillFileManager::_initialize_external_spill_session(ExternalSpillSession* spill_session) { + auto query_context = spill_session->_query_context.lock(); + if (query_context == nullptr) { + return Status::Cancelled("Query ended before the external spill session was initialized"); + } + auto* data_dir = _get_store_for_spill(); + if (data_dir == nullptr) { + return Status::Error( + "no available disk can be used for spill."); + } + + const auto query_dir = data_dir->get_spill_data_path(spill_session->_query_id); + { + // QueryContext teardown uses the regular pending-deletion path while this lease is live. + std::lock_guard lock(_pending_query_spill_directories_mutex); + ++_external_spill_directory_leases[query_dir]; + } + query_context->record_spill_data_dir(data_dir); + spill_session->_data_dir = data_dir; + spill_session->_path = query_dir + "/" + spill_session->_relative_path; + return Status::OK(); +} + +void SpillFileManager::_release_external_spill_session(ExternalSpillSession* spill_session) { + std::lock_guard session_lock(spill_session->_mutex); + if (spill_session->_data_dir == nullptr) { + return; + } + + if (spill_session->_accounted_bytes > 0) { + // Match SpillFile::gc(): QueryContext owns physical cleanup and its retry path, while the + // writer releases logical usage when its lifetime ends. + spill_session->_data_dir->update_spill_data_usage(-spill_session->_accounted_bytes); + spill_session->_accounted_bytes = 0; + } + + const auto query_dir = spill_session->_data_dir->get_spill_data_path(spill_session->_query_id); + std::lock_guard directory_lock(_pending_query_spill_directories_mutex); + auto it = _external_spill_directory_leases.find(query_dir); + DCHECK(it != _external_spill_directory_leases.end()); + if (it == _external_spill_directory_leases.end()) { + return; + } + DCHECK_GT(it->second, 0); + if (--it->second == 0) { + _external_spill_directory_leases.erase(it); + } +} + +SpillDataDir* SpillFileManager::_get_store_for_spill() { + auto data_dirs = _get_stores_for_spill(TStorageMedium::type::SSD); + if (data_dirs.empty()) { + data_dirs = _get_stores_for_spill(TStorageMedium::type::HDD); + } + // Select the first available data dir (sorted by usage ascending). + return data_dirs.empty() ? nullptr : data_dirs.front(); +} + void SpillFileManager::delete_spill_file(SpillFileSPtr spill_file) { if (!spill_file) { LOG(WARNING) << "[spill][delete] null spill_file"; @@ -196,6 +358,13 @@ void SpillFileManager::delete_query_spill_directory(const std::string& query_id, Status SpillFileManager::_try_delete_query_spill_directory( const PendingQuerySpillDirectory& pending_directory) { + { + std::lock_guard lock(_pending_query_spill_directories_mutex); + if (_external_spill_directory_leases.contains(pending_directory.query_dir)) { + return Status::InternalError("external spill directory is still in use: {}", + pending_directory.query_dir); + } + } DBUG_EXECUTE_IF("fault_inject::spill_file_manager::delete_query_spill_directory", { return Status::Error("injected query spill directory deletion failure"); }); diff --git a/be/src/exec/spill/spill_file_manager.h b/be/src/exec/spill/spill_file_manager.h index 1e3042d6ad3aaa..db6f2dced91435 100644 --- a/be/src/exec/spill/spill_file_manager.h +++ b/be/src/exec/spill/spill_file_manager.h @@ -40,6 +40,8 @@ class AtomicGauge; using UIntGauge = AtomicGauge; class MetricEntity; struct MetricPrototype; +class QueryContext; +class ResourceContext; class SpillFileManager; class SpillDataDir { @@ -112,6 +114,38 @@ class SpillDataDir { IntGauge* spill_disk_has_spill_data = nullptr; IntGauge* spill_disk_has_spill_gc_data = nullptr; }; + +// Adapts one external writer to the same root selection, capacity accounting and query cleanup +// used by Doris spill files. +class ExternalSpillSession { +public: + ~ExternalSpillSession(); + + Status get_paths(std::vector* paths); + + Status reserve(const std::string& path, int64_t bytes); + + void update_accounting(const std::string& path, int64_t current_bytes_delta, + int64_t write_bytes, int64_t read_bytes); + +private: + friend class SpillFileManager; + + ExternalSpillSession(SpillFileManager* manager, QueryContext* query_context, + std::string relative_path); + bool _contains(const std::string& path) const; + + SpillFileManager* _manager; + std::weak_ptr _query_context; + std::shared_ptr _resource_context; + std::string _query_id; + std::string _relative_path; + SpillDataDir* _data_dir = nullptr; + std::string _path; + int64_t _accounted_bytes = 0; + std::mutex _mutex; +}; + class SpillFileManager { public: ~SpillFileManager(); @@ -127,6 +161,12 @@ class SpillFileManager { // e.g. "query_id/sort-node_id-task_id-unique_id" Status create_spill_file(const std::string& relative_path, SpillFileSPtr& spill_file); + // Create a lazy managed session for an external spill implementation. A spill root is selected + // and registered only when the external implementation first requests its path. + Status create_external_spill_session(const std::string& relative_path, + QueryContext* query_context, + std::unique_ptr* spill_session); + /// Get a unique ID for constructing spill file paths. uint64_t next_id() { return id_++; } @@ -144,6 +184,8 @@ class SpillFileManager { void update_spill_read_bytes(int64_t bytes) { _spill_read_bytes_counter->increment(bytes); } private: + friend class ExternalSpillSession; + struct PendingQuerySpillDirectory { int failed_count {0}; std::string query_dir; @@ -154,15 +196,22 @@ class SpillFileManager { void _spill_gc_thread_callback(); Status _try_delete_query_spill_directory(const PendingQuerySpillDirectory& pending_directory); void _retry_pending_query_spill_directories(); + Status _initialize_external_spill_session(ExternalSpillSession* spill_session); + void _release_external_spill_session(ExternalSpillSession* spill_session); std::vector _get_stores_for_spill(TStorageMedium::type storage_medium); + SpillDataDir* _get_store_for_spill(); std::unordered_map> _spill_store_map; CountDownLatch _stop_background_threads_latch; std::shared_ptr _spill_gc_thread; + // Query cleanup uses the regular pending-deletion path. External leases only defer deletion + // while an SDK task can still access the same query directory; filesystem I/O never holds this + // mutex. std::mutex _pending_query_spill_directories_mutex; std::vector _pending_query_spill_directories; + std::unordered_map _external_spill_directory_leases; std::atomic_uint64_t id_ = 0; diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index bd9b849ba7a4de..1aca3b3a73399f 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -573,6 +573,17 @@ class RuntimeState { _mc_commit_datas.emplace_back(mc_commit_data); } + std::vector paimon_commit_messages() const { + std::lock_guard lock(_paimon_commit_messages_mutex); + return _paimon_commit_messages; + } + + void add_paimon_commit_messages(const std::vector& commit_messages) { + std::lock_guard lock(_paimon_commit_messages_mutex); + _paimon_commit_messages.insert(_paimon_commit_messages.end(), commit_messages.begin(), + commit_messages.end()); + } + // local runtime filter mgr, the runtime filter do not have remote target or // not need local merge should regist here. the instance exec finish, the local // runtime filter mgr can release the memory of local runtime filter @@ -1012,6 +1023,9 @@ class RuntimeState { mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; + mutable std::mutex _paimon_commit_messages_mutex; + std::vector _paimon_commit_messages; + std::vector> _op_id_to_local_state; std::unique_ptr _sink_local_state; diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index 0b54a8cb11dd6e..948436c607d31f 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -606,6 +606,9 @@ class Object { bool uninitialized() const { return _obj == nullptr; } + // Access the JNI handle without changing ownership. + jobject get() const { return _obj; } + void reset(JNIEnv* env) { if (_obj == nullptr) { return; diff --git a/be/test/exec/sink/paimon_jni_memory_manager_test.cpp b/be/test/exec/sink/paimon_jni_memory_manager_test.cpp new file mode 100644 index 00000000000000..4453b21f4b3ddc --- /dev/null +++ b/be/test/exec/sink/paimon_jni_memory_manager_test.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" + +#include "common/config.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/defer_op.h" + +namespace doris { + +TEST(PaimonJniMemoryManagerTest, DivideQueryBudgetBySinkPipelineTaskCount) { + constexpr int64_t QUERY_LIMIT = 256L * 1024 * 1024; + constexpr int64_t CONFIGURED_LIMIT = 512L * 1024 * 1024; + constexpr int SINK_PIPELINE_TASKS = 4; + + const int64_t old_configured_limit = config::paimon_jni_writer_memory_pool_limit_bytes; + Defer restore_config { + [&] { config::paimon_jni_writer_memory_pool_limit_bytes = old_configured_limit; }}; + config::paimon_jni_writer_memory_pool_limit_bytes = CONFIGURED_LIMIT; + + TUniqueId query_id; + query_id.hi = 1; + query_id.lo = 2; + TQueryOptions query_options; + query_options.__set_mem_limit(QUERY_LIMIT); + query_options.__set_query_type(TQueryType::SELECT); + TNetworkAddress fe_address; + fe_address.hostname = "127.0.0.1"; + fe_address.port = 9030; + auto query_ctx = + QueryContext::create(query_id, ExecEnv::GetInstance(), query_options, fe_address, true, + fe_address, QuerySource::INTERNAL_FRONTEND); + ASSERT_NE(query_ctx, nullptr); + + auto state = RuntimeState::create_unique(query_id, 0, query_options, query_ctx->query_globals, + ExecEnv::GetInstance(), query_ctx.get()); + state->set_task_num(SINK_PIPELINE_TASKS); + // Paimon must not depend on this FE-provided OLAP sink field. + state->set_num_local_sink(1); + + std::unique_ptr manager; + ASSERT_TRUE(PaimonJniMemoryManager::create(state.get(), &manager).ok()); + ASSERT_NE(manager, nullptr); + EXPECT_EQ(manager->memory_limit(), QUERY_LIMIT / SINK_PIPELINE_TASKS); +} + +} // namespace doris diff --git a/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp new file mode 100644 index 00000000000000..387e8932793338 --- /dev/null +++ b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +#include + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +namespace doris { + +TEST(PaimonWriteBackendFactoryTest, SelectBackendType) { + TPaimonTableSink sink; + EXPECT_EQ(PaimonBackendType::JNI, PaimonWriteBackendFactory::select_backend_type(sink)); + + sink.__set_backend_type(TPaimonWriteBackendType::FFI); + EXPECT_EQ(PaimonBackendType::FFI, PaimonWriteBackendFactory::select_backend_type(sink)); +} + +TEST(JniPaimonWriteBackendTest, OpenAbiAndWriteModes) { + EXPECT_STREQ( + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZZLjava/lang/" + "String;JJJ)V", + PAIMON_JNI_WRITER_OPEN_SIGNATURE); + + auto append = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::APPEND); + EXPECT_FALSE(append.overwrite); + EXPECT_FALSE(append.changelog); + + auto overwrite = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::OVERWRITE); + EXPECT_TRUE(overwrite.overwrite); + EXPECT_FALSE(overwrite.changelog); + + auto changelog = PaimonJniWriterOpenMode::from_write_mode(TPaimonWriteMode::CHANGELOG); + EXPECT_FALSE(changelog.overwrite); + EXPECT_TRUE(changelog.changelog); +} + +} // namespace doris diff --git a/be/test/vec/spill/spill_file_test.cpp b/be/test/vec/spill/spill_file_test.cpp index c4d4f140635f8d..1a39a576b238d3 100644 --- a/be/test/vec/spill/spill_file_test.cpp +++ b/be/test/vec/spill/spill_file_test.cpp @@ -92,7 +92,7 @@ class SpillFileTest : public testing::Test { auto st = io::global_local_filesystem()->create_directory(spill_data_dir->path(), false); ASSERT_TRUE(st.ok()) << "create directory failed: " << st.to_string(); auto second_spill_data_dir = std::make_unique( - _second_spill_dir, 1024L * 1024 * 128, TStorageMedium::HDD); + _second_spill_dir, 1024L * 1024 * 128, TStorageMedium::SSD); st = io::global_local_filesystem()->create_directory(second_spill_data_dir->path(), false); ASSERT_TRUE(st.ok()) << "create directory failed: " << st.to_string(); @@ -416,10 +416,15 @@ TEST_F(SpillFileTest, OpenCanRetryAfterFailure) { ASSERT_TRUE(st.ok()); } - const auto part_path = + const auto first_part_path = std::filesystem::path(_spill_dir) / "spill" / "test_query" / "open_retry" / "0"; - const auto backup_path = - std::filesystem::path(_spill_dir) / "spill" / "test_query" / "open_retry" / "0.bak"; + const auto second_part_path = + std::filesystem::path(_second_spill_dir) / "spill" / "test_query" / "open_retry" / "0"; + const auto part_path = + std::filesystem::exists(first_part_path) ? first_part_path : second_part_path; + ASSERT_TRUE(std::filesystem::exists(part_path)); + auto backup_path = part_path; + backup_path += ".bak"; std::filesystem::rename(part_path, backup_path); @@ -934,13 +939,17 @@ TEST_F(SpillFileTest, GCCleansUpFiles) { st = writer->close(); ASSERT_TRUE(st.ok()); - // Remember the spill directory path - spill_file_dir = _data_dir_ptr->get_spill_data_path() + "/test_query/gc_test"; - - // Verify directory exists + // Remember the selected spill directory path. bool exists = false; - st = io::global_local_filesystem()->exists(spill_file_dir, &exists); - ASSERT_TRUE(st.ok()); + for (auto* data_dir : {_data_dir_ptr, _second_data_dir_ptr}) { + auto candidate = data_dir->get_spill_data_path() + "/test_query/gc_test"; + st = io::global_local_filesystem()->exists(candidate, &exists); + ASSERT_TRUE(st.ok()); + if (exists) { + spill_file_dir = std::move(candidate); + break; + } + } ASSERT_TRUE(exists); // spill_file goes out of scope here, destructor calls gc() @@ -1383,10 +1392,17 @@ TEST_F(SpillFileTest, DeleteSpillFileThroughManagerSynchronously) { st = writer->close(); ASSERT_TRUE(st.ok()); - auto spill_file_dir = _data_dir_ptr->get_spill_data_path("test_query/mgr_delete"); + std::string spill_file_dir; bool exists = false; - st = io::global_local_filesystem()->exists(spill_file_dir, &exists); - ASSERT_TRUE(st.ok()); + for (auto* data_dir : {_data_dir_ptr, _second_data_dir_ptr}) { + auto candidate = data_dir->get_spill_data_path("test_query/mgr_delete"); + st = io::global_local_filesystem()->exists(candidate, &exists); + ASSERT_TRUE(st.ok()); + if (exists) { + spill_file_dir = std::move(candidate); + break; + } + } ASSERT_TRUE(exists); ExecEnv::GetInstance()->spill_file_mgr()->delete_spill_file(spill_file); @@ -1409,6 +1425,163 @@ TEST_F(SpillFileTest, ManagerNextId) { ASSERT_EQ(id3, id2 + 1); } +TEST_F(SpillFileTest, ManagerAllocatesExternalSpillSessionOnManagedRoot) { + TUniqueId query_id; + query_id.hi = 21; + query_id.lo = 22; + auto query_id_str = print_id(query_id); + auto query_ctx = MockQueryContext::create(query_id); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + + ASSERT_TRUE(st.ok()) << st.to_string(); + std::vector paths; + st = spill_session->get_paths(&paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(paths.size(), 1); + const std::string first_path = _data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"; + const std::string second_path = + _second_data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"; + ASSERT_TRUE(paths.front() == first_path || paths.front() == second_path); + bool exists = false; + for (const auto& path : paths) { + st = io::global_local_filesystem()->exists(path, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_FALSE(exists); + } + + const std::string& selected_path = paths.front(); + const std::string channel = selected_path + "/paimon-io-test/channel"; + ASSERT_TRUE(spill_session->reserve(channel, 1024).ok()); + auto* selected_data_dir = selected_path == first_path ? _data_dir_ptr : _second_data_dir_ptr; + auto* unselected_data_dir = + selected_data_dir == _data_dir_ptr ? _second_data_dir_ptr : _data_dir_ptr; + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 1024); + ASSERT_EQ(unselected_data_dir->get_spill_data_bytes(), 0); + spill_session->update_accounting(channel, -256, 0, 0); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 768); + _create_residual_file(channel); + + // Query teardown must not remove a directory while an asynchronous external writer can still + // use its native callback. The regular spill GC handles deferred cleanup after lease release. + query_ctx.reset(); + auto query_dir = selected_data_dir->get_spill_data_path(query_id_str); + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_TRUE(exists); + + spill_session.reset(); + // Match SpillFile::gc(): logical usage is released with the writer, while QueryContext owns + // physical deletion and retries. + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); + ASSERT_EQ(unselected_data_dir->get_spill_data_bytes(), 0); + + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_TRUE(exists); + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + st = io::global_local_filesystem()->exists(query_dir, &exists); + ASSERT_TRUE(st.ok()); + ASSERT_FALSE(exists); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); +} + +TEST_F(SpillFileTest, ExternalSpillSessionSkipsFullManagedRoot) { + TUniqueId query_id; + query_id.hi = 23; + query_id.lo = 24; + auto query_id_str = print_id(query_id); + auto query_ctx = MockQueryContext::create(query_id); + + const int64_t unavailable_bytes = _data_dir_ptr->get_spill_data_limit() + 1; + _data_dir_ptr->update_spill_data_usage(unavailable_bytes); + Defer release_full_root([&]() { _data_dir_ptr->update_spill_data_usage(-unavailable_bytes); }); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + ASSERT_TRUE(st.ok()) << st.to_string(); + + std::vector paths; + st = spill_session->get_paths(&paths); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(paths.size(), 1); + ASSERT_EQ(paths.front(), _second_data_dir_ptr->get_spill_data_path(query_id_str) + "/paimon"); +} + +TEST_F(SpillFileTest, ExternalSpillDirectoryCleanupRetriesAfterLeaseRelease) { + ExecEnv::GetInstance()->spill_file_mgr()->stop(); + TUniqueId query_id; + query_id.hi = 33; + query_id.lo = 34; + auto query_ctx = MockQueryContext::create(query_id); + + std::unique_ptr spill_session; + ASSERT_TRUE(ExecEnv::GetInstance() + ->spill_file_mgr() + ->create_external_spill_session("paimon", query_ctx.get(), &spill_session) + .ok()); + std::vector paths; + ASSERT_TRUE(spill_session->get_paths(&paths).ok()); + auto* selected_data_dir = + paths.front().starts_with(_data_dir_ptr->get_spill_data_path(print_id(query_id))) + ? _data_dir_ptr + : _second_data_dir_ptr; + ASSERT_TRUE(spill_session->reserve(paths.front() + "/paimon-io/channel", 1024).ok()); + _create_residual_file(paths.front() + "/paimon-io/channel"); + + const bool previous_enable_debug_points = config::enable_debug_points; + constexpr auto debug_point_name = + "fault_inject::spill_file_manager::delete_query_spill_directory"; + Defer restore_debug_point([&] { + DebugPoints::instance()->remove(debug_point_name); + config::enable_debug_points = previous_enable_debug_points; + }); + auto debug_point = std::make_shared(); + debug_point->execute_limit = 1; + config::enable_debug_points = true; + DebugPoints::instance()->add(debug_point_name, debug_point); + + query_ctx.reset(); + spill_session.reset(); + ASSERT_EQ(selected_data_dir->get_spill_data_bytes(), 0); + + const auto query_dir = selected_data_dir->get_spill_data_path(print_id(query_id)); + bool exists = false; + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_TRUE(exists); + + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_TRUE(exists); + ExecEnv::GetInstance()->spill_file_mgr()->gc(10000); + ASSERT_TRUE(io::global_local_filesystem()->exists(query_dir, &exists).ok()); + ASSERT_FALSE(exists); +} + +TEST_F(SpillFileTest, ExternalSpillSessionIsLazyWhenNoRootAvailable) { + TUniqueId query_id; + query_id.hi = 25; + query_id.lo = 26; + auto query_ctx = MockQueryContext::create(query_id); + + _data_dir_ptr->update_spill_data_usage(_data_dir_ptr->get_spill_data_limit()); + _second_data_dir_ptr->update_spill_data_usage(_second_data_dir_ptr->get_spill_data_limit()); + Defer release_full_roots([&]() { + _data_dir_ptr->update_spill_data_usage(-_data_dir_ptr->get_spill_data_limit()); + _second_data_dir_ptr->update_spill_data_usage( + -_second_data_dir_ptr->get_spill_data_limit()); + }); + + std::unique_ptr spill_session; + auto st = ExecEnv::GetInstance()->spill_file_mgr()->create_external_spill_session( + "paimon", query_ctx.get(), &spill_session); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_NE(spill_session, nullptr); +} + TEST_F(SpillFileTest, ManagerCreateMultipleFiles) { const int num_files = 5; std::vector files; @@ -1601,7 +1774,8 @@ TEST_F(SpillFileTest, DataDirCapacityTracking) { spill_file); ASSERT_TRUE(st.ok()); - auto initial_bytes = _data_dir_ptr->get_spill_data_bytes(); + auto initial_bytes = + _data_dir_ptr->get_spill_data_bytes() + _second_data_dir_ptr->get_spill_data_bytes(); SpillFileWriterSPtr writer; st = spill_file->create_writer(_runtime_state.get(), _profile.get(), writer); @@ -1617,7 +1791,8 @@ TEST_F(SpillFileTest, DataDirCapacityTracking) { st = writer->close(); ASSERT_TRUE(st.ok()); - auto after_write_bytes = _data_dir_ptr->get_spill_data_bytes(); + auto after_write_bytes = + _data_dir_ptr->get_spill_data_bytes() + _second_data_dir_ptr->get_spill_data_bytes(); ASSERT_GT(after_write_bytes, initial_bytes); } diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-scanner/pom.xml index fa7c27e4e98319..8bad697778abf4 100644 --- a/fe/be-java-extensions/paimon-scanner/pom.xml +++ b/fe/be-java-extensions/paimon-scanner/pom.xml @@ -61,6 +61,11 @@ under the License. paimon-format + + org.apache.paimon + paimon-arrow + +