From ed80f642ede4f140346165b3545a4aa48478ad20 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 26 Sep 2026 19:26:39 +0800 Subject: [PATCH] [core] Reject a merge fan-in below two up front A local-sort.max-num-file-handles of 1 has no validator; with at least two spilled runs the merge arithmetic overflowed (numMerges = ceil(n/0)) and the flush crashed with a confusing subList argument error, restart-looping the job on the same configuration. Reject a fan-in below 2 in the external merger constructor. Assisted-by: GLM-5.3 --- .../apache/paimon/sort/AbstractBinaryExternalMerger.java | 9 +++++++++ .../apache/paimon/sort/BinaryExternalSortBufferTest.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/sort/AbstractBinaryExternalMerger.java b/paimon-core/src/main/java/org/apache/paimon/sort/AbstractBinaryExternalMerger.java index 417ec7a20741..1526e8fe8ed6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/sort/AbstractBinaryExternalMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/sort/AbstractBinaryExternalMerger.java @@ -18,6 +18,7 @@ package org.apache.paimon.sort; +import org.apache.paimon.CoreOptions; import org.apache.paimon.compression.BlockCompressionFactory; import org.apache.paimon.data.AbstractPagedOutputView; import org.apache.paimon.disk.ChannelReaderInputView; @@ -37,6 +38,8 @@ import java.util.Comparator; import java.util.List; +import static org.apache.paimon.utils.Preconditions.checkArgument; + /** * Spilled files Merger of {@link BinaryExternalSortBuffer}. It merges {@link #maxFanIn} spilled * files at most once. @@ -64,6 +67,12 @@ public AbstractBinaryExternalMerger( SpillChannelManager channelManager, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize) { + checkArgument( + maxFanIn >= 2, + "The fan-in for external merge sort must be at least 2, but was %s. " + + "Please adjust '%s'.", + maxFanIn, + CoreOptions.LOCAL_SORT_MAX_NUM_FILE_HANDLES.key()); this.ioManager = ioManager; this.pageSize = pageSize; this.maxFanIn = maxFanIn; diff --git a/paimon-core/src/test/java/org/apache/paimon/sort/BinaryExternalSortBufferTest.java b/paimon-core/src/test/java/org/apache/paimon/sort/BinaryExternalSortBufferTest.java index a23f5104cbb8..0c0cf20397bf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/sort/BinaryExternalSortBufferTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/sort/BinaryExternalSortBufferTest.java @@ -428,4 +428,13 @@ public BinaryRow next() { return row; } } + + @Test + void testFanInBelowTwoFailsFast() { + // a fan-in of 1 previously overflowed the merge arithmetic at flush time and + // restart-looped on the same config; it must be rejected up front + assertThatThrownBy(() -> createBuffer(1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("fan-in for external merge sort must be at least 2"); + } }