From f4e7ab4fb60a24a2dcbd8ad6aea13a9111500362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 11 Aug 2026 14:46:37 +0800 Subject: [PATCH 1/2] [core] Speed up row-id manifest sorting --- docs/generated/core_configuration.html | 6 + .../java/org/apache/paimon/CoreOptions.java | 13 + .../org/apache/paimon/utils/ByteArrayKey.java | 3 +- .../paimon/utils/ByteArrayLookupKey.java | 48 +- .../apache/paimon/utils/ByteArrayKeyTest.java | 17 + .../org/apache/paimon/manifest/FileEntry.java | 13 + .../apache/paimon/manifest/ManifestFile.java | 487 +++++++++++ .../operation/ManifestEntryRunMerge.java | 534 ++++++++++++ .../operation/ManifestEntryRunMergeEntry.java | 435 ++++++++++ .../operation/ManifestEntryRunMergePlan.java | 801 ++++++++++++++++++ .../paimon/operation/ManifestFileSorter.java | 416 ++++++--- .../paimon/manifest/ManifestFileMetaTest.java | 538 +++++++++++- .../org/apache/avro/file/RawBlockReader.java | 194 +++++ .../paimon/format/avro/AvroBlockReader.java | 79 +- .../paimon/format/avro/AvroFileFormat.java | 27 + .../avro/primitive/PrimitiveAvroBlock.java | 55 ++ .../primitive/PrimitiveAvroRecordReader.java | 648 ++++++++++++++ .../avro/primitive/PrimitiveAvroWriter.java | 61 ++ .../format/avro/AvroFileFormatTest.java | 161 ++++ 19 files changed, 4412 insertions(+), 124 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java create mode 100644 paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java create mode 100644 paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java create mode 100644 paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java create mode 100644 paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 70989a7bc76b..84a716637c95 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1023,6 +1023,12 @@ String Partition field name to sort manifest entries by. Validated by schema validation, if not configured, defaults to the first partition field. + +
manifest-sort.run-merge-optimize.enabled
+ true + Boolean + Whether to use streaming run merge for RowID-based manifest sorting. When disabled, the external sorter is used without changing the RowID sort semantics. +
manifest.compression
"zstd" diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index c65da81a80e1..923b5cafadc1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -589,6 +589,15 @@ public InlineElement getDescription() { + " skipped. Set to a larger value to allow more aggressive" + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it."); + public static final ConfigOption MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED = + key("manifest-sort.run-merge-optimize.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to use streaming run merge for RowID-based manifest sorting." + + " When disabled, the external sorter is used without changing" + + " the RowID sort semantics."); + public static final ConfigOption PARTITION_DEFAULT_NAME = key("partition.default-name") .stringType() @@ -3063,6 +3072,10 @@ public long manifestSortMaxRewriteSize() { return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes(); } + public boolean manifestSortRunMergeOptimizeEnabled() { + return options.get(MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED); + } + public String partitionDefaultName() { return options.get(PARTITION_DEFAULT_NAME); } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java index 09d9ded426a4..274e20abdc0a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java @@ -47,8 +47,7 @@ byte[] bytes() { public boolean equals(Object obj) { return obj == this || (obj instanceof ByteArrayKey && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes)) - || (obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes())); + || (obj instanceof ByteArrayLookupKey && obj.equals(this)); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java index aaa913ace7ee..023a6b686609 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java @@ -20,8 +20,6 @@ import javax.annotation.Nullable; -import java.util.Arrays; - import static org.apache.paimon.utils.Preconditions.checkArgument; /** @@ -33,6 +31,8 @@ public final class ByteArrayLookupKey { private @Nullable byte[] bytes; + private int offset; + private int length; private int hash; public ByteArrayLookupKey() {} @@ -43,12 +43,26 @@ public ByteArrayLookupKey(byte[] bytes) { public void reset(byte[] bytes) { checkArgument(bytes != null, "Byte array cannot be null."); + reset(bytes, 0, bytes.length); + } + + public void reset(byte[] bytes, int offset, int length) { + checkArgument(bytes != null, "Byte array cannot be null."); + checkArgument(offset >= 0 && length >= 0 && offset <= bytes.length - length); this.bytes = bytes; - this.hash = Arrays.hashCode(bytes); + this.offset = offset; + this.length = length; + int hash = 1; + for (int i = offset; i < offset + length; i++) { + hash = 31 * hash + bytes[i]; + } + this.hash = hash; } public void clear() { bytes = null; + offset = 0; + length = 0; hash = 0; } @@ -62,14 +76,38 @@ public boolean equals(Object obj) { return obj == this || (bytes != null && obj instanceof ByteArrayKey - && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes())) + && equals(((ByteArrayKey) obj).bytes())) || (bytes != null && obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes)); + && equals((ByteArrayLookupKey) obj)); } @Override public int hashCode() { return hash; } + + private boolean equals(byte[] other) { + if (length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other[i]) { + return false; + } + } + return true; + } + + private boolean equals(ByteArrayLookupKey other) { + if (other.bytes == null || length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other.bytes[other.offset + i]) { + return false; + } + } + return true; + } } diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java index 89c2db09d426..8248cfcb36cc 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java @@ -59,6 +59,23 @@ void testReusableMapLookup() { assertThat(lookup.hashCode()).isZero(); } + @Test + void testReusableSliceLookup() { + Map values = new HashMap<>(); + ByteArrayKey key = new ByteArrayKey(new byte[] {1, 2, 3}); + values.put(key, "value"); + ByteArrayLookupKey lookup = new ByteArrayLookupKey(); + + lookup.reset(new byte[] {9, 1, 2, 3, 8}, 1, 3); + assertThat(lookup).isEqualTo(key); + assertThat(key).isEqualTo(lookup); + assertThat(lookup.hashCode()).isEqualTo(key.hashCode()); + assertThat(values.get(lookup)).isEqualTo("value"); + + lookup.clear(); + assertThat(new ByteArrayLookupKey(new byte[] {1, 2, 3})).isNotEqualTo(lookup); + } + @Test void testLookupEqualityLifecycle() { ByteArrayLookupKey first = new ByteArrayLookupKey(new byte[] {1}); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java index e33f2e6f8563..178a4fa25852 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java @@ -213,6 +213,19 @@ public ReusableIdentifier replaceWithPartition(BinaryManifestEntry entry) { return appendEntryFields(entry); } + /** Replaces this encoding with an already serialized identifier. */ + public ReusableIdentifier replace(byte[] value, int offset, int valueLength) { + checkArgument(value != null, "Serialized identifier cannot be null."); + checkArgument( + offset >= 0 && valueLength >= 0 && offset <= value.length - valueLength, + "Identifier byte range is invalid."); + length = 0; + ensureCapacity(valueLength); + System.arraycopy(value, offset, bytes, 0, valueLength); + length = valueLength; + return this; + } + private ReusableIdentifier appendEntryFields(BinaryManifestEntry entry) { putInt(entry.bucket()); BinaryDataFileMeta file = entry.file(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 45e0a743785a..39d5d126f7ce 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -19,12 +19,19 @@ package org.apache.paimon.manifest; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsCollector; +import org.apache.paimon.format.avro.AvroFileFormat; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroBlock; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroWriter; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.io.RollingFileWriterImpl; @@ -39,6 +46,7 @@ import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.FileUtils; import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.ObjectsFile; import org.apache.paimon.utils.PathFactory; import org.apache.paimon.utils.SegmentsCache; @@ -47,9 +55,12 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.function.Function; /** @@ -62,6 +73,7 @@ public class ManifestFile extends ObjectsFile { private final SchemaManager schemaManager; private final RowType partitionType; + private final AvroFileFormat avroFileFormat; private final FormatWriterFactory writerFactory; private final long suggestedFileSize; @@ -69,6 +81,7 @@ private ManifestFile( FileIO fileIO, SchemaManager schemaManager, RowType partitionType, + AvroFileFormat avroFileFormat, ManifestEntrySerializer serializer, FormatWriterFactory writerFactory, String compression, @@ -88,6 +101,7 @@ private ManifestFile( cache); this.schemaManager = schemaManager; this.partitionType = partitionType; + this.avroFileFormat = avroFileFormat; this.writerFactory = writerFactory; this.suggestedFileSize = suggestedFileSize; } @@ -209,6 +223,94 @@ private static CloseableIterator createManifestIterator( } } + /** Opens an allocation-free reader for selected primitive manifest fields. */ + public PrimitiveAvroRecordReader scanPrimitive( + String fileName, @Nullable Long fileSize, Projection projection) { + try { + return avroFileFormat.createPrimitiveReader( + fileIO, + pathFactory.toPath(fileName), + ManifestEntry.MANIFEST_ROW_TYPE, + projection.projectedType()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest file " + fileName, e); + } + } + + /** Reusable statistics needed when an encoded manifest record is copied directly. */ + public static final class EncodedManifestEntry { + + private byte kind; + private BinaryRow partition; + private int bucket; + private int level; + private long schemaId; + private long firstRowId; + private long rowCount; + + public EncodedManifestEntry replace( + byte kind, + BinaryRow partition, + int bucket, + int level, + long schemaId, + long firstRowId, + long rowCount) { + this.kind = kind; + this.partition = partition; + this.bucket = bucket; + this.level = level; + this.schemaId = schemaId; + this.firstRowId = firstRowId; + this.rowCount = rowCount; + return this; + } + } + + /** Aggregate statistics for an encoded Avro block copied without decompression. */ + public static final class EncodedManifestBlock { + + private final long addedFiles; + private final long schemaId; + private final int minBucket; + private final int maxBucket; + private final int minLevel; + private final int maxLevel; + private final long minRowId; + private final long maxRowId; + private final @Nullable BinaryRow nullPartition; + private final long nullPartitionCount; + private final @Nullable BinaryRow minNonNullPartition; + private final @Nullable BinaryRow maxNonNullPartition; + + public EncodedManifestBlock( + long addedFiles, + long schemaId, + int minBucket, + int maxBucket, + int minLevel, + int maxLevel, + long minRowId, + long maxRowId, + @Nullable BinaryRow nullPartition, + long nullPartitionCount, + @Nullable BinaryRow minNonNullPartition, + @Nullable BinaryRow maxNonNullPartition) { + this.addedFiles = addedFiles; + this.schemaId = schemaId; + this.minBucket = minBucket; + this.maxBucket = maxBucket; + this.minLevel = minLevel; + this.maxLevel = maxLevel; + this.minRowId = minRowId; + this.maxRowId = maxRowId; + this.nullPartition = nullPartition; + this.nullPartitionCount = nullPartitionCount; + this.minNonNullPartition = minNonNullPartition; + this.maxNonNullPartition = maxNonNullPartition; + } + } + @VisibleForTesting public long suggestedFileSize() { return suggestedFileSize; @@ -283,6 +385,11 @@ public ManifestEntryWriter createManifestEntryWriter(Path manifestPath) { return new ManifestEntryWriter(writerFactory, manifestPath, compression); } + /** Creates the Manifest-specific writer used by the primitive Avro merge path. */ + public PrimitiveManifestRollingWriter createPrimitiveRollingWriter() { + return new PrimitiveManifestRollingWriter(); + } + /** Writer for {@link ManifestEntry}. */ public class ManifestEntryWriter extends SingleFileWriter { @@ -318,6 +425,10 @@ public void write(ManifestEntry entry) throws IOException { super.write(entry); } + collectStats(entry); + } + + private void collectStats(ManifestEntry entry) { switch (entry.kind()) { case ADD: numAddedFiles++; @@ -365,6 +476,376 @@ public ManifestFileMeta result() throws IOException { } } + /** + * Hard-coded rolling writer for Manifest Avro records. + * + *

This deliberately does not extend the generic rolling-writer abstractions: encoded Avro + * records and blocks are an implementation detail of manifest run merging. + */ + public final class PrimitiveManifestRollingWriter implements AutoCloseable { + + private final List results = new ArrayList<>(); + private final List completedPaths = new ArrayList<>(); + private @Nullable PrimitiveManifestFileWriter currentWriter; + private long recordCount; + private boolean closed; + + public void write(ManifestEntry entry) throws IOException { + try { + currentWriter().write(entry); + afterWrite(1, false); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + public void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadata) + throws IOException { + try { + currentWriter().writeEncoded(encodedRecord, metadata); + afterWrite(1, false); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + public void writeEncodedBlock( + PrimitiveAvroBlock block, EncodedManifestBlock metadata, long blockRecordCount) + throws IOException { + if (blockRecordCount != block.recordCount()) { + throw new IllegalArgumentException( + String.format( + "Manifest block record count mismatch: expected %s, actual %s.", + blockRecordCount, block.recordCount())); + } + try { + currentWriter().writeEncodedBlock(block, metadata); + afterWrite(blockRecordCount, true); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + private PrimitiveManifestFileWriter currentWriter() { + if (closed) { + throw new IllegalStateException("Manifest writer has already closed."); + } + if (currentWriter == null) { + currentWriter = new PrimitiveManifestFileWriter(pathFactory.newPath()); + } + return currentWriter; + } + + private void afterWrite(long addedRecords, boolean forceSizeCheck) throws IOException { + recordCount = Math.addExact(recordCount, addedRecords); + if (currentWriter.reachTargetSize( + forceSizeCheck || recordCount % RollingFileWriter.CHECK_ROLLING_RECORD_CNT == 0, + suggestedFileSize)) { + closeCurrentWriter(); + } + } + + private void closeCurrentWriter() throws IOException { + if (currentWriter == null) { + return; + } + currentWriter.close(); + ManifestFileMeta result = currentWriter.result(); + completedPaths.add(currentWriter.path); + results.add(result); + currentWriter = null; + } + + public long recordCount() { + return recordCount; + } + + public List result() { + if (!closed) { + throw new IllegalStateException( + "Cannot access manifest results before closing the writer."); + } + return results; + } + + public void abort() { + if (currentWriter != null) { + currentWriter.abort(); + currentWriter = null; + } + for (Path path : completedPaths) { + fileIO.deleteQuietly(path); + } + completedPaths.clear(); + results.clear(); + closed = true; + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + try { + closeCurrentWriter(); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } finally { + closed = true; + } + } + } + + /** Single-file counterpart of {@link PrimitiveManifestRollingWriter}. */ + private final class PrimitiveManifestFileWriter { + + private final Path path; + private final SimpleStatsCollector partitionStatsCollector; + private final SimpleStatsConverter partitionStatsSerializer; + private final Map encodedPartitionCounts = new IdentityHashMap<>(); + private final long[] repeatedNullCounts = new long[partitionType.getFieldCount()]; + private @Nullable PositionOutputStream out; + private @Nullable PrimitiveAvroWriter writer; + private @Nullable Long outputBytes; + private long numAddedFiles; + private long numDeletedFiles; + private long schemaId = Long.MIN_VALUE; + private int minBucket = Integer.MAX_VALUE; + private int maxBucket = Integer.MIN_VALUE; + private int minLevel = Integer.MAX_VALUE; + private int maxLevel = Integer.MIN_VALUE; + private @Nullable RowIdStats rowIdStats = new RowIdStats(); + private boolean closed; + + private PrimitiveManifestFileWriter(Path path) { + this.path = path; + this.partitionStatsCollector = new SimpleStatsCollector(partitionType); + this.partitionStatsSerializer = new SimpleStatsConverter(partitionType); + boolean outputCreated = false; + try { + out = fileIO.newOutputStream(path, false); + outputCreated = true; + writer = + avroFileFormat.createPrimitiveWriter( + out, ManifestEntry.MANIFEST_ROW_TYPE, compression); + } catch (IOException failure) { + IOUtils.closeQuietly(writer); + IOUtils.closeQuietly(out); + if (outputCreated) { + fileIO.deleteQuietly(path); + } + throw new UncheckedIOException( + "Failed to create primitive manifest writer for " + path, failure); + } catch (RuntimeException | Error failure) { + IOUtils.closeQuietly(writer); + IOUtils.closeQuietly(out); + if (outputCreated) { + fileIO.deleteQuietly(path); + } + throw failure; + } + } + + private void write(ManifestEntry entry) throws IOException { + ensureOpen(); + writer.addElement( + entry instanceof BinaryManifestEntry + ? ((BinaryManifestEntry) entry).fullRow() + : serializer.toRow(entry)); + collectStats(entry); + } + + private void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadata) + throws IOException { + ensureOpen(); + writer.addEncoded(encodedRecord); + collectStats(metadata); + addEncodedPartition(metadata.partition, 1); + } + + private void writeEncodedBlock(PrimitiveAvroBlock block, EncodedManifestBlock metadata) + throws IOException { + ensureOpen(); + writer.addEncodedBlock(block); + collectStats(metadata); + if (metadata.nullPartitionCount > 0) { + addEncodedPartition(metadata.nullPartition, metadata.nullPartitionCount); + } + if (metadata.minNonNullPartition != null) { + addEncodedPartition(metadata.minNonNullPartition, 1); + if (metadata.maxNonNullPartition != metadata.minNonNullPartition) { + addEncodedPartition(metadata.maxNonNullPartition, 1); + } + } + } + + private void collectStats(ManifestEntry entry) { + switch (entry.kind()) { + case ADD: + numAddedFiles++; + break; + case DELETE: + numDeletedFiles++; + break; + default: + throw new UnsupportedOperationException("Unknown entry kind: " + entry.kind()); + } + schemaId = Math.max(schemaId, entry.file().schemaId()); + minBucket = Math.min(minBucket, entry.bucket()); + maxBucket = Math.max(maxBucket, entry.bucket()); + minLevel = Math.min(minLevel, entry.level()); + maxLevel = Math.max(maxLevel, entry.level()); + if (rowIdStats != null) { + Long firstRowId = entry.file().firstRowId(); + if (firstRowId == null) { + rowIdStats = null; + } else { + rowIdStats.collect(firstRowId, entry.file().rowCount()); + } + } + partitionStatsCollector.collect(entry.partition()); + } + + private void collectStats(EncodedManifestEntry entry) { + switch (FileKind.fromByteValue(entry.kind)) { + case ADD: + numAddedFiles++; + break; + case DELETE: + numDeletedFiles++; + break; + default: + throw new UnsupportedOperationException("Unknown entry kind: " + entry.kind); + } + schemaId = Math.max(schemaId, entry.schemaId); + minBucket = Math.min(minBucket, entry.bucket); + maxBucket = Math.max(maxBucket, entry.bucket); + minLevel = Math.min(minLevel, entry.level); + maxLevel = Math.max(maxLevel, entry.level); + if (rowIdStats != null) { + rowIdStats.collect(entry.firstRowId, entry.rowCount); + } + } + + private void collectStats(EncodedManifestBlock block) { + numAddedFiles = Math.addExact(numAddedFiles, block.addedFiles); + schemaId = Math.max(schemaId, block.schemaId); + minBucket = Math.min(minBucket, block.minBucket); + maxBucket = Math.max(maxBucket, block.maxBucket); + minLevel = Math.min(minLevel, block.minLevel); + maxLevel = Math.max(maxLevel, block.maxLevel); + if (rowIdStats != null) { + rowIdStats.collectRange(block.minRowId, block.maxRowId); + } + } + + private void addEncodedPartition(@Nullable BinaryRow partition, long count) { + if (partition == null || count <= 0) { + return; + } + long[] value = + encodedPartitionCounts.computeIfAbsent(partition, ignored -> new long[1]); + value[0] = Math.addExact(value[0], count); + } + + private SimpleColStats[] partitionStats() { + for (Map.Entry entry : encodedPartitionCounts.entrySet()) { + BinaryRow partition = entry.getKey(); + partitionStatsCollector.collect(partition); + long repeated = entry.getValue()[0] - 1; + if (repeated <= 0) { + continue; + } + for (int field = 0; field < partition.getFieldCount(); field++) { + if (partition.isNullAt(field)) { + repeatedNullCounts[field] = + Math.addExact(repeatedNullCounts[field], repeated); + } + } + } + encodedPartitionCounts.clear(); + SimpleColStats[] stats = partitionStatsCollector.extract(); + for (int field = 0; field < stats.length; field++) { + if (repeatedNullCounts[field] == 0) { + continue; + } + SimpleColStats current = stats[field]; + stats[field] = + new SimpleColStats( + current.min(), + current.max(), + Math.addExact(current.nullCount(), repeatedNullCounts[field])); + } + return stats; + } + + private boolean reachTargetSize(boolean suggestedCheck, long targetSize) + throws IOException { + ensureOpen(); + return writer.reachTargetSize(suggestedCheck, targetSize); + } + + private void ensureOpen() { + if (closed || writer == null) { + throw new IllegalStateException("Manifest writer has already closed."); + } + } + + private void abort() { + IOUtils.closeQuietly(writer); + writer = null; + IOUtils.closeQuietly(out); + out = null; + fileIO.deleteQuietly(path); + closed = true; + } + + private void close() throws IOException { + if (closed) { + return; + } + try { + writer.close(); + writer = null; + out.flush(); + outputBytes = out.getPos(); + out.close(); + out = null; + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } finally { + closed = true; + } + } + + private ManifestFileMeta result() { + if (!closed || outputBytes == null) { + throw new IllegalStateException( + "Cannot access manifest result before closing the writer."); + } + return new ManifestFileMeta( + path.getName(), + outputBytes, + numAddedFiles, + numDeletedFiles, + partitionStatsSerializer.toBinaryAllMode(partitionStats()), + numAddedFiles + numDeletedFiles > 0 + ? schemaId + : schemaManager.latest().get().id(), + minBucket, + maxBucket, + minLevel, + maxLevel, + rowIdStats == null ? null : rowIdStats.minRowId, + rowIdStats == null ? null : rowIdStats.maxRowId); + } + } + private static class RowIdStats { private long minRowId = Long.MAX_VALUE; @@ -374,6 +855,11 @@ private void collect(long firstRowId, long rowCount) { minRowId = Math.min(minRowId, firstRowId); maxRowId = Math.max(maxRowId, firstRowId + rowCount - 1); } + + private void collectRange(long minRowId, long maxRowId) { + this.minRowId = Math.min(this.minRowId, minRowId); + this.maxRowId = Math.max(this.maxRowId, maxRowId); + } } /** Creator of {@link ManifestFile}. */ @@ -417,6 +903,7 @@ public ManifestFile create() { fileIO, schemaManager, partitionType, + (AvroFileFormat) fileFormat, new ManifestEntrySerializer(), fileFormat.createWriterFactory(entryType), compression, diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java new file mode 100644 index 000000000000..4e0648b24770 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java @@ -0,0 +1,534 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader.Record; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestBlock; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; + +/** Streaming merge of the naturally sorted runs in data-evolution manifest files. */ +final class ManifestEntryRunMerge { + + private static final int FRAGMENTED_RUN_THRESHOLD = 64; + private static final long MAX_IN_MEMORY_FRAGMENTED_ENTRIES = 25_000L; + private static final int MAX_STREAM_CURSORS = 128; + private static final int MAX_STREAM_READ_AMPLIFICATION = 8; + static final int KIND = 0; + static final int PARTITION = 1; + static final int BUCKET = 2; + static final int FILE_NAME = 3; + static final int ROW_COUNT = 4; + static final int LEVEL = 5; + static final int SCHEMA_ID = 6; + static final int FIRST_ROW_ID = 7; + static final int MAX_SEQUENCE_NUMBER = 8; + static final int EXTRA_FILES = 9; + static final int EMBEDDED_FILE_INDEX = 10; + static final int EXTERNAL_PATH = 11; + static final BinaryManifestEntry.Projection ENTRY_LAYOUT = entryLayout(); + + private ManifestEntryRunMerge() {} + + private static BinaryManifestEntry.Projection entryLayout() { + List fields = new ArrayList<>(); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.PARTITION)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add( + ManifestEntry.MANIFEST_ROW_TYPE + .getField(ManifestEntry.FILE) + .newType( + DataFileMeta.SCHEMA.project( + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH))); + return BinaryManifestEntry.Projection.create(new RowType(false, fields)); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge. The caller must + * fall back to the spillable external sorter in that case. + */ + @Nullable + static List sortAndWriteFullEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.Filter filter = + new ManifestEntryRunMergeEntry.Filter(deletedIdentifiers, deletedRowIds); + ManifestEntryRunMergePlan plan = + discoverRuns(section, sortKey, manifestFile, filter, manifestReadParallelism); + if (plan == null) { + return null; + } + return plan.mergeToManifest(sortKey, manifestFile, filter, newFilesForAbort); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge or primitive + * manifest reading is unavailable. The caller must fall back to the spillable external sorter. + */ + @Nullable + static Pair, List> sortAndWriteMinorEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + @Nullable Integer manifestReadParallelism) + throws Exception { + CompactFileIdentifierSet deletedIdentifiers = new CompactFileIdentifierSet(); + ManifestFileSorter.DeletedRowIdSet deletedRowIds = new ManifestFileSorter.DeletedRowIdSet(); + ManifestEntryRunMergeEntry.Filter.Minor filter = + new ManifestEntryRunMergeEntry.Filter.Minor(deletedIdentifiers, deletedRowIds); + try { + ManifestEntryRunMergePlan plan; + try { + plan = + discoverRuns( + section, sortKey, manifestFile, filter, manifestReadParallelism); + } finally { + deletedRowIds.releaseRangeIndex(); + } + if (plan == null) { + return null; + } + return plan.mergeMinorToManifest( + sortKey, + manifestFile, + filter, + deletedIdentifiers, + deletedRowIds, + newFilesForAbort); + } finally { + deletedIdentifiers.release(); + } + } + + @Nullable + private static ManifestEntryRunMergePlan discoverRuns( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.PartitionDictionary partitions = + new ManifestEntryRunMergeEntry.PartitionDictionary(sortKey); + List sources = new ArrayList<>(); + int streamCursorCount = 0; + long inMemoryEntries = 0; + List discovered = new ArrayList<>(section.size()); + if (section.size() <= 1 + || manifestReadParallelism == null + || manifestReadParallelism <= 1) { + for (ManifestFileMeta meta : section) { + Discovery.DiscoveredManifest manifest = + discoverManifestRuns(meta, manifestFile, partitions, filter); + if (manifest.requiresExternalSort) { + return null; + } + discovered.add(manifest); + } + } else { + Function> reader = + meta -> { + try { + return Collections.singletonList( + discoverManifestRuns(meta, manifestFile, partitions, filter)); + } catch (Exception e) { + throw new RuntimeException( + "Failed to discover sorted Avro runs in " + meta.fileName(), e); + } + }; + for (Discovery.DiscoveredManifest manifest : + sequentialBatchedExecute(reader, section, manifestReadParallelism)) { + discovered.add(manifest); + } + } + for (int manifestIndex = 0; manifestIndex < section.size(); manifestIndex++) { + ManifestFileMeta meta = section.get(manifestIndex); + Discovery.DiscoveredManifest manifest = discovered.get(manifestIndex); + if (manifest.requiresExternalSort) { + return null; + } + if (manifest.fragmented) { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + inMemoryEntries += entryCount; + if (inMemoryEntries > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return null; + } + sources.add(new ManifestEntryRunMergePlan.Source.FragmentedManifestSpec(meta)); + streamCursorCount++; + } else { + sources.addAll(manifest.runs); + streamCursorCount += manifest.runs.size(); + } + if (streamCursorCount > MAX_STREAM_CURSORS) { + return null; + } + } + partitions.finish(); + for (Discovery.DiscoveredManifest manifest : discovered) { + manifest.finishFiltering(filter); + manifest.updatePartitionRanks(partitions); + } + return new ManifestEntryRunMergePlan(sources, partitions); + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + try (PrimitiveAvroRecordReader reader = + manifestFile.scanPrimitive(meta.fileName(), meta.fileSize(), ENTRY_LAYOUT)) { + return discoverManifestRuns(meta, reader, partitions, filter); + } catch (UnsupportedOperationException unsupported) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + PrimitiveAvroRecordReader reader, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + List runs = new ArrayList<>(); + List blocks = new ArrayList<>(); + ManifestEntryRunMergeEntry.Key previous = new ManifestEntryRunMergeEntry.Key(); + ManifestEntryRunMergeEntry.Key current = new ManifestEntryRunMergeEntry.Key(); + boolean hasPrevious = false; + long runStart = 0; + long position = 0; + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + boolean fragmented = false; + while (reader.hasNext()) { + Record record = reader.next(); + current.replace(record, partitions); + filter.observe(record, current); + if (fragmented) { + position++; + continue; + } + if (record.blockRecordIndex() == 0) { + blocks.add( + new Discovery.BlockInfo( + record.blockOrdinal(), + position, + reader.rawBlockCopySupported(), + current.stableCopy())); + } + Discovery.BlockInfo block = blocks.get(blocks.size() - 1); + block.collect(record, current, partitions, filter); + boolean inversion = + hasPrevious && compareDiscoveryKeys(previous, current, partitions) > 0; + if (inversion) { + if (record.blockRecordIndex() > 0) { + block.sorted = false; + } + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + runStart = position; + if (runs.size() >= FRAGMENTED_RUN_THRESHOLD) { + if (entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + fragmented = true; + runs.clear(); + blocks.clear(); + position++; + continue; + } + } + position++; + if (record.blockRecordIndex() + 1 == record.blockRecordCount()) { + ManifestEntryRunMergeEntry.Key stableLastKey = current.stableCopy(); + block.finish(position, stableLastKey); + previous.copyFrom(stableLastKey); + } else { + previous.copyFrom(current); + } + hasPrevious = true; + } + if (fragmented) { + return Discovery.DiscoveredManifest.fragmented(); + } + if (position > runStart) { + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + } + if (exceedsStreamingReadAmplification(runs, blocks.size())) { + return entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES + ? Discovery.DiscoveredManifest.requiresExternalSort() + : Discovery.DiscoveredManifest.fragmented(); + } + return Discovery.DiscoveredManifest.runs(runs, blocks); + } + + private static boolean exceedsStreamingReadAmplification( + List runs, int blockCount) { + if (runs.size() <= 1 || blockCount == 0) { + return false; + } + + long prefixBlocksRead = 0; + for (ManifestEntryRunMergePlan.Source.ManifestRunSpec run : runs) { + prefixBlocksRead += run.prefixBlockCount(); + } + return prefixBlocksRead > (long) blockCount * MAX_STREAM_READ_AMPLIFICATION; + } + + private static int compareDiscoveryKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + return compareRemainingKeys( + left, right, partitions.compareIds(left.partitionId, right.partitionId)); + } + + static int compareMergeKeys( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + return compareRemainingKeys( + left, right, Integer.compare(left.partitionRank, right.partitionRank)); + } + + private static int compareRemainingKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + int comparison) { + if (comparison == 0) { + comparison = Byte.compare(left.kind, right.kind); + } + if (comparison == 0) { + comparison = Long.compare(left.firstRowId, right.firstRowId); + } + if (comparison == 0) { + comparison = Long.compare(left.rangeEnd, right.rangeEnd); + } + if (comparison == 0) { + comparison = Long.compare(left.reverseSequence, right.reverseSequence); + } + if (comparison == 0) { + comparison = compareBytes(left, right); + } + return comparison; + } + + private static int compareBytes( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + int minLength = Math.min(left.fileNameLength, right.fileNameLength); + for (int i = 0; i < minLength; i++) { + int leftByte = left.fileNameBytes[left.fileNameOffset + i] & 0xFF; + int rightByte = right.fileNameBytes[right.fileNameOffset + i] & 0xFF; + if (leftByte != rightByte) { + return leftByte - rightByte; + } + } + return left.fileNameLength - right.fileNameLength; + } + + /** Results and Avro block metadata collected while discovering natural manifest runs. */ + static final class Discovery { + + private Discovery() {} + + static final class DiscoveredManifest { + + final List runs; + final List blocks; + final boolean fragmented; + final boolean requiresExternalSort; + + DiscoveredManifest( + List runs, + List blocks, + boolean fragmented, + boolean requiresExternalSort) { + this.runs = runs; + this.blocks = blocks; + this.fragmented = fragmented; + this.requiresExternalSort = requiresExternalSort; + } + + static DiscoveredManifest runs( + List runs, + List blocks) { + return new DiscoveredManifest(runs, blocks, false, false); + } + + static DiscoveredManifest fragmented() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), true, false); + } + + static DiscoveredManifest requiresExternalSort() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), false, true); + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + for (BlockInfo block : blocks) { + block.updatePartitionRanks(partitions); + } + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + for (BlockInfo block : blocks) { + block.finishFiltering(filter); + } + } + } + + static final class BlockInfo { + + final long ordinal; + final long start; + final ManifestEntryRunMergeEntry.Key firstKey; + boolean eligible; + boolean sorted = true; + long end; + ManifestEntryRunMergeEntry.Key lastKey; + long addedFiles; + long schemaId = Long.MIN_VALUE; + int minBucket = Integer.MAX_VALUE; + int maxBucket = Integer.MIN_VALUE; + int minLevel = Integer.MAX_VALUE; + int maxLevel = Integer.MIN_VALUE; + long minRowId = Long.MAX_VALUE; + long maxRowId = Long.MIN_VALUE; + BinaryRow nullPartition; + long nullPartitionCount; + BinaryRow minNonNullPartition; + BinaryRow maxNonNullPartition; + EncodedManifestBlock metadata; + + BlockInfo( + long ordinal, + long start, + boolean eligible, + ManifestEntryRunMergeEntry.Key firstKey) { + this.ordinal = ordinal; + this.start = start; + this.eligible = eligible; + this.firstKey = firstKey; + } + + void collect( + Record record, + ManifestEntryRunMergeEntry.Key key, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) { + BinaryRow partition = partitions.partition(key.partitionId); + eligible &= partition.getFieldCount() == 1 && filter.copyable(record, key); + if (!eligible) { + return; + } + addedFiles++; + schemaId = Math.max(schemaId, record.longValue(SCHEMA_ID)); + int bucket = (int) record.longValue(BUCKET); + minBucket = Math.min(minBucket, bucket); + maxBucket = Math.max(maxBucket, bucket); + int level = (int) record.longValue(LEVEL); + minLevel = Math.min(minLevel, level); + maxLevel = Math.max(maxLevel, level); + minRowId = Math.min(minRowId, key.firstRowId); + maxRowId = Math.max(maxRowId, key.rangeEnd); + if (partition.isNullAt(0)) { + nullPartition = partition; + nullPartitionCount++; + } else { + if (minNonNullPartition == null) { + minNonNullPartition = partition; + } + maxNonNullPartition = partition; + } + } + + void finish(long end, ManifestEntryRunMergeEntry.Key lastKey) { + this.end = end; + this.lastKey = lastKey; + if (eligible && sorted) { + metadata = + new EncodedManifestBlock( + addedFiles, + schemaId, + minBucket, + maxBucket, + minLevel, + maxLevel, + minRowId, + maxRowId, + nullPartition, + nullPartitionCount, + minNonNullPartition, + maxNonNullPartition); + } + } + + boolean copyable(long runStart, long runEnd) { + return metadata != null && start >= runStart && end <= runEnd; + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + if (metadata != null && !filter.copyableAfterDiscovery(minRowId, maxRowId)) { + metadata = null; + } + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + firstKey.partitionRank = partitions.rank(firstKey.partitionId); + lastKey.partitionRank = partitions.rank(lastKey.partitionId); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java new file mode 100644 index 000000000000..7cb861abefca --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader.Record; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.utils.ByteArrayKey; +import org.apache.paimon.utils.ByteArrayLookupKey; +import org.apache.paimon.utils.SerializationUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Entry-level state shared by manifest run discovery and merge execution. */ +final class ManifestEntryRunMergeEntry { + + private ManifestEntryRunMergeEntry() {} + + static final class Key { + + int partitionId; + int partitionRank; + byte kind; + long firstRowId; + long rangeEnd; + long reverseSequence; + byte[] fileNameBytes; + int fileNameOffset; + int fileNameLength; + + static Key viewOf(BinaryManifestEntry entry, PartitionDictionary partitions) { + Key key = new Key(); + key.replace(entry, partitions); + return key; + } + + void replace(BinaryManifestEntry entry, PartitionDictionary partitions) { + long firstRowId = entry.file().nonNullFirstRowId(); + this.partitionId = partitions.id(entry.partitionBytes()); + this.partitionRank = partitions.rank(partitionId); + this.kind = entry.kind().toByteValue(); + this.firstRowId = firstRowId; + this.rangeEnd = firstRowId + entry.file().rowCount() - 1L; + this.reverseSequence = Long.MAX_VALUE - entry.file().maxSequenceNumber(); + this.fileNameBytes = entry.file().fileNameBinary().toBytes(); + this.fileNameOffset = 0; + this.fileNameLength = fileNameBytes.length; + } + + void replace(Record record, PartitionDictionary partitions) { + checkState( + !record.isNull(ManifestEntryRunMerge.FIRST_ROW_ID), + "First row id should not be null."); + this.partitionId = + partitions.id( + record.bytes(ManifestEntryRunMerge.PARTITION), + record.offset(ManifestEntryRunMerge.PARTITION), + record.length(ManifestEntryRunMerge.PARTITION)); + this.partitionRank = partitions.rank(partitionId); + this.kind = (byte) record.longValue(ManifestEntryRunMerge.KIND); + this.firstRowId = record.longValue(ManifestEntryRunMerge.FIRST_ROW_ID); + this.rangeEnd = firstRowId + record.longValue(ManifestEntryRunMerge.ROW_COUNT) - 1L; + this.reverseSequence = + Long.MAX_VALUE - record.longValue(ManifestEntryRunMerge.MAX_SEQUENCE_NUMBER); + this.fileNameBytes = record.bytes(ManifestEntryRunMerge.FILE_NAME); + this.fileNameOffset = record.offset(ManifestEntryRunMerge.FILE_NAME); + this.fileNameLength = record.length(ManifestEntryRunMerge.FILE_NAME); + } + + void copyFrom(Key key) { + this.partitionId = key.partitionId; + this.partitionRank = key.partitionRank; + this.kind = key.kind; + this.firstRowId = key.firstRowId; + this.rangeEnd = key.rangeEnd; + this.reverseSequence = key.reverseSequence; + this.fileNameBytes = key.fileNameBytes; + this.fileNameOffset = key.fileNameOffset; + this.fileNameLength = key.fileNameLength; + } + + Key stableCopy() { + Key copy = new Key(); + copy.copyFrom(this); + copy.fileNameBytes = + Arrays.copyOfRange( + fileNameBytes, fileNameOffset, fileNameOffset + fileNameLength); + copy.fileNameOffset = 0; + return copy; + } + + void clear() { + fileNameBytes = null; + } + } + + /** Interns variable-width partition bytes once and assigns comparator-compatible ranks. */ + static final class PartitionDictionary { + + final ManifestFileSorter.RowIdEntrySortKey sortKey; + final Map ids = new ConcurrentHashMap<>(); + final ThreadLocal lookup = + ThreadLocal.withInitial(ByteArrayLookupKey::new); + volatile BinaryRow[] partitions = new BinaryRow[16]; + int partitionCount; + int[] ranks; + + PartitionDictionary(ManifestFileSorter.RowIdEntrySortKey sortKey) { + this.sortKey = sortKey; + } + + int id(byte[] bytes) { + return id(bytes, 0, bytes.length); + } + + int id(byte[] bytes, int offset, int length) { + ByteArrayLookupKey lookupKey = lookup.get(); + lookupKey.reset(bytes, offset, length); + try { + Integer existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + synchronized (this) { + existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + checkState(ranks == null, "Full manifest scan found an unknown partition."); + byte[] canonical = Arrays.copyOfRange(bytes, offset, offset + length); + int id = partitionCount; + if (id == partitions.length) { + partitions = Arrays.copyOf(partitions, partitions.length << 1); + } + partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); + ids.put(new ByteArrayKey(canonical), id); + partitionCount = id + 1; + return id; + } + } finally { + lookupKey.clear(); + } + } + + int compareIds(int left, int right) { + return sortKey.comparePartitions(partitions[left], partitions[right]); + } + + void finish() { + List order = new ArrayList<>(partitionCount); + for (int id = 0; id < partitionCount; id++) { + order.add(id); + } + order.sort((left, right) -> compareIds(left, right)); + ranks = new int[partitionCount]; + int rank = 0; + for (int position = 0; position < order.size(); position++) { + if (position > 0 && compareIds(order.get(position - 1), order.get(position)) != 0) { + rank++; + } + ranks[order.get(position)] = rank; + } + } + + int rank(int id) { + return ranks == null ? 0 : ranks[id]; + } + + BinaryRow partition(int id) { + return partitions[id]; + } + } + + static class Filter { + + final CompactFileIdentifierSet deletedIdentifiers; + final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final ThreadLocal identifier = + ThreadLocal.withInitial(IdentifierEncoder::new); + + Filter( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + this.deletedIdentifiers = deletedIdentifiers; + this.deletedRowIds = deletedRowIds; + } + + boolean include(BinaryManifestEntry entry) { + return entry.isAdd() && !deletedIdentifiers.contains(entry); + } + + boolean include(Record record, Key key) { + if (key.kind != FileKind.ADD.toByteValue()) { + return false; + } + if (!deletedRowIds.contains(key.firstRowId)) { + return true; + } + + ReusableIdentifier reusable = identifier.get().replace(record); + return !deletedIdentifiers.contains(reusable); + } + + boolean copyable(Record record, Key key) { + return include(record, key); + } + + void observe(Record record, Key key) {} + + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + return true; + } + + ReusableIdentifier identifier(Record record) { + return identifier.get().replace(record); + } + + static final class Minor extends Filter { + + Minor( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + super(deletedIdentifiers, deletedRowIds); + } + + @Override + boolean include(BinaryManifestEntry entry) { + return true; + } + + @Override + boolean include(Record record, Key key) { + return true; + } + + @Override + boolean copyable(Record record, Key key) { + return key.kind == FileKind.ADD.toByteValue(); + } + + @Override + void observe(Record record, Key key) { + if (key.kind != FileKind.DELETE.toByteValue()) { + return; + } + ReusableIdentifier reusable = identifier(record); + synchronized (this) { + deletedIdentifiers.add(reusable); + deletedRowIds.add(key.firstRowId); + } + } + + @Override + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + // A DELETE preserves the deleted ADD's globally unique first RowID. A range hit may + // be a false positive and only disables block copying; a miss proves the block has + // no deleted ADD. + return !deletedRowIds.intersects(minRowId, maxRowId); + } + } + + private static final class IdentifierEncoder { + + byte[] bytes = new byte[256]; + int length; + final AvroSlice avroSlice = new AvroSlice(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + + ReusableIdentifier replace(Record record) { + length = 0; + putRaw(record, ManifestEntryRunMerge.PARTITION); + putInt((int) record.longValue(ManifestEntryRunMerge.BUCKET)); + putInt((int) record.longValue(ManifestEntryRunMerge.LEVEL)); + putRaw(record, ManifestEntryRunMerge.FILE_NAME); + putStringArray(record, ManifestEntryRunMerge.EXTRA_FILES); + putNullableRaw(record, ManifestEntryRunMerge.EMBEDDED_FILE_INDEX); + putNullableRaw(record, ManifestEntryRunMerge.EXTERNAL_PATH); + return identifier.replace(bytes, 0, length); + } + + void putRaw(Record record, int field) { + int valueLength = record.length(field); + putInt(valueLength); + appendRaw(record.bytes(field), record.offset(field), valueLength); + } + + void putNullableRaw(Record record, int field) { + if (record.isNull(field)) { + putInt(-1); + } else { + putRaw(record, field); + } + } + + void putStringArray(Record record, int field) { + AvroSlice slice = avroSlice.reset(record, field); + long count = 0; + while (true) { + long blockCount = slice.readLong(); + if (blockCount == 0) { + break; + } + if (blockCount < 0) { + blockCount = -blockCount; + slice.readLong(); + } + count = Math.addExact(count, blockCount); + for (long i = 0; i < blockCount; i++) { + slice.skipBytes(); + } + } + checkState(count <= Integer.MAX_VALUE, "Too many extra files in manifest entry."); + putInt((int) count); + + slice.reset(record, field); + while (true) { + long blockCount = slice.readLong(); + if (blockCount == 0) { + break; + } + if (blockCount < 0) { + blockCount = -blockCount; + slice.readLong(); + } + for (long i = 0; i < blockCount; i++) { + int valueLength = slice.readLength(); + putInt(valueLength); + appendRaw(slice.bytes, slice.position, valueLength); + slice.position += valueLength; + } + } + slice.checkConsumed(); + } + + void putInt(int value) { + ensureCapacity(Integer.BYTES); + bytes[length++] = (byte) (value >>> 24); + bytes[length++] = (byte) (value >>> 16); + bytes[length++] = (byte) (value >>> 8); + bytes[length++] = (byte) value; + } + + void appendRaw(byte[] value, int offset, int valueLength) { + checkState( + offset >= 0 && valueLength >= 0 && offset <= value.length - valueLength, + "Identifier byte range is invalid."); + ensureCapacity(valueLength); + System.arraycopy(value, offset, bytes, length, valueLength); + length += valueLength; + } + + void ensureCapacity(int additional) { + int required = Math.addExact(length, additional); + if (required <= bytes.length) { + return; + } + int grown = Math.max(required, bytes.length + (bytes.length >>> 1)); + bytes = Arrays.copyOf(bytes, grown); + } + } + + private static final class AvroSlice { + + byte[] bytes; + int position; + int end; + + AvroSlice reset(Record record, int field) { + bytes = record.bytes(field); + position = record.offset(field); + end = position + record.length(field); + return this; + } + + long readLong() { + long raw = 0; + int shift = 0; + while (position < end && shift < 64) { + int value = bytes[position++] & 0xFF; + raw |= (long) (value & 0x7F) << shift; + if ((value & 0x80) == 0) { + return (raw >>> 1) ^ -(raw & 1L); + } + shift += 7; + } + throw new IllegalStateException("Invalid Avro variable-length integer."); + } + + int readLength() { + long valueLength = readLong(); + checkState( + valueLength >= 0 + && valueLength <= Integer.MAX_VALUE + && valueLength <= end - position, + "Invalid Avro byte sequence length %s.", + valueLength); + return (int) valueLength; + } + + void skipBytes() { + int valueLength = readLength(); + position += valueLength; + } + + void checkConsumed() { + checkState(position == end, "Manifest field contains trailing Avro bytes."); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java new file mode 100644 index 000000000000..b85dafe41758 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -0,0 +1,801 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroBlock; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader.Record; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestBlock; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestEntry; +import org.apache.paimon.manifest.ManifestFile.PrimitiveManifestRollingWriter; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.paimon.utils.Preconditions.checkState; + +final class ManifestEntryRunMergePlan { + + final List sources; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + + ManifestEntryRunMergePlan( + List sources, ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + this.sources = sources; + this.partitions = partitions; + } + + List mergeToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Collections.emptyList(); + } + List files = writeSelected(selectionTree, manifestFile); + newFilesForAbort.addAll(files); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + Pair, List> mergeMinorToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Pair.of(Collections.emptyList(), Collections.emptyList()); + } + Pair, List> files = + writeMinorSelected( + selectionTree, manifestFile, deletedIdentifiers, deletedRowIds); + newFilesForAbort.addAll(files.getLeft()); + newFilesForAbort.addAll(files.getRight()); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + static List writeSelected( + SelectionTree selectionTree, ManifestFile manifestFile) throws Exception { + PrimitiveManifestRollingWriter writer = manifestFile.createPrimitiveRollingWriter(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + writer.writeEncodedBlock( + cursor.encodedBlock(), + cursor.blockMetadata(), + cursor.blockRecordCount()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + cursor.materializeCurrent(); + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + selectionTree.update(winner, cursor.advance()); + } + } catch (Exception e) { + failure = e; + } finally { + if (failure != null) { + writer.abort(); + throw failure; + } + writer.close(); + } + return writer.result(); + } + + private static Pair, List> writeMinorSelected( + SelectionTree selectionTree, + ManifestFile manifestFile, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) + throws Exception { + PrimitiveManifestRollingWriter addWriter = manifestFile.createPrimitiveRollingWriter(); + PrimitiveManifestRollingWriter deleteWriter = manifestFile.createPrimitiveRollingWriter(); + CompactFileIdentifierSet matchedEntries = new CompactFileIdentifierSet(); + CompactFileIdentifierSet emittedDeletes = new CompactFileIdentifierSet(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + addWriter.writeEncodedBlock( + cursor.encodedBlock(), + cursor.blockMetadata(), + cursor.blockRecordCount()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + + cursor.materializeCurrent(); + if (cursor.key().kind == FileKind.ADD.toByteValue()) { + if (!deletedRowIds.contains(cursor.key().firstRowId)) { + writeCurrent(addWriter, cursor); + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (deletedIdentifiers.contains(identifier)) { + matchedEntries.add(identifier); + } else { + writeCurrent(addWriter, cursor); + } + } + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (!matchedEntries.contains(identifier) + && !emittedDeletes.contains(identifier)) { + emittedDeletes.add(identifier); + writeCurrent(deleteWriter, cursor); + } + } + selectionTree.update(winner, cursor.advance()); + } + addWriter.close(); + deleteWriter.close(); + } catch (Exception e) { + failure = e; + } finally { + matchedEntries.release(); + emittedDeletes.release(); + if (failure != null) { + addWriter.abort(); + deleteWriter.abort(); + throw failure; + } + } + return Pair.of(addWriter.result(), deleteWriter.result()); + } + + private static void writeCurrent(PrimitiveManifestRollingWriter writer, Cursor cursor) + throws Exception { + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + } + + static void closeCursors(List cursors) throws Exception { + Exception failure = null; + for (Cursor cursor : cursors) { + try { + cursor.close(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + + /** Describes the manifest inputs which become cursors when this plan starts executing. */ + static final class Source { + + private Source() {} + + interface Spec { + + Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception; + } + + static final class ManifestRunSpec implements Spec { + + final ManifestFileMeta meta; + final long start; + final long end; + final List blocks; + + ManifestRunSpec( + ManifestFileMeta meta, + long start, + long end, + List blocks) { + this.meta = meta; + this.start = start; + this.end = end; + this.blocks = blocks; + } + + long prefixBlockCount() { + long lastBlockOrdinal = -1; + for (ManifestEntryRunMerge.Discovery.BlockInfo block : blocks) { + if (block.start >= end) { + break; + } + if (block.end > start) { + lastBlockOrdinal = block.ordinal; + } + } + checkState(lastBlockOrdinal >= 0, "Manifest run does not contain an Avro block."); + return lastBlockOrdinal + 1; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new PrimitiveManifestRunCursor( + manifestFile, meta, start, end, blocks, filter, partitions); + } + } + + static final class FragmentedManifestSpec implements Spec { + + final ManifestFileMeta meta; + + FragmentedManifestSpec(ManifestFileMeta meta) { + this.meta = meta; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new InMemoryManifestCursor(manifestFile, meta, sortKey, filter, partitions); + } + } + } + + interface Cursor extends AutoCloseable { + + boolean advance() throws Exception; + + boolean hasCurrent(); + + @Nullable + BinaryManifestEntry current(); + + @Nullable + EncodedManifestEntry metadata(); + + ManifestEntryRunMergeEntry.Key key(); + + @Nullable + ByteBuffer encodedRecord(); + + ReusableIdentifier identifier(); + + default boolean hasCopyableBlock() { + return false; + } + + default ManifestEntryRunMergeEntry.Key blockLastKey() { + throw new UnsupportedOperationException(); + } + + default PrimitiveAvroBlock encodedBlock() { + throw new UnsupportedOperationException(); + } + + default EncodedManifestBlock blockMetadata() { + throw new UnsupportedOperationException(); + } + + default long blockRecordCount() { + throw new UnsupportedOperationException(); + } + + default boolean advanceAfterBlock() throws Exception { + throw new UnsupportedOperationException(); + } + + default void materializeCurrent() throws Exception {} + + @Override + void close() throws Exception; + } + + static final class PrimitiveManifestRunCursor implements Cursor { + + final PrimitiveAvroRecordReader reader; + final ManifestEntryRunMergeEntry.Filter filter; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + final ManifestEntryRunMergeEntry.Key key = new ManifestEntryRunMergeEntry.Key(); + final EncodedManifestEntry metadata = new EncodedManifestEntry(); + final List blocks; + final long runStart; + final long runEnd; + int blockIndex; + long nextReaderBlockOrdinal; + long decodedRemaining; + boolean rawBlock; + @Nullable Record current; + @Nullable ManifestEntryRunMerge.Discovery.BlockInfo currentBlock; + boolean closed; + + PrimitiveManifestRunCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + long start, + long end, + List blocks, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + this.reader = + manifestFile.scanPrimitive( + meta.fileName(), meta.fileSize(), ManifestEntryRunMerge.ENTRY_LAYOUT); + this.filter = filter; + this.partitions = partitions; + this.blocks = blocks; + this.runStart = start; + this.runEnd = end; + try { + while (blockIndex < blocks.size() && blocks.get(blockIndex).end <= start) { + blockIndex++; + } + checkState( + blockIndex < blocks.size(), + "Manifest run starts after the end of the file."); + } catch (Exception e) { + try { + reader.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + } + + @Override + public boolean advance() throws Exception { + current = null; + while (true) { + if (decodedRemaining == 0) { + if (!prepareNextBlock()) { + key.clear(); + close(); + return false; + } + if (rawBlock) { + return true; + } + } + checkState(reader.hasNext(), "Manifest block ends before its discovered boundary."); + Record next = reader.next(); + decodedRemaining--; + key.replace(next, partitions); + if (filter.include(next, key)) { + current = next; + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + (int) next.longValue(ManifestEntryRunMerge.BUCKET), + (int) next.longValue(ManifestEntryRunMerge.LEVEL), + next.longValue(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + next.longValue(ManifestEntryRunMerge.ROW_COUNT)); + return true; + } + } + } + + boolean prepareNextBlock() throws Exception { + rawBlock = false; + current = null; + while (blockIndex < blocks.size()) { + ManifestEntryRunMerge.Discovery.BlockInfo info = blocks.get(blockIndex); + if (info.start >= runEnd) { + return false; + } + while (nextReaderBlockOrdinal < info.ordinal) { + checkState(reader.hasNextRawBlock(), "Manifest block ordinal is missing."); + reader.nextRawBlock(); + reader.skipCurrentBlock(); + nextReaderBlockOrdinal++; + } + checkState( + reader.hasNextRawBlock(), "Manifest run ends after the end of the file."); + reader.nextRawBlock(); + nextReaderBlockOrdinal++; + currentBlock = info; + if (info.copyable(runStart, runEnd)) { + rawBlock = true; + key.copyFrom(info.firstKey); + return true; + } + + long overlapStart = Math.max(runStart, info.start); + long overlapEnd = Math.min(runEnd, info.end); + long prefix = overlapStart - info.start; + for (long i = 0; i < prefix; i++) { + checkState(reader.hasNext(), "Manifest run starts after the end of its block."); + reader.next(); + } + decodedRemaining = overlapEnd - overlapStart; + blockIndex++; + if (decodedRemaining > 0) { + return true; + } + reader.skipCurrentBlock(); + } + return false; + } + + @Override + public boolean hasCurrent() { + return current != null || rawBlock; + } + + @Override + public BinaryManifestEntry current() { + return null; + } + + @Override + public EncodedManifestEntry metadata() { + return metadata; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return key; + } + + @Override + public ByteBuffer encodedRecord() { + return current == null ? null : current.encoded(); + } + + @Override + public ReusableIdentifier identifier() { + checkState(current != null, "Manifest entry has not been materialized."); + return filter.identifier(current); + } + + @Override + public boolean hasCopyableBlock() { + return rawBlock; + } + + @Override + public ManifestEntryRunMergeEntry.Key blockLastKey() { + return currentBlock.lastKey; + } + + @Override + public PrimitiveAvroBlock encodedBlock() { + return reader.currentRawBlock(); + } + + @Override + public EncodedManifestBlock blockMetadata() { + return currentBlock.metadata; + } + + @Override + public long blockRecordCount() { + return currentBlock.end - currentBlock.start; + } + + @Override + public boolean advanceAfterBlock() throws Exception { + checkState(rawBlock, "There is no raw block to advance."); + reader.skipCurrentBlock(); + rawBlock = false; + blockIndex++; + return advance(); + } + + @Override + public void materializeCurrent() throws Exception { + if (!rawBlock) { + return; + } + rawBlock = false; + decodedRemaining = currentBlock.end - currentBlock.start; + checkState(decodedRemaining > 0, "Raw Avro block is empty."); + checkState(reader.hasNext(), "Manifest block cannot be decompressed."); + Record next = reader.next(); + decodedRemaining--; + key.replace(next, partitions); + checkState( + filter.include(next, key), + "Copyable manifest block contains a filtered entry."); + current = next; + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + (int) next.longValue(ManifestEntryRunMerge.BUCKET), + (int) next.longValue(ManifestEntryRunMerge.LEVEL), + next.longValue(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + next.longValue(ManifestEntryRunMerge.ROW_COUNT)); + blockIndex++; + } + + @Override + public void close() throws Exception { + if (closed) { + return; + } + closed = true; + current = null; + currentBlock = null; + rawBlock = false; + key.clear(); + reader.close(); + } + } + + static final class InMemoryManifestCursor implements Cursor { + + final List entries; + final BinaryManifestEntry current = BinaryManifestEntry.fullProjection().createEntry(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + int position = -1; + + InMemoryManifestCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + this.entries = new ArrayList<>((int) entryCount); + InternalRowSerializer serializer = + new InternalRowSerializer(ManifestEntry.MANIFEST_ROW_TYPE); + BinaryManifestEntry view = BinaryManifestEntry.fullProjection().createEntry(); + try (CloseableIterator iterator = + manifestFile.scan( + meta.fileName(), + meta.fileSize(), + BinaryManifestEntry.fullProjection())) { + while (iterator.hasNext()) { + BinaryManifestEntry entry = iterator.next(); + if (!filter.include(entry)) { + continue; + } + BinaryRow row = serializer.toBinaryRow(entry.fullRow()).copy(); + entries.add( + new StoredEntry( + row, + ManifestEntryRunMergeEntry.Key.viewOf( + view.replace(row), partitions))); + } + } + entries.sort( + (left, right) -> ManifestEntryRunMerge.compareMergeKeys(left.key, right.key)); + view.clear(); + } + + @Override + public boolean advance() { + position++; + if (position >= entries.size()) { + current.clear(); + return false; + } + StoredEntry stored = entries.get(position); + current.replace(stored.row); + return true; + } + + @Override + public boolean hasCurrent() { + return position >= 0 && position < entries.size(); + } + + @Override + public BinaryManifestEntry current() { + return current; + } + + @Override + public EncodedManifestEntry metadata() { + return null; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return entries.get(position).key; + } + + @Override + public ByteBuffer encodedRecord() { + return null; + } + + @Override + public ReusableIdentifier identifier() { + return identifier.replaceWithPartition(current); + } + + @Override + public void close() { + current.clear(); + identifier.release(); + entries.clear(); + position = -1; + } + } + + private static final class StoredEntry { + + final BinaryRow row; + final ManifestEntryRunMergeEntry.Key key; + + StoredEntry(BinaryRow row, ManifestEntryRunMergeEntry.Key key) { + this.row = row; + this.key = key; + } + } + + /** Fixed-size tournament tree which selects a cursor with one comparison per tree level. */ + private static final class SelectionTree { + + final List cursors; + final int leafBase; + final int[] winners; + + SelectionTree(List cursors) { + this.cursors = cursors; + int base = 1; + while (base < cursors.size()) { + base <<= 1; + } + this.leafBase = base; + this.winners = new int[leafBase << 1]; + Arrays.fill(winners, -1); + for (int cursor = 0; cursor < cursors.size(); cursor++) { + if (cursors.get(cursor).hasCurrent()) { + winners[leafBase + cursor] = cursor; + } + } + for (int node = leafBase - 1; node > 0; node--) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int winner() { + return winners[1]; + } + + Cursor cursor(int index) { + return cursors.get(index); + } + + void update(int cursor, boolean hasCurrent) { + int node = leafBase + cursor; + winners[node] = hasCurrent ? cursor : -1; + while ((node >>= 1) > 0) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int select(int left, int right) { + if (left < 0) { + return right; + } + if (right < 0) { + return left; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + cursors.get(left).key(), cursors.get(right).key()); + return comparison < 0 || (comparison == 0 && left < right) ? left : right; + } + + boolean blockPrecedesOthers(int cursor, ManifestEntryRunMergeEntry.Key blockLastKey) { + for (int other = 0; other < cursors.size(); other++) { + if (other == cursor || !cursors.get(other).hasCurrent()) { + continue; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + blockLastKey, cursors.get(other).key()); + if (comparison > 0 || (comparison == 0 && cursor > other)) { + return false; + } + } + return true; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index bee9ddc18792..d08b876ff025 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -70,9 +70,11 @@ public class ManifestFileSorter { /** Context object that carries shared state across compaction methods. */ static class CompactionContext { final boolean fullCompaction; + final boolean runMergeOptimizeEnabled; final ManifestSortKey sortKey; final ManifestEntryExternalSort.ExternalSortConfig externalSortConfig; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -81,38 +83,43 @@ static class CompactionContext { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; final List levelRuns; final List pickedRuns; CompactionContext( boolean fullCompaction, + boolean runMergeOptimizeEnabled, ManifestSortKey sortKey, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort, + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; + this.runMergeOptimizeEnabled = runMergeOptimizeEnabled; this.sortKey = sortKey; this.externalSortConfig = externalSortConfig; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } /** Check whether the given manifest file is marked for unsorted compaction. */ - boolean isMarkedForUnsortedCompaction(ManifestFileMeta file) { - return compactWithoutSort.containsKey(file); + boolean isMarkedForDefaultCompaction(ManifestFileMeta file) { + return defaultCompactFiles.containsKey(file); } } /** Result of classifying manifest files. */ - private static class ClassifyResult { + private static class ManifestClassification { final List lsmFiles; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -121,29 +128,153 @@ private static class ClassifyResult { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; - ClassifyResult( + ManifestClassification( List lsmFiles, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort) { + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; } } /** Binary identifiers and partition values collected from DELETE entries. */ private static class DeletedEntryInfo { final CompactFileIdentifierSet identifiers; + final DeletedRowIdSet rowIds; final Set partitions; - private DeletedEntryInfo(CompactFileIdentifierSet identifiers, Set partitions) { + private DeletedEntryInfo( + CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, + Set partitions) { this.identifiers = identifiers; + this.rowIds = rowIds; this.partitions = partitions; } } + /** Primitive set used by RowID full compaction to avoid rebuilding file identifiers. */ + static final class DeletedRowIdSet { + + private static final long EMPTY = Long.MIN_VALUE; + private long[] table = emptyTable(16); + private int size; + private boolean containsMinValue; + private @Nullable long[] sortedRowIds; + + void add(long value) { + if (value == EMPTY) { + if (!containsMinValue) { + containsMinValue = true; + size++; + sortedRowIds = null; + } + return; + } + if ((size + 1) * 2 > table.length) { + grow(); + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return; + } + slot = (slot + 1) & (table.length - 1); + } + table[slot] = value; + size++; + sortedRowIds = null; + } + + boolean contains(long value) { + if (value == EMPTY) { + return containsMinValue; + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return true; + } + slot = (slot + 1) & (table.length - 1); + } + return false; + } + + boolean intersects(long minInclusive, long maxInclusive) { + if (minInclusive > maxInclusive) { + return true; + } + long[] values = sortedRowIds(); + int position = java.util.Arrays.binarySearch(values, minInclusive); + if (position < 0) { + position = -position - 1; + } + return position < values.length && values[position] <= maxInclusive; + } + + private long[] sortedRowIds() { + if (sortedRowIds != null) { + return sortedRowIds; + } + long[] values = new long[size]; + int position = 0; + if (containsMinValue) { + values[position++] = EMPTY; + } + for (long value : table) { + if (value != EMPTY) { + values[position++] = value; + } + } + if (position != size) { + throw new IllegalStateException("Failed to snapshot deleted RowID set."); + } + java.util.Arrays.sort(values); + sortedRowIds = values; + return values; + } + + void releaseRangeIndex() { + sortedRowIds = null; + } + + private void grow() { + long[] previous = table; + if (previous.length >= (1 << 30)) { + throw new IllegalStateException("Too many deleted RowIDs in one manifest group."); + } + table = emptyTable(previous.length << 1); + int previousSize = size; + size = containsMinValue ? 1 : 0; + for (long value : previous) { + if (value != EMPTY) { + add(value); + } + } + if (size != previousSize) { + throw new IllegalStateException("Failed to grow deleted RowID set."); + } + } + + private static int slot(long value, int length) { + value ^= value >>> 33; + value *= 0xff51afd7ed558ccdL; + value ^= value >>> 33; + return ((int) value) & (length - 1); + } + + private static long[] emptyTable(int length) { + long[] table = new long[length]; + java.util.Arrays.fill(table, EMPTY); + return table; + } + } + /** * Try to sort-rewrite the merged manifest list by a configured partition field. If the sort * field cannot be resolved, the input is returned as-is. @@ -160,6 +291,7 @@ static List trySortCompaction( @Nullable IOManager ioManager) throws Exception { String sortPartitionField = options.manifestSortPartitionField(); + boolean runMergeOptimizeEnabled = options.manifestSortRunMergeOptimizeEnabled(); long suggestedMetaSize = options.manifestTargetSize().getBytes(); int suggestedMinMetaCount = options.manifestMergeMinCount(); long fullCompactionThreshold = options.manifestFullCompactionThresholdSize().getBytes(); @@ -178,6 +310,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, fullCompactionThreshold, @@ -196,6 +329,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, maxRewriteSize, @@ -218,6 +352,7 @@ private static Optional> tryFullCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long fullCompactionThreshold, @@ -246,6 +381,7 @@ private static Optional> tryFullCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -254,19 +390,19 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort full compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort full compact skipped: no runs picked and no defaultCompactFiles."); return Optional.empty(); } LOG.info( "Manifest sort full compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -280,7 +416,7 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Split into sections and merge small adjacent sections List

sections = splitIntoSections(pickedFiles, ctx); @@ -324,6 +460,7 @@ private static List tryMinorCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long maxRewriteSize, @@ -341,6 +478,7 @@ private static List tryMinorCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -349,19 +487,19 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort minor compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort minor compact skipped: no runs picked and no defaultCompactFiles."); return input; } LOG.info( "Manifest sort minor compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -388,7 +526,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -450,26 +588,29 @@ private static CompactionContext prepareCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int maxSizeAmplificationPercent, int sortedRunSizeRatio, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, @Nullable Integer manifestReadParallelism) { + boolean rowIdSort = dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input); + boolean useRunMergeOptimize = rowIdSort && runMergeOptimizeEnabled; // Step 1: Resolve sort key. Data evolution tables prefer RowID ranges when available. - ManifestSortKey sortKey = - createSortKey(dataEvolutionEnabled, input, sortPartitionField, partitionType); + ManifestSortKey sortKey = createSortKey(rowIdSort, sortPartitionField, partitionType); // Step 2: Classify manifests into LSM files and collect delete entries. - ClassifyResult classifyResult = + ManifestClassification classification = classifyManifests( input, fullCompaction, manifestFile, partitionType, suggestedMetaSize, + useRunMergeOptimize, manifestReadParallelism); - List lsmFiles = classifyResult.lsmFiles; + List lsmFiles = classification.lsmFiles; // Step 3: Build level-sorted runs from LSM files based on partition order. List levelRuns = @@ -482,10 +623,12 @@ private static CompactionContext prepareCompaction( return new CompactionContext( fullCompaction, + useRunMergeOptimize, sortKey, externalSortConfig, - classifyResult.deleteEntries, - classifyResult.compactWithoutSort, + classification.deleteEntries, + classification.deletedRowIds, + classification.defaultCompactFiles, levelRuns, pickedRuns); } @@ -494,30 +637,34 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

Full compaction: small files and files overlapping delete partitions go into - * compactWithoutSort; the rest are returned as lsmFiles. + * defaultCompactFiles; the rest are returned as lsmFiles. * - *

Non-full compaction: small files go to compactWithoutSort for minor-style merge; the rest + *

Non-full compaction: small files go to defaultCompactFiles for minor-style merge; the rest * are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and compactWithoutSort + * @return classification containing lsmFiles, deleteEntries, and defaultCompactFiles */ - private static ClassifyResult classifyManifests( + private static ManifestClassification classifyManifests( List input, boolean fullCompaction, ManifestFile manifestFile, RowType partitionType, long suggestedMetaSize, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map compactWithoutSort = new LinkedHashMap<>(); + Map defaultCompactFiles = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); CompactFileIdentifierSet classifiedDeleteEntries = new CompactFileIdentifierSet(); + DeletedRowIdSet deletedRowIds = new DeletedRowIdSet(); Set deletePartitions = Collections.emptySet(); PartitionPredicate predicate = null; if (fullCompaction) { DeletedEntryInfo deletedEntries = - readDeletedEntries(manifestFile, input, manifestReadParallelism); + readDeletedEntries( + manifestFile, input, runMergeOptimizeEnabled, manifestReadParallelism); classifiedDeleteEntries = deletedEntries.identifiers; + deletedRowIds = deletedEntries.rowIds; deletePartitions = deletedEntries.partitions; // Build partition predicate from delete entries for overlap detection. @@ -546,18 +693,21 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - compactWithoutSort.put(file, inDeleteRange); + defaultCompactFiles.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, compactWithoutSort); + return new ManifestClassification( + lsmFiles, classifiedDeleteEntries, deletedRowIds, defaultCompactFiles); } private static DeletedEntryInfo readDeletedEntries( ManifestFile manifestFile, List manifestFiles, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); + DeletedRowIdSet rowIds = new DeletedRowIdSet(); Set partitions = new HashSet<>(); List filesWithDeletes = new ArrayList<>(); for (ManifestFileMeta meta : manifestFiles) { @@ -569,12 +719,26 @@ private static DeletedEntryInfo readDeletedEntries( if (filesWithDeletes.size() <= 1 || (manifestReadParallelism != null && manifestReadParallelism <= 1)) { for (ManifestFileMeta meta : filesWithDeletes) { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, false); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + false); } } else { Function> reader = meta -> { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, true); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + true); return Collections.singletonList(Boolean.TRUE); }; for (Boolean ignored : @@ -582,14 +746,16 @@ private static DeletedEntryInfo readDeletedEntries( // Iteration waits for each bounded batch of parallel reads. } } - return new DeletedEntryInfo(identifiers, partitions); + return new DeletedEntryInfo(identifiers, rowIds, partitions); } private static void collectDeletedEntries( ManifestFileMeta meta, ManifestFile manifestFile, CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, Set partitions, + boolean runMergeOptimizeEnabled, boolean synchronize) { try (CloseableIterator entries = manifestFile.scan( @@ -605,10 +771,16 @@ private static void collectDeletedEntries( if (synchronize) { synchronized (identifiers) { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } else { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } @@ -693,7 +865,7 @@ static List buildLevelSortedRuns( /** * Split picked files into sections. Files with overlapping sort-key intervals go into the same - * section. Each section is built with pre-computed totalSize and hasUnsortedCompactMeta. + * section. Each section is built with pre-computed totalSize and hasDefaultCompactFile. */ static List

splitIntoSections( List pickedFiles, CompactionContext ctx) { @@ -714,7 +886,7 @@ static List
splitIntoSections( currentSectionFiles.add(first); currentSectionTotalSize += first.fileSize(); - boolean currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(first); + boolean currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(first); ManifestFileMeta sectionMaxFile = first; for (int i = 1; i < pickedFiles.size(); i++) { @@ -726,20 +898,20 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); // start a new section currentSectionFiles = new ArrayList<>(); currentSectionTotalSize = 0; currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(file); + currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(file); sectionMaxFile = file; } else { currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - if (!currentSectionHasUnsortedCompactMeta - && ctx.isMarkedForUnsortedCompaction(file)) { - currentSectionHasUnsortedCompactMeta = true; + if (!currentSectionHasDefaultCompactFile + && ctx.isMarkedForDefaultCompaction(file)) { + currentSectionHasDefaultCompactFile = true; } if (sortKey.compareMax(file, sectionMaxFile) > 0) { sectionMaxFile = file; @@ -750,7 +922,7 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); return sections; } @@ -793,7 +965,7 @@ private static List
mergeSmallAdjacentSections( *
  • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
  • Subsequent overflows: If the section has files in compactWithoutSort (needs unsorted + *
  • Subsequent overflows: If the section has files in defaultCompactFiles (needs default * compaction), unsortedCompactSection is called to process it in smaller chunks. * Otherwise, the section is skipped. * @@ -904,9 +1076,9 @@ private static Section splitSectionAndRewriteHead( List tailFiles = new ArrayList<>(); long headSize = 0; long tailSize = 0; - // Whether tail section has files in compactWithoutSort, if true, the section need to + // Whether the tail section has files in defaultCompactFiles. If so, the section needs to // be rewritten. - boolean tailHasUnsortedCompactMeta = false; + boolean tailHasDefaultCompactFile = false; for (ManifestFileMeta file : section.files) { // Rewrite budget is enforced at manifest-file granularity. Include the first file that @@ -918,8 +1090,8 @@ private static Section splitSectionAndRewriteHead( } else { tailFiles.add(file); tailSize += file.fileSize(); - if (ctx.isMarkedForUnsortedCompaction(file)) { - tailHasUnsortedCompactMeta = true; + if (ctx.isMarkedForDefaultCompaction(file)) { + tailHasDefaultCompactFile = true; } } } @@ -929,7 +1101,7 @@ private static Section splitSectionAndRewriteHead( if (tailFiles.isEmpty()) { return null; } - return new Section(tailFiles, tailSize, tailHasUnsortedCompactMeta); + return new Section(tailFiles, tailSize, tailHasDefaultCompactFile); } /** @@ -947,7 +1119,7 @@ private static void rewriteSectionBeyondBudget( int suggestedMinMetaCount, @Nullable Integer manifestReadParallelism) throws Exception { - if (section.hasUnsortedCompactMeta) { + if (section.hasDefaultCompactFile) { unsortedCompactSection( section.files, output, @@ -967,8 +1139,8 @@ private static void rewriteSectionBeyondBudget( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, unsortedCompactSection is triggered when compactWithoutSort is non-empty, - * regardless of the manifest count. This is because files in compactWithoutSort either: + * this sort path, unsortedCompactSection is triggered when defaultCompactFiles is non-empty, + * regardless of the manifest count. This is because files in defaultCompactFiles either: * *

      *
    • Are small files needing consolidation @@ -1038,7 +1210,7 @@ private static void rewriteSection( @Nullable Integer manifestReadParallelism) throws Exception { // Skip rewrite for single file not in delete-range. - if (section.size() == 1 && !ctx.compactWithoutSort.getOrDefault(section.get(0), false)) { + if (section.size() == 1 && !ctx.defaultCompactFiles.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; } @@ -1064,24 +1236,38 @@ private static void rewriteFull( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - List sorted = - ManifestEntryExternalSort.sortAndWriteFullEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - ctx.deleteEntries, - manifestReadParallelism); + List sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteFullEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + ctx.deletedRowIds, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteFullEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + manifestReadParallelism); + } if (!sorted.isEmpty()) { output.addSortedFiles(sorted); } } /** - * Minor compaction path: collect DELETE entries in memory while external-sorting all entries, - * then write surviving ADD entries from the sorted stream and remaining DELETE entries from - * memory. + * Minor compaction path: collect DELETE identities, merge the existing sorted runs, and write + * surviving ADD entries and unmatched DELETE entries separately. Falls back to external sort + * when the input is not suitable for run merge. */ private static void rewriteMinor( List section, @@ -1091,14 +1277,26 @@ private static void rewriteMinor( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - Pair, List> sorted = - ManifestEntryExternalSort.sortAndWriteMinorEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - manifestReadParallelism); + Pair, List> sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteMinorEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteMinorEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } if (!sorted.getLeft().isEmpty()) { output.addSortedFiles(sorted.getLeft()); @@ -1119,11 +1317,8 @@ private static boolean containsNoDeleteEntries(List section) { } private static ManifestSortKey createSortKey( - boolean dataEvolutionEnabled, - List input, - String sortPartitionField, - RowType partitionType) { - if (dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input)) { + boolean rowIdSort, String sortPartitionField, RowType partitionType) { + if (rowIdSort) { // RowID sorting uses the configured partition field as the primary key when specified, // otherwise it uses the full partition row to preserve partition locality. It then // orders files by RowID. @@ -1202,6 +1397,11 @@ void replaceExternalSortRow( InternalRow binaryManifestRow(BinaryRow row); } + interface RowIdEntrySortKey extends ManifestSortKey { + + int comparePartitions(BinaryRow left, BinaryRow right); + } + private static class PartitionSortKey implements ManifestSortKey { private final RecordComparator fieldComparator; @@ -1273,7 +1473,7 @@ public InternalRow binaryManifestRow(BinaryRow row) { } } - private static class RowIdSortKey implements ManifestSortKey { + private static class RowIdSortKey implements RowIdEntrySortKey { @Nullable private final RecordComparator partitionComparator; private final InternalRow.FieldGetter[] partitionFieldGetters; @@ -1288,21 +1488,8 @@ private RowIdSortKey( this.partitionComparator = partitionComparator; this.partitionFieldGetters = createPartitionFieldGetters(partitionType, partitionSortFields); - - List fieldTypes = new ArrayList<>(); - for (int partitionSortField : partitionSortFields) { - fieldTypes.add(partitionType.getTypeAt(partitionSortField)); - } - // ADD must precede DELETE for the same partition. Minor compaction streams the sorted - // rows once and uses this ordering to eliminate a matching pair without retaining all - // ADD identifiers. - fieldTypes.add(DataTypes.TINYINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.STRING()); - fieldTypes.add(ManifestEntry.MANIFEST_ROW_TYPE); - this.externalSortRowType = DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + this.externalSortRowType = + createRowIdExternalSortRowType(partitionType, partitionSortFields); this.sortFieldNum = externalSortRowType.getFieldCount() - 1; this.externalSortKeyFields = createSequentialFields(sortFieldNum); } @@ -1336,7 +1523,7 @@ public boolean isAfterMax(ManifestFileMeta file, ManifestFileMeta maxFile) { return c > 0; } } - return Long.compare(nonNullMinRowId(file), nonNullMaxRowId(maxFile)) > 0; + return nonNullMinRowId(file) > nonNullMaxRowId(maxFile); } @Override @@ -1373,6 +1560,11 @@ public InternalRow binaryManifestRow(BinaryRow row) { return row.getRow(sortFieldNum, ManifestEntry.MANIFEST_ROW_TYPE.getFieldCount()); } + @Override + public int comparePartitions(BinaryRow left, BinaryRow right) { + return partitionComparator == null ? 0 : partitionComparator.compare(left, right); + } + private int comparePartitionMin(ManifestFileMeta a, ManifestFileMeta b) { if (partitionComparator == null) { return 0; @@ -1412,6 +1604,26 @@ private static long rowIdRangeEnd(ManifestEntry entry) { } } + private static RowType createRowIdExternalSortRowType( + RowType partitionType, int[] partitionSortFields) { + List fieldTypes = new ArrayList<>(partitionSortFields.length + 6); + for (int partitionSortField : partitionSortFields) { + fieldTypes.add(partitionType.getTypeAt(partitionSortField)); + } + // ADD must precede DELETE for the same partition. Minor compaction streams the sorted rows + // once and uses this ordering to eliminate a matching pair without retaining all ADD + // identifiers. + Collections.addAll( + fieldTypes, + DataTypes.TINYINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.STRING(), + ManifestEntry.MANIFEST_ROW_TYPE); + return DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + } + private static int[] createSequentialFields(int fieldCount) { int[] fields = new int[fieldCount]; for (int i = 0; i < fieldCount; i++) { @@ -1530,12 +1742,12 @@ public void addDeleteFiles(List files) { static class Section { final List files; final long totalSize; - final boolean hasUnsortedCompactMeta; + final boolean hasDefaultCompactFile; - Section(List files, long totalSize, boolean hasUnsortedCompactMeta) { + Section(List files, long totalSize, boolean hasDefaultCompactFile) { this.files = files; this.totalSize = totalSize; - this.hasUnsortedCompactMeta = hasUnsortedCompactMeta; + this.hasDefaultCompactFile = hasDefaultCompactFile; } /** Create a merged section from two sections. */ @@ -1545,7 +1757,7 @@ static Section merge(Section a, Section b) { return new Section( merged, a.totalSize + b.totalSize, - a.hasUnsortedCompactMeta || b.hasUnsortedCompactMeta); + a.hasDefaultCompactFile || b.hasDefaultCompactFile); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 7dff697e8fc6..9fa1edb7615e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -59,10 +59,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -70,6 +72,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -883,6 +886,28 @@ private void beforeFirstRead() throws IOException { } } + private static class CountingReadFileIO extends LocalFileIO { + + private final Map readCounts = new ConcurrentHashMap<>(); + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + readCounts + .computeIfAbsent(path.getName(), ignored -> new AtomicInteger()) + .incrementAndGet(); + return super.newInputStream(path); + } + + private int readCount(String fileName) { + AtomicInteger count = readCounts.get(fileName); + return count == null ? 0 : count.get(); + } + + private void resetReadCounts() { + readCounts.clear(); + } + } + // ==================== Manifest Sort Tests ==================== /** @@ -1263,6 +1288,386 @@ public void testDataEvolutionManifestSortByPartitionAndRowId() { } } + @Test + public void testDisablingRunMergeOptimizePreservesDataEvolutionRowIdSort() { + assertThat( + CoreOptions.fromMap(Collections.emptyMap()) + .manifestSortRunMergeOptimizeEnabled()) + .isTrue(); + + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "row-30", 0, 30, 5), + makeRowIdEntry(true, "row-10", 0, 10, 5)), + makeManifest( + makeRowIdEntry(true, "row-20", 0, 20, 5), + makeRowIdEntry(true, "row-0", 0, 0, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("manifest-sort.run-merge-optimize.enabled", "false"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + CoreOptions coreOptions = CoreOptions.fromMap(testOptions.toMap()); + + assertThat(coreOptions.manifestSortRunMergeOptimizeEnabled()).isFalse(); + + List merged = + ManifestFileMerger.merge(input, manifestFile, getPartitionType(), coreOptions); + + assertEquivalentEntries(input, merged); + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("row-0", "row-10", "row-20", "row-30"); + } + + @Test + public void testDataEvolutionManifestRunMergeSecondaryKeys() { + List firstManifest = new ArrayList<>(); + firstManifest.add(makeRowIdEntry(true, "range-short", 0, 100, 5, 1)); + firstManifest.add(makeRowIdEntry(true, "sequence-newer", 0, 100, 10, 9)); + for (int i = 19; i >= 10; i--) { + firstManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List secondManifest = new ArrayList<>(); + for (int i = 9; i >= 0; i--) { + secondManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List input = new ArrayList<>(); + input.add(makeManifest(firstManifest.toArray(new ManifestEntry[0]))); + input.add(makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + List expected = new ArrayList<>(); + expected.add("range-short"); + expected.add("sequence-newer"); + for (int i = 0; i < 20; i++) { + expected.add(String.format("tie-%02d", i)); + } + assertThat(readEntries(merged).stream().map(e -> e.file().fileName())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesExactDeleteIdentifier() { + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "deleted", 0, 100, 5), + makeRowIdEntry(true, "same-row-id-survivor", 0, 100, 5)), + makeManifest(makeRowIdEntry(false, "deleted", 0, 100, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("same-row-id-survivor"); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesRawDeleteIdentityFields() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry survivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + List input = + Arrays.asList(makeManifest(deleted, survivor), makeManifest(delete)); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged)).singleElement().isEqualTo(survivor); + } + + @Test + public void testDataEvolutionManifestRunMergeManyPartitions() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (int partition = 39; partition >= 0; partition--) { + ManifestEntry entry = + makeRowIdEntry( + true, + String.format("partition-%02d", partition), + partition, + partition * 10L, + 5); + (partition >= 20 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(e -> e.partition().getInt(0))) + .containsExactlyElementsOf( + IntStream.range(0, 40).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFragmentedSmallManifests() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 199; firstRowId >= 0; firstRowId--) { + ManifestEntry entry = + makeRowIdEntry(true, String.format("row-%03d", firstRowId), 0, firstRowId, 1); + (firstRowId >= 100 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 200).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFallsBackForLargeFragmentedManifest() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 12_500; firstRowId--) { + firstManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + for (long firstRowId = 12_499; firstRowId >= 0; firstRowId--) { + secondManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionMinorRunMergeFallsBackForLargeFragmentedManifest() { + List fragmentedEntries = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 0; firstRowId--) { + fragmentedEntries.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])), + makeManifest( + makeRowIdEntry(false, "row-12500", 0, 12_500, 1), + makeRowIdEntry(true, "row-30000", 0, 30_000, 1))); + + List expected = + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList()); + expected.remove(Long.valueOf(12_500L)); + expected.add(30_000L); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat(runMergeResult.stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeLimitsReadAmplification() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + + int runCount = 20; + int entriesPerRun = 600; + List fragmentedEntries = new ArrayList<>(); + for (int run = runCount - 1; run >= 0; run--) { + long runStart = (long) run * entriesPerRun; + for (int entry = 0; entry < entriesPerRun; entry++) { + long firstRowId = runStart + entry; + fragmentedEntries.add( + makeRowIdEntry( + true, + String.format("fragmented-%05d", firstRowId), + 0, + firstRowId, + 1)); + } + } + + ManifestFileMeta fragmented = makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])); + ManifestFileMeta overlap = + makeManifest(makeRowIdEntry(true, "overlap", 0, entriesPerRun / 2L, 1)); + fileIO.resetReadCounts(); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + Arrays.asList(fragmented, overlap), + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(fileIO.readCount(fragmented.fileName())).isEqualTo(2); + List expectedRowIds = + LongStream.range(0, (long) runCount * entriesPerRun) + .boxed() + .collect(Collectors.toList()); + expectedRowIds.add(entriesPerRun / 2L); + Collections.sort(expectedRowIds); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expectedRowIds); + } + + @Test + public void testDataEvolutionManifestRunMergePreservesBlockStats() { + List spanningManifest = new ArrayList<>(); + List middleManifest = new ArrayList<>(); + for (long firstRowId = 0; firstRowId < 5_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + for (long firstRowId = 10_000; firstRowId < 15_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 7, firstRowId, 1)); + } + for (long firstRowId = 5_000; firstRowId < 10_000; firstRowId++) { + middleManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(spanningManifest.toArray(new ManifestEntry[0])), + makeManifest(middleManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + ManifestFileMeta output = merged.get(0); + assertThat(output.numAddedFiles()).isEqualTo(15_000); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.minRowId()).isZero(); + assertThat(output.maxRowId()).isEqualTo(14_999); + assertThat(output.partitionStats().minValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().maxValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().nullCounts().getLong(0)).isEqualTo(10_000); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 15_000).boxed().collect(Collectors.toList())); + } + @Test public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId() { RowType multiPartitionType = RowType.of(new IntType(), new IntType(), new IntType()); @@ -1394,6 +1799,101 @@ public void testDataEvolutionMinorManifestSortPreservesUnmatchedDeleteEntries() .containsExactly("ADD-new-row20", "ADD-survivor-row30", "DELETE-old-row10"); } + @Test + public void testDataEvolutionMinorRunMergeMatchesExternalSort() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry sameRowIdSurvivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry unmatchedDelete = makeRowIdEntry(false, "old-row-200", 0, 200, 5); + List input = + Arrays.asList( + makeManifest( + deleted, + sameRowIdSurvivor, + makeRowIdEntry(true, "survivor-row-300", 0, 300, 5)), + makeManifest(delete, unmatchedDelete, unmatchedDelete)); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat( + runMergeResult.stream() + .map(entry -> entry.kind() + "-" + entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly( + "ADD-same-file-name", "ADD-survivor-row-300", "DELETE-old-row-200"); + assertThat(runMergeResult.get(0)).isEqualTo(sameRowIdSurvivor); + } + + @Test + public void testDataEvolutionMinorRunMergeCollectsDeletesDuringDiscovery() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + ManifestFileMeta base = + makeManifest( + makeRowIdEntry(true, "deleted-row-10", 0, 10, 5), + makeRowIdEntry(true, "survivor-row-20", 0, 20, 5)); + ManifestFileMeta delta = + makeManifest( + makeRowIdEntry(true, "survivor-row-30", 0, 30, 5), + makeRowIdEntry(false, "deleted-row-10", 0, 10, 5)); + fileIO.resetReadCounts(); + + List merged = mergeMinorManifestEntries(Arrays.asList(base, delta), true); + + assertThat(fileIO.readCount(base.fileName())).isEqualTo(2); + assertThat(fileIO.readCount(delta.fileName())).isEqualTo(2); + assertThat( + readEntries(merged).stream() + .map(entry -> entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly("survivor-row-20", "survivor-row-30"); + } + + private List mergeMinorManifestEntries( + List input, boolean runMergeOptimizeEnabled) { + Options options = new Options(); + options.set("manifest-sort.enabled", "true"); + options.set( + "manifest-sort.run-merge-optimize.enabled", + Boolean.toString(runMergeOptimizeEnabled)); + options.set("data-evolution.enabled", "true"); + options.set("manifest.full-compaction-threshold-size", Long.MAX_VALUE + "B"); + return ManifestFileMerger.merge( + input, manifestFile, getPartitionType(), CoreOptions.fromMap(options.toMap())); + } + /** * Test manifest sort with a multi-field partition type. * @@ -1909,20 +2409,46 @@ private List readFileNames( /** Create a ManifestEntry with row ID metadata for data evolution manifest sort tests. */ private ManifestEntry makeRowIdEntry( - boolean isAdd, String fileName, int partition, long firstRowId, long rowCount) { + boolean isAdd, String fileName, Integer partition, long firstRowId, long rowCount) { return makeRowIdEntry(isAdd, fileName, partition, firstRowId, rowCount, 0); } private ManifestEntry makeRowIdEntry( boolean isAdd, String fileName, - int partition, + Integer partition, long firstRowId, long rowCount, long sequenceNumber) { + return makeRowIdEntry( + isAdd, + fileName, + partition, + firstRowId, + rowCount, + sequenceNumber, + Collections.emptyList(), + null, + null); + } + + private ManifestEntry makeRowIdEntry( + boolean isAdd, + String fileName, + Integer partition, + long firstRowId, + long rowCount, + long sequenceNumber, + List extraFiles, + byte[] embeddedIndex, + String externalPath) { BinaryRow binaryRow = new BinaryRow(1); BinaryRowWriter writer = new BinaryRowWriter(binaryRow); - writer.writeInt(0, partition); + if (partition == null) { + writer.setNullAt(0); + } else { + writer.writeInt(0, partition); + } writer.complete(); return ManifestEntry.create( @@ -1942,13 +2468,13 @@ private ManifestEntry makeRowIdEntry( sequenceNumber, 0, 0, - Collections.emptyList(), + extraFiles, Timestamp.fromEpochMillis(200000), 0L, - null, + embeddedIndex, FileSource.APPEND, null, - null, + externalPath, firstRowId, Collections.singletonList("f0"))); } diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java new file mode 100644 index 000000000000..96a19bab95f8 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.file; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** Package bridge exposing Avro's compressed blocks without reflection. */ +public final class RawBlockReader extends DataFileStream { + + public RawBlockReader(InputStream input) throws IOException { + super(input, new NoOpDatumReader()); + } + + public boolean hasNextRawBlock() { + return super.hasNextBlock(); + } + + public RawBlock nextRawBlock(RawBlock reuse) throws IOException { + DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.block); + return reuse == null + ? new RawBlock(raw, resolveCodec(), getSchema()) + : reuse.replace(raw, resolveCodec(), getSchema()); + } + + /** Appends a source block directly, allowing Avro to recompress it when codecs differ. */ + public static void appendBlock(DataFileWriter writer, RawBlock block) + throws IOException { + if (block.decompressed) { + throw new IllegalStateException("A decompressed Avro block cannot be copied raw."); + } + writer.appendAllFrom( + new SingleBlockStream(block.schema, block.codec, block.block), false); + } + + /** Reusable Avro block. */ + public static final class RawBlock { + + private DataBlock block; + private Codec codec; + private Schema schema; + private boolean decompressed; + private ByteBuffer decompressedBuffer; + + private RawBlock(DataBlock block, Codec codec, Schema schema) { + replace(block, codec, schema); + } + + private RawBlock replace(DataBlock block, Codec codec, Schema schema) { + this.block = block; + this.codec = codec; + this.schema = schema; + this.decompressed = false; + this.decompressedBuffer = null; + return this; + } + + public long recordCount() { + return block.getNumEntries(); + } + + public int compressedSize() { + if (decompressed) { + throw new IllegalStateException("The Avro block has already been decompressed."); + } + return block.getBlockSize(); + } + + public ByteBuffer decompress(ByteBuffer reuse) throws IOException { + if (!decompressed) { + if (codec instanceof ZstandardCodec) { + ByteBuffer source = block.getAsByteBuffer(); + ByteBuffer target = + reuse != null && reuse.hasArray() + ? reuse + : ByteBuffer.allocate(256 * 1024); + int size = 0; + try (InputStream compressed = + new ByteArrayInputStream( + source.array(), + source.arrayOffset() + source.position(), + source.remaining()); + InputStream input = ZstandardLoader.input(compressed, true)) { + while (true) { + if (size == target.capacity()) { + int grownCapacity = + target.capacity() == 0 + ? 256 * 1024 + : Math.multiplyExact(target.capacity(), 2); + ByteBuffer grown = ByteBuffer.allocate(grownCapacity); + System.arraycopy( + target.array(), + target.arrayOffset(), + grown.array(), + grown.arrayOffset(), + size); + target = grown; + } + int read = + input.read( + target.array(), + target.arrayOffset() + size, + target.capacity() - size); + if (read < 0) { + break; + } + size += read; + } + } + target.position(0); + target.limit(size); + decompressedBuffer = target.duplicate(); + } else { + block.decompressUsing(codec); + decompressedBuffer = block.getAsByteBuffer().duplicate(); + } + decompressed = true; + } + return decompressedBuffer.duplicate(); + } + } + + private static final class SingleBlockStream extends DataFileStream { + + private final Schema schema; + private final Codec codec; + private DataBlock block; + + private SingleBlockStream(Schema schema, Codec codec, DataBlock block) throws IOException { + super(new NoOpDatumReader()); + this.schema = schema; + this.codec = codec; + this.block = block; + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + Codec resolveCodec() { + return codec; + } + + @Override + boolean hasNextBlock() { + return block != null; + } + + @Override + DataBlock nextRawBlock(DataBlock reuse) { + DataBlock result = block; + block = null; + return result; + } + + @Override + public void close() {} + } + + private static final class NoOpDatumReader implements DatumReader { + + @Override + public void setSchema(Schema schema) {} + + @Override + public D read(D reuse, Decoder decoder) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index ef98b15fa035..1a89e24bd5ff 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -22,8 +22,10 @@ import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; -import org.apache.avro.file.DataFileStream; -import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.RawBlockReader; + +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; @@ -31,27 +33,30 @@ import java.nio.ByteBuffer; /** - * Reader which exposes decompressed blocks from an Avro object container file. + * Reader which exposes compressed and decompressed blocks from an Avro object container file. * *

      This reader owns the input stream and closes it when construction fails or {@link #close()} is * called. */ public final class AvroBlockReader implements Closeable { - private final DataFileStream reader; + private final RawBlockReader reader; + private @Nullable RawBlock borrowedRawBlock; + private @Nullable ByteBuffer decompressionBuffer; private long currentBlockRecordCount = -1; public AvroBlockReader(InputStream input) throws IOException { try { - this.reader = new DataFileStream<>(input, new GenericDatumReader<>()); + this.reader = new RawBlockReader(input); } catch (IOException | RuntimeException | Error e) { IOUtils.closeQuietly(input); throw e; } } - Schema schema() { + /** Returns the writer schema stored in the Avro file header. */ + public Schema schema() { return reader.getSchema(); } @@ -62,7 +67,7 @@ public AvroRecordDecoder createRecordDecoder() { /** Returns whether another block is available. */ public boolean hasNextBlock() throws IOException { - return replaceAvroRuntimeException(reader::hasNext); + return replaceAvroRuntimeException(reader::hasNextRawBlock); } /** @@ -85,8 +90,9 @@ public byte[] nextBlock() throws IOException { * {@link #hasNextBlock()}, {@link #nextBlock()}, or this method, or when this reader is closed. */ public BorrowedBlock nextBorrowedBlock() throws IOException { - ByteBuffer block = replaceAvroRuntimeException(reader::nextBlock); - currentBlockRecordCount = reader.getBlockCount(); + borrowedRawBlock = nextRawBlock(borrowedRawBlock); + ByteBuffer block = borrowedRawBlock.decompress(decompressionBuffer); + decompressionBuffer = block; return new BorrowedBlock( block.array(), block.arrayOffset() + block.position(), @@ -94,6 +100,22 @@ public BorrowedBlock nextBorrowedBlock() throws IOException { currentBlockRecordCount); } + /** + * Returns the next compressed block, optionally reusing the supplied holder and its storage. + * + *

      A block returned with a non-null reuse argument remains valid only until that holder is + * reused again. The block can be skipped without decompression, decompressed lazily, or copied + * directly to a compatible Avro writer. + */ + public RawBlock nextRawBlock(@Nullable RawBlock reuse) throws IOException { + RawBlockReader.RawBlock block = + replaceAvroRuntimeException( + () -> reader.nextRawBlock(reuse == null ? null : reuse.block)); + RawBlock result = reuse == null ? new RawBlock(block) : reuse.replace(block); + currentBlockRecordCount = result.recordCount(); + return result; + } + /** Returns the record count of the last block returned by a block-reading method. */ public long currentBlockRecordCount() { if (currentBlockRecordCount < 0) { @@ -150,6 +172,45 @@ public long recordCount() { } } + /** Reusable compressed Avro block. */ + public static final class RawBlock { + + private RawBlockReader.RawBlock block; + + private RawBlock(RawBlockReader.RawBlock block) { + this.block = block; + } + + private RawBlock replace(RawBlockReader.RawBlock block) { + this.block = block; + return this; + } + + public long recordCount() { + return block.recordCount(); + } + + /** Returns the compressed payload size. This block must not have been decompressed. */ + public int compressedSize() { + return block.compressedSize(); + } + + /** + * Lazily decompresses this block, reusing the supplied heap buffer when possible. + * + *

      The returned view remains owned by this block and is invalidated when this holder is + * reused for another block. + */ + public ByteBuffer decompress(@Nullable ByteBuffer reuse) throws IOException { + return block.decompress(reuse); + } + + /** Copies this still-compressed block directly to a compatible Avro writer. */ + public void appendTo(DataFileWriter writer) throws IOException { + RawBlockReader.appendBlock(writer, block); + } + } + @FunctionalInterface private interface IOSupplier { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 43105fd1cd51..11ab121acf8f 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -25,6 +25,11 @@ import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroWriter; +import org.apache.paimon.fs.CloseShieldOutputStream; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; @@ -85,6 +90,28 @@ public FormatWriterFactory createWriterFactory(RowType type) { return new RowAvroWriterFactory(type); } + public PrimitiveAvroRecordReader createPrimitiveReader( + FileIO fileIO, Path path, RowType dataSchemaRowType, RowType projectedRowType) + throws IOException { + Schema expectedSchema = + AvroSchemaConverter.convertToSchema( + dataSchemaRowType, options.get(AVRO_ROW_NAME_MAPPING)); + return new PrimitiveAvroRecordReader( + fileIO.newInputStream(path), expectedSchema, projectedRowType); + } + + public PrimitiveAvroWriter createPrimitiveWriter( + PositionOutputStream out, RowType rowType, String compression) throws IOException { + Schema schema = + AvroSchemaConverter.convertToSchema(rowType, options.get(AVRO_ROW_NAME_MAPPING)); + AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType); + DataFileWriter writer = new DataFileWriter<>(datumWriter); + writer.setCodec(createCodecFactory(compression)); + writer.setFlushOnEveryBlock(false); + writer.create(schema, new CloseShieldOutputStream(out)); + return new PrimitiveAvroWriter(writer, out); + } + @Override public Optional createStatsExtractor( RowType type, SimpleColStatsCollector.Factory[] statsCollectors) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java new file mode 100644 index 000000000000..f7a28032692f --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.avro.primitive; + +import org.apache.avro.file.RawBlockReader.RawBlock; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** Reusable compressed Avro block exposed by {@link PrimitiveAvroRecordReader}. */ +public final class PrimitiveAvroBlock { + + private RawBlock block; + + PrimitiveAvroBlock(RawBlock block) { + this.block = block; + } + + PrimitiveAvroBlock replace(RawBlock block) { + this.block = block; + return this; + } + + RawBlock rawBlock() { + return block; + } + + public long recordCount() { + return block.recordCount(); + } + + public int compressedSize() { + return block.compressedSize(); + } + + ByteBuffer decompress(ByteBuffer reuse) throws IOException { + return block.decompress(reuse); + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java new file mode 100644 index 000000000000..2e413d4815ad --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.avro.primitive; + +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.IOUtils; + +import org.apache.avro.Schema; +import org.apache.avro.file.RawBlockReader; +import org.apache.avro.file.RawBlockReader.RawBlock; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * Allocation-free reader for a small set of primitive fields in encoded Avro records. + * + *

      The returned record is reused and remains valid only until the reader advances. Byte and + * string fields are exposed as slices of the decompressed Avro block. + */ +public final class PrimitiveAvroRecordReader implements AutoCloseable { + + private final RawBlockReader stream; + private final Action rootAction; + private final Record record; + private final boolean rawBlockCopySupported; + + private BinaryDecoder decoder; + private ByteBuffer encodedBlock; + private byte[] blockBytes; + private int blockOffset; + private int blockLength; + private long blockRemaining; + private RawBlock rawBlock; + private PrimitiveAvroBlock block; + private long blockOrdinal = -1; + private long blockRecordIndex; + private long blockRecordCount; + private boolean blockDecoded; + private ByteBuffer decompressionBuffer; + + public PrimitiveAvroRecordReader( + InputStream input, Schema expectedEncodedSchema, RowType projectedRowType) + throws IOException { + RawBlockReader opened = null; + try { + opened = new RawBlockReader(input); + if (!hasSameBinaryEncoding(opened.getSchema(), expectedEncodedSchema)) { + throw new UnsupportedOperationException( + "Avro schema is not binary-compatible with the requested encoded row type."); + } + this.rawBlockCopySupported = opened.getSchema().equals(expectedEncodedSchema); + + Selection root = Selection.from(projectedRowType); + this.record = new Record(root.valueCount()); + this.rootAction = action(opened.getSchema(), root); + this.stream = opened; + } catch (IOException | RuntimeException | Error failure) { + IOUtils.closeQuietly(opened == null ? input : opened); + throw failure; + } + } + + public boolean hasNext() throws IOException { + return hasNextInternal(); + } + + private boolean hasNextInternal() throws IOException { + while (blockRemaining == 0) { + if (!loadNextBlock()) { + return false; + } + } + decodeCurrentBlock(); + return true; + } + + public boolean rawBlockCopySupported() { + return rawBlockCopySupported; + } + + /** + * Returns whether another compressed block is available. The current block must be consumed. + */ + public boolean hasNextRawBlock() { + ensureBlockConsumed(); + return stream.hasNextRawBlock(); + } + + /** Loads, but does not decompress, the next Avro block. */ + public PrimitiveAvroBlock nextRawBlock() throws IOException { + ensureBlockConsumed(); + if (!loadNextBlock()) { + throw new NoSuchElementException(); + } + return block; + } + + public PrimitiveAvroBlock currentRawBlock() { + if (block == null || blockRemaining == 0) { + throw new IllegalStateException("No current Avro block."); + } + return block; + } + + /** Marks the current compressed block consumed without decompressing it. */ + public void skipCurrentBlock() { + if (block == null || blockRemaining == 0) { + throw new IllegalStateException("No current Avro block."); + } + blockRemaining = 0; + blockDecoded = false; + } + + public Record next() throws IOException { + if (!hasNextInternal()) { + throw new NoSuchElementException(); + } + decodeNextRecord(); + return record; + } + + private void decodeNextRecord() throws IOException { + record.clearNulls(); + record.blockOrdinal = blockOrdinal; + record.blockRecordIndex = blockRecordIndex++; + record.blockRecordCount = blockRecordCount; + int start = position(); + rootAction.read(decoder, record, this); + int end = position(); + encodedBlock.position(blockOffset - encodedBlock.arrayOffset() + start); + encodedBlock.limit(blockOffset - encodedBlock.arrayOffset() + end); + record.encoded = encodedBlock; + blockRemaining--; + } + + private boolean loadNextBlock() throws IOException { + if (!stream.hasNextRawBlock()) { + return false; + } + rawBlock = stream.nextRawBlock(rawBlock); + block = block == null ? new PrimitiveAvroBlock(rawBlock) : block.replace(rawBlock); + blockRemaining = block.recordCount(); + blockRecordCount = blockRemaining; + blockRecordIndex = 0; + blockOrdinal++; + blockDecoded = false; + return true; + } + + private void decodeCurrentBlock() throws IOException { + if (blockDecoded) { + return; + } + ByteBuffer nextBlock = block.decompress(decompressionBuffer); + decompressionBuffer = nextBlock; + if (nextBlock.hasArray()) { + encodedBlock = nextBlock; + blockBytes = nextBlock.array(); + blockOffset = nextBlock.arrayOffset() + nextBlock.position(); + blockLength = nextBlock.remaining(); + } else { + blockBytes = new byte[nextBlock.remaining()]; + nextBlock.get(blockBytes); + encodedBlock = ByteBuffer.wrap(blockBytes); + blockOffset = 0; + blockLength = blockBytes.length; + } + decoder = DecoderFactory.get().binaryDecoder(blockBytes, blockOffset, blockLength, decoder); + blockDecoded = true; + } + + private void ensureBlockConsumed() { + if (blockRemaining != 0) { + throw new IllegalStateException("The current Avro block has not been consumed."); + } + } + + private int position() throws IOException { + return blockLength - decoder.inputStream().available(); + } + + @Override + public void close() throws IOException { + record.clear(); + stream.close(); + } + + private static boolean hasSameBinaryEncoding(Schema left, Schema right) { + if (left.getType() != right.getType()) { + return false; + } + switch (left.getType()) { + case RECORD: + if (left.getFields().size() != right.getFields().size()) { + return false; + } + for (int i = 0; i < left.getFields().size(); i++) { + Schema.Field leftField = left.getFields().get(i); + Schema.Field rightField = right.getFields().get(i); + if (!leftField.name().equals(rightField.name()) + || !hasSameBinaryEncoding(leftField.schema(), rightField.schema())) { + return false; + } + } + return true; + case ARRAY: + return hasSameBinaryEncoding(left.getElementType(), right.getElementType()); + case MAP: + return hasSameBinaryEncoding(left.getValueType(), right.getValueType()); + case UNION: + if (left.getTypes().size() != right.getTypes().size()) { + return false; + } + for (int i = 0; i < left.getTypes().size(); i++) { + if (!hasSameBinaryEncoding(left.getTypes().get(i), right.getTypes().get(i))) { + return false; + } + } + return true; + case ENUM: + return left.getEnumSymbols().equals(right.getEnumSymbols()); + case FIXED: + return left.getFixedSize() == right.getFixedSize(); + default: + return true; + } + } + + /** Reusable primitive projection and original encoded record. */ + public static final class Record { + + private final long[] longs; + private final byte[][] bytes; + private final int[] offsets; + private final int[] lengths; + private final boolean[] nulls; + private ByteBuffer encoded; + private long blockOrdinal; + private long blockRecordIndex; + private long blockRecordCount; + + private Record(int valueCount) { + this.longs = new long[valueCount]; + this.bytes = new byte[valueCount][]; + this.offsets = new int[valueCount]; + this.lengths = new int[valueCount]; + this.nulls = new boolean[valueCount]; + } + + public long longValue(int index) { + return longs[index]; + } + + public boolean isNull(int index) { + return nulls[index]; + } + + public byte[] bytes(int index) { + return bytes[index]; + } + + public int offset(int index) { + return offsets[index]; + } + + public int length(int index) { + return lengths[index]; + } + + public ByteBuffer encoded() { + if (encoded == null) { + throw new IllegalStateException("Encoded record has been cleared."); + } + return encoded; + } + + public long blockOrdinal() { + return blockOrdinal; + } + + public long blockRecordIndex() { + return blockRecordIndex; + } + + public long blockRecordCount() { + return blockRecordCount; + } + + private void clearNulls() { + for (int i = 0; i < nulls.length; i++) { + nulls[i] = false; + } + } + + private void clear() { + encoded = null; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = null; + } + } + } + + private interface Action { + + void read(Decoder decoder, Record record, PrimitiveAvroRecordReader reader) + throws IOException; + } + + private static Action action(Schema schema, Selection selection) { + if (selection.type != null) { + return selectedAction(schema, selection); + } + if (isNullableRecord(schema)) { + Action nonNullAction = action(schema.getTypes().get(1), selection); + return (decoder, record, reader) -> { + int branch = decoder.readIndex(); + if (branch == 0) { + selection.markNull(record); + } else if (branch == 1) { + nonNullAction.read(decoder, record, reader); + } else { + throw new IOException("Invalid nullable record union branch " + branch); + } + }; + } + if (schema.getType() != Schema.Type.RECORD) { + throw mismatch(selection, schema); + } + Action[] fields = new Action[schema.getFields().size()]; + for (int i = 0; i < fields.length; i++) { + Schema.Field field = schema.getFields().get(i); + Selection child = selection.children.get(field.name()); + fields[i] = child == null ? skipAction(field.schema()) : action(field.schema(), child); + } + for (String child : selection.children.keySet()) { + if (schema.getField(child) == null) { + throw new IllegalArgumentException( + "Projected field " + + child + + " does not exist in Avro record " + + schema.getFullName() + + '.'); + } + } + return (decoder, record, reader) -> { + for (Action field : fields) { + field.read(decoder, record, reader); + } + }; + } + + private static boolean isNullableRecord(Schema schema) { + return schema.getType() == Schema.Type.UNION + && schema.getTypes().size() == 2 + && schema.getTypes().get(0).getType() == Schema.Type.NULL + && schema.getTypes().get(1).getType() == Schema.Type.RECORD; + } + + private static Action selectedAction(Schema schema, Selection selection) { + DataType type = selection.type; + if (!directlyDecodable(type.getTypeRoot())) { + return rawAction(schema, selection); + } + + if (!type.isNullable()) { + return nonNullValueAction(schema, selection); + } + if (!isNullable(schema)) { + throw mismatch(selection, schema); + } + Action nonNull = nonNullValueAction(schema.getTypes().get(1), selection); + return (decoder, record, reader) -> { + int branch = decoder.readIndex(); + if (branch == 0) { + record.nulls[selection.index] = true; + } else if (branch == 1) { + nonNull.read(decoder, record, reader); + } else { + throw new IOException("Invalid nullable field union branch " + branch); + } + }; + } + + private static boolean directlyDecodable(DataTypeRoot typeRoot) { + switch (typeRoot) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case CHAR: + case VARCHAR: + case BINARY: + case VARBINARY: + return true; + default: + return false; + } + } + + private static Action nonNullValueAction(Schema schema, Selection selection) { + int index = selection.index; + switch (selection.type.getTypeRoot()) { + case BOOLEAN: + require(schema, Schema.Type.BOOLEAN, selection); + return (decoder, record, reader) -> + record.longs[index] = decoder.readBoolean() ? 1L : 0L; + case TINYINT: + case SMALLINT: + case INTEGER: + require(schema, Schema.Type.INT, selection); + return (decoder, record, reader) -> record.longs[index] = decoder.readInt(); + case BIGINT: + require(schema, Schema.Type.LONG, selection); + return (decoder, record, reader) -> record.longs[index] = decoder.readLong(); + case CHAR: + case VARCHAR: + require(schema, Schema.Type.STRING, selection); + return bytesAction(index); + case BINARY: + case VARBINARY: + require(schema, Schema.Type.BYTES, selection); + return bytesAction(index); + default: + throw new IllegalStateException( + "Unsupported primitive type " + selection.type.getTypeRoot()); + } + } + + private static Action rawAction(Schema schema, Selection selection) { + int index = selection.index; + if (selection.type.isNullable()) { + if (!isNullable(schema)) { + throw mismatch(selection, schema); + } + Action skip = skipAction(schema.getTypes().get(1)); + return (decoder, record, reader) -> { + int start = reader.position(); + int branch = decoder.readIndex(); + if (branch == 0) { + record.nulls[index] = true; + } else if (branch == 1) { + skip.read(decoder, record, reader); + } else { + throw new IOException("Invalid nullable field union branch " + branch); + } + captureRaw(record, reader, index, start); + }; + } + + Action skip = skipAction(schema); + return (decoder, record, reader) -> { + int start = reader.position(); + skip.read(decoder, record, reader); + captureRaw(record, reader, index, start); + }; + } + + private static void captureRaw( + Record record, PrimitiveAvroRecordReader reader, int index, int start) + throws IOException { + int end = reader.position(); + record.bytes[index] = reader.blockBytes; + record.offsets[index] = reader.blockOffset + start; + record.lengths[index] = end - start; + } + + private static Action bytesAction(int index) { + return (decoder, record, reader) -> { + long length = decoder.readLong(); + if (length < 0 || length > Integer.MAX_VALUE) { + throw new IOException("Invalid Avro byte sequence length " + length); + } + record.bytes[index] = reader.blockBytes; + record.offsets[index] = reader.blockOffset + reader.position(); + record.lengths[index] = (int) length; + decoder.skipFixed((int) length); + }; + } + + private static boolean isNullable(Schema schema) { + return schema.getType() == Schema.Type.UNION + && schema.getTypes().size() == 2 + && schema.getTypes().get(0).getType() == Schema.Type.NULL; + } + + private static void require(Schema schema, Schema.Type type, Selection selection) { + if (schema.getType() != type) { + throw mismatch(selection, schema); + } + } + + private static IllegalArgumentException mismatch(Selection selection, Schema schema) { + return new IllegalArgumentException( + "Projected field " + + selection.path + + " is incompatible with Avro schema " + + schema); + } + + private static Action skipAction(Schema schema) { + switch (schema.getType()) { + case NULL: + return (decoder, record, reader) -> decoder.readNull(); + case BOOLEAN: + return (decoder, record, reader) -> decoder.readBoolean(); + case INT: + return (decoder, record, reader) -> decoder.readInt(); + case LONG: + return (decoder, record, reader) -> decoder.readLong(); + case FLOAT: + return (decoder, record, reader) -> decoder.readFloat(); + case DOUBLE: + return (decoder, record, reader) -> decoder.readDouble(); + case STRING: + return (decoder, record, reader) -> decoder.skipString(); + case BYTES: + return (decoder, record, reader) -> decoder.skipBytes(); + case ENUM: + return (decoder, record, reader) -> decoder.readEnum(); + case FIXED: + int fixedSize = schema.getFixedSize(); + return (decoder, record, reader) -> decoder.skipFixed(fixedSize); + case RECORD: + Action[] fields = new Action[schema.getFields().size()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = skipAction(schema.getFields().get(i).schema()); + } + return (decoder, record, reader) -> { + for (Action field : fields) { + field.read(decoder, record, reader); + } + }; + case UNION: + Action[] branches = new Action[schema.getTypes().size()]; + for (int i = 0; i < branches.length; i++) { + branches[i] = skipAction(schema.getTypes().get(i)); + } + return (decoder, record, reader) -> { + int branch = decoder.readIndex(); + if (branch < 0 || branch >= branches.length) { + throw new IOException("Invalid Avro union branch " + branch); + } + branches[branch].read(decoder, record, reader); + }; + case ARRAY: + Action element = skipAction(schema.getElementType()); + return (decoder, record, reader) -> { + for (long count = decoder.readArrayStart(); + count != 0; + count = decoder.arrayNext()) { + for (long i = 0; i < count; i++) { + element.read(decoder, record, reader); + } + } + }; + case MAP: + Action value = skipAction(schema.getValueType()); + return (decoder, record, reader) -> { + for (long count = decoder.readMapStart(); + count != 0; + count = decoder.mapNext()) { + for (long i = 0; i < count; i++) { + decoder.skipString(); + value.read(decoder, record, reader); + } + } + }; + default: + throw new IllegalArgumentException("Unsupported Avro schema " + schema); + } + } + + private static final class Selection { + + private final Map children = new HashMap<>(); + private DataType type; + private int index = -1; + private String path = ""; + private int valueCount; + + private static Selection from(RowType projectedRowType) { + Selection root = new Selection(); + int[] nextIndex = new int[] {0}; + for (DataField field : projectedRowType.getFields()) { + root.children.put(field.name(), from(field.type(), field.name(), nextIndex)); + } + root.valueCount = nextIndex[0]; + return root; + } + + private static Selection from(DataType type, String path, int[] nextIndex) { + Selection selection = new Selection(); + selection.path = path; + if (type.getTypeRoot() != DataTypeRoot.ROW) { + selection.type = type; + selection.index = nextIndex[0]++; + return selection; + } + + for (DataField field : ((RowType) type).getFields()) { + selection.children.put( + field.name(), from(field.type(), path + '.' + field.name(), nextIndex)); + } + return selection; + } + + private int valueCount() { + return valueCount; + } + + private void markNull(Record record) { + if (type != null) { + record.nulls[index] = true; + } + for (Selection child : children.values()) { + child.markNull(record); + } + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java new file mode 100644 index 000000000000..195ea9bad776 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.avro.primitive; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.PositionOutputStream; + +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.RawBlockReader; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** Avro writer which accepts normal rows, encoded records and compressed blocks. */ +public final class PrimitiveAvroWriter implements AutoCloseable { + + private final DataFileWriter writer; + private final PositionOutputStream out; + + public PrimitiveAvroWriter(DataFileWriter writer, PositionOutputStream out) { + this.writer = writer; + this.out = out; + } + + public void addElement(InternalRow element) throws IOException { + writer.append(element); + } + + public void addEncoded(ByteBuffer record) throws IOException { + writer.appendEncoded(record); + } + + public void addEncodedBlock(PrimitiveAvroBlock block) throws IOException { + RawBlockReader.appendBlock(writer, block.rawBlock()); + } + + public boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws IOException { + return suggestedCheck && out.getPos() >= targetSize; + } + + @Override + public void close() throws IOException { + writer.close(); + } +} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index b8cb87f1c42c..131b65e2f9e9 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -18,6 +18,8 @@ package org.apache.paimon.format.avro; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; @@ -25,6 +27,8 @@ import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.avro.AvroBlockReader.BorrowedBlock; +import org.apache.paimon.format.avro.AvroBlockReader.RawBlock; +import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; @@ -50,6 +54,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.NoSuchElementException; @@ -283,6 +289,161 @@ void testRowReaderProjectsIntoReusedRow() throws IOException { assertThat(decoder.isEnd()).isTrue(); } + @Test + void testPrimitiveReader() throws IOException { + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT().notNull()) + .field("value", DataTypes.BIGINT()) + .build(); + AvroFileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(1, 10L)); + writer.addElement(GenericRow.of(2, null)); + writer.close(); + } + + assertThatThrownBy( + () -> + format.createPrimitiveReader( + fileIO, + file, + rowType, + RowType.builder() + .field("missing", DataTypes.INT().notNull()) + .build())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("missing"); + + try (PrimitiveAvroRecordReader reader = + format.createPrimitiveReader(fileIO, file, rowType, rowType)) { + PrimitiveAvroRecordReader.Record first = reader.next(); + assertThat(first.longValue(0)).isEqualTo(1); + assertThat(first.longValue(1)).isEqualTo(10L); + + PrimitiveAvroRecordReader.Record second = reader.next(); + assertThat(second.longValue(0)).isEqualTo(2); + assertThat(second.isNull(1)).isTrue(); + assertThat(reader.hasNext()).isFalse(); + } + } + + @Test + void testPrimitiveReaderFactoryDerivesNestedProjectionAndKinds() throws IOException { + RowType fileType = + RowType.builder() + .field("embedded", DataTypes.VARBINARY(100)) + .field("external", DataTypes.STRING()) + .field("extra", DataTypes.ARRAY(DataTypes.STRING().notNull()).notNull()) + .build(); + RowType rowType = + RowType.builder() + .field("ignored", DataTypes.INT().notNull()) + .field("partition", DataTypes.VARBINARY(100).notNull()) + .field("file", fileType.notNull()) + .build(); + RowType projectedType = + new RowType( + false, + Arrays.asList( + rowType.getField("partition"), + rowType.getField("file") + .newType( + fileType.project("external", "extra", "embedded") + .notNull()))); + AvroFileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + byte[] partition = new byte[] {1, 2, 3}; + byte[] embedded = new byte[] {4, 5}; + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement( + GenericRow.of( + 9, + partition, + GenericRow.of( + embedded, + BinaryString.fromString("path"), + new GenericArray( + new Object[] { + BinaryString.fromString("a"), + BinaryString.fromString("b") + })))); + writer.addElement( + GenericRow.of( + 10, + partition, + GenericRow.of(null, null, new GenericArray(new Object[0])))); + writer.close(); + } + + try (PrimitiveAvroRecordReader reader = + format.createPrimitiveReader(fileIO, file, rowType, projectedType)) { + PrimitiveAvroRecordReader.Record first = reader.next(); + assertThat(slice(first, 0)).containsExactly(partition); + assertThat(new String(slice(first, 1), StandardCharsets.UTF_8)).isEqualTo("path"); + assertThat(first.length(2)).isPositive(); + assertThat(slice(first, 3)).containsExactly(embedded); + + PrimitiveAvroRecordReader.Record second = reader.next(); + assertThat(slice(second, 0)).containsExactly(partition); + assertThat(second.isNull(1)).isTrue(); + assertThat(second.length(2)).isPositive(); + assertThat(second.isNull(3)).isTrue(); + assertThat(reader.hasNext()).isFalse(); + } + } + + @Test + void testPrimitiveReaderFactoryReadsLargeZstdBlock() throws IOException { + RowType rowType = + RowType.builder() + .field("payload", DataTypes.VARBINARY(500_000).notNull()) + .field("id", DataTypes.INT().notNull()) + .build(); + AvroFileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + byte[] payload = new byte[400_000]; + Arrays.fill(payload, (byte) 7); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(payload, 42)); + writer.close(); + } + + try (AvroBlockReader blockReader = new AvroBlockReader(fileIO.newInputStream(file))) { + RawBlock block = blockReader.nextRawBlock(null); + assertThat(block.recordCount()).isEqualTo(1); + assertThat(block.compressedSize()).isPositive(); + ByteBuffer decoded = block.decompress(null); + assertThat(decoded.remaining()).isGreaterThan(payload.length); + assertThat(decoded.get(10)).isEqualTo((byte) 7); + assertThat(block.decompress(ByteBuffer.allocate(1))).isEqualTo(decoded); + } + + try (PrimitiveAvroRecordReader reader = + format.createPrimitiveReader(fileIO, file, rowType, rowType.project("id"))) { + assertThat(reader.hasNext()).isTrue(); + assertThat(reader.next().longValue(0)).isEqualTo(42); + assertThat(reader.hasNext()).isFalse(); + } + } + + private static byte[] slice(PrimitiveAvroRecordReader.Record record, int index) { + return Arrays.copyOfRange( + record.bytes(index), + record.offset(index), + record.offset(index) + record.length(index)); + } + @Test void testGetRealIOException() throws IOException { RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()); From 783d0cfcaf55d000ebf360e7b9c96c29979548a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 11 Aug 2026 15:24:37 +0800 Subject: [PATCH 2/2] [format] Refactor Avro raw block APIs --- .../apache/paimon/manifest/ManifestFile.java | 12 +- .../operation/ManifestEntryRunMergePlan.java | 6 +- .../java/org/apache/avro/file/RawBlock.java | 175 ++++++++++++++++++ .../org/apache/avro/file/RawBlockReader.java | 140 +------------- .../paimon/format/avro/AvroBlockReader.java | 51 +---- ...veAvroWriter.java => AvroBlockWriter.java} | 11 +- .../paimon/format/avro/AvroFileFormat.java | 5 +- ...mitiveAvroBlock.java => AvroRawBlock.java} | 28 ++- .../primitive/PrimitiveAvroRecordReader.java | 32 ++-- .../format/avro/AvroFileFormatTest.java | 3 +- 10 files changed, 235 insertions(+), 228 deletions(-) create mode 100644 paimon-format/src/main/java/org/apache/avro/file/RawBlock.java rename paimon-format/src/main/java/org/apache/paimon/format/avro/{primitive/PrimitiveAvroWriter.java => AvroBlockWriter.java} (81%) rename paimon-format/src/main/java/org/apache/paimon/format/avro/{primitive/PrimitiveAvroBlock.java => AvroRawBlock.java} (59%) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 39d5d126f7ce..b2cc38d15741 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -25,10 +25,10 @@ import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsCollector; +import org.apache.paimon.format.avro.AvroBlockWriter; import org.apache.paimon.format.avro.AvroFileFormat; -import org.apache.paimon.format.avro.primitive.PrimitiveAvroBlock; +import org.apache.paimon.format.avro.AvroRawBlock; import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; -import org.apache.paimon.format.avro.primitive.PrimitiveAvroWriter; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; @@ -512,7 +512,7 @@ public void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadata } public void writeEncodedBlock( - PrimitiveAvroBlock block, EncodedManifestBlock metadata, long blockRecordCount) + AvroRawBlock block, EncodedManifestBlock metadata, long blockRecordCount) throws IOException { if (blockRecordCount != block.recordCount()) { throw new IllegalArgumentException( @@ -609,7 +609,7 @@ private final class PrimitiveManifestFileWriter { private final Map encodedPartitionCounts = new IdentityHashMap<>(); private final long[] repeatedNullCounts = new long[partitionType.getFieldCount()]; private @Nullable PositionOutputStream out; - private @Nullable PrimitiveAvroWriter writer; + private @Nullable AvroBlockWriter writer; private @Nullable Long outputBytes; private long numAddedFiles; private long numDeletedFiles; @@ -630,7 +630,7 @@ private PrimitiveManifestFileWriter(Path path) { out = fileIO.newOutputStream(path, false); outputCreated = true; writer = - avroFileFormat.createPrimitiveWriter( + avroFileFormat.createBlockWriter( out, ManifestEntry.MANIFEST_ROW_TYPE, compression); } catch (IOException failure) { IOUtils.closeQuietly(writer); @@ -667,7 +667,7 @@ private void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadat addEncodedPartition(metadata.partition, 1); } - private void writeEncodedBlock(PrimitiveAvroBlock block, EncodedManifestBlock metadata) + private void writeEncodedBlock(AvroRawBlock block, EncodedManifestBlock metadata) throws IOException { ensureOpen(); writer.addEncodedBlock(block); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java index b85dafe41758..4dbdbb404d85 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -20,7 +20,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.serializer.InternalRowSerializer; -import org.apache.paimon.format.avro.primitive.PrimitiveAvroBlock; +import org.apache.paimon.format.avro.AvroRawBlock; import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader.Record; import org.apache.paimon.manifest.BinaryManifestEntry; @@ -369,7 +369,7 @@ default ManifestEntryRunMergeEntry.Key blockLastKey() { throw new UnsupportedOperationException(); } - default PrimitiveAvroBlock encodedBlock() { + default AvroRawBlock encodedBlock() { throw new UnsupportedOperationException(); } @@ -560,7 +560,7 @@ public ManifestEntryRunMergeEntry.Key blockLastKey() { } @Override - public PrimitiveAvroBlock encodedBlock() { + public AvroRawBlock encodedBlock() { return reader.currentRawBlock(); } diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java new file mode 100644 index 000000000000..8e8d9b164195 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.file; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** Reusable compressed Avro block. */ +public final class RawBlock { + + private DataFileStream.DataBlock block; + private Codec codec; + private Schema schema; + private boolean decompressed; + private ByteBuffer decompressedBuffer; + + RawBlock(DataFileStream.DataBlock block, Codec codec, Schema schema) { + replace(block, codec, schema); + } + + RawBlock replace(DataFileStream.DataBlock block, Codec codec, Schema schema) { + this.block = block; + this.codec = codec; + this.schema = schema; + this.decompressed = false; + this.decompressedBuffer = null; + return this; + } + + DataFileStream.DataBlock dataBlock() { + return block; + } + + public long recordCount() { + return block.getNumEntries(); + } + + public int compressedSize() { + if (decompressed) { + throw new IllegalStateException("The Avro block has already been decompressed."); + } + return block.getBlockSize(); + } + + public ByteBuffer decompress(ByteBuffer reuse) throws IOException { + if (!decompressed) { + if (codec instanceof ZstandardCodec) { + ByteBuffer source = block.getAsByteBuffer(); + ByteBuffer target = + reuse != null && reuse.hasArray() ? reuse : ByteBuffer.allocate(256 * 1024); + int size = 0; + try (InputStream compressed = + new ByteArrayInputStream( + source.array(), + source.arrayOffset() + source.position(), + source.remaining()); + InputStream input = ZstandardLoader.input(compressed, true)) { + while (true) { + if (size == target.capacity()) { + int grownCapacity = + target.capacity() == 0 + ? 256 * 1024 + : Math.multiplyExact(target.capacity(), 2); + ByteBuffer grown = ByteBuffer.allocate(grownCapacity); + System.arraycopy( + target.array(), + target.arrayOffset(), + grown.array(), + grown.arrayOffset(), + size); + target = grown; + } + int read = + input.read( + target.array(), + target.arrayOffset() + size, + target.capacity() - size); + if (read < 0) { + break; + } + size += read; + } + } + target.position(0); + target.limit(size); + decompressedBuffer = target.duplicate(); + } else { + block.decompressUsing(codec); + decompressedBuffer = block.getAsByteBuffer().duplicate(); + } + decompressed = true; + } + return decompressedBuffer.duplicate(); + } + + /** Returns a single-block stream for appending this compressed block to an Avro writer. */ + public DataFileStream asStream() throws IOException { + if (decompressed) { + throw new IllegalStateException("A decompressed Avro block cannot be copied raw."); + } + return new SingleBlockStream(schema, codec, block); + } + + private static final class SingleBlockStream extends DataFileStream { + + private final Schema schema; + private final Codec codec; + private DataBlock block; + + private SingleBlockStream(Schema schema, Codec codec, DataBlock block) throws IOException { + super(new NoOpDatumReader()); + this.schema = schema; + this.codec = codec; + this.block = block; + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + Codec resolveCodec() { + return codec; + } + + @Override + boolean hasNextBlock() { + return block != null; + } + + @Override + DataBlock nextRawBlock(DataBlock reuse) { + DataBlock result = block; + block = null; + return result; + } + + @Override + public void close() {} + } + + private static final class NoOpDatumReader implements DatumReader { + + @Override + public void setSchema(Schema schema) {} + + @Override + public D read(D reuse, Decoder decoder) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java index 96a19bab95f8..43a68c54a44d 100644 --- a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -22,10 +22,8 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.io.Decoder; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; /** Package bridge exposing Avro's compressed blocks without reflection. */ public final class RawBlockReader extends DataFileStream { @@ -39,148 +37,12 @@ public boolean hasNextRawBlock() { } public RawBlock nextRawBlock(RawBlock reuse) throws IOException { - DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.block); + DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.dataBlock()); return reuse == null ? new RawBlock(raw, resolveCodec(), getSchema()) : reuse.replace(raw, resolveCodec(), getSchema()); } - /** Appends a source block directly, allowing Avro to recompress it when codecs differ. */ - public static void appendBlock(DataFileWriter writer, RawBlock block) - throws IOException { - if (block.decompressed) { - throw new IllegalStateException("A decompressed Avro block cannot be copied raw."); - } - writer.appendAllFrom( - new SingleBlockStream(block.schema, block.codec, block.block), false); - } - - /** Reusable Avro block. */ - public static final class RawBlock { - - private DataBlock block; - private Codec codec; - private Schema schema; - private boolean decompressed; - private ByteBuffer decompressedBuffer; - - private RawBlock(DataBlock block, Codec codec, Schema schema) { - replace(block, codec, schema); - } - - private RawBlock replace(DataBlock block, Codec codec, Schema schema) { - this.block = block; - this.codec = codec; - this.schema = schema; - this.decompressed = false; - this.decompressedBuffer = null; - return this; - } - - public long recordCount() { - return block.getNumEntries(); - } - - public int compressedSize() { - if (decompressed) { - throw new IllegalStateException("The Avro block has already been decompressed."); - } - return block.getBlockSize(); - } - - public ByteBuffer decompress(ByteBuffer reuse) throws IOException { - if (!decompressed) { - if (codec instanceof ZstandardCodec) { - ByteBuffer source = block.getAsByteBuffer(); - ByteBuffer target = - reuse != null && reuse.hasArray() - ? reuse - : ByteBuffer.allocate(256 * 1024); - int size = 0; - try (InputStream compressed = - new ByteArrayInputStream( - source.array(), - source.arrayOffset() + source.position(), - source.remaining()); - InputStream input = ZstandardLoader.input(compressed, true)) { - while (true) { - if (size == target.capacity()) { - int grownCapacity = - target.capacity() == 0 - ? 256 * 1024 - : Math.multiplyExact(target.capacity(), 2); - ByteBuffer grown = ByteBuffer.allocate(grownCapacity); - System.arraycopy( - target.array(), - target.arrayOffset(), - grown.array(), - grown.arrayOffset(), - size); - target = grown; - } - int read = - input.read( - target.array(), - target.arrayOffset() + size, - target.capacity() - size); - if (read < 0) { - break; - } - size += read; - } - } - target.position(0); - target.limit(size); - decompressedBuffer = target.duplicate(); - } else { - block.decompressUsing(codec); - decompressedBuffer = block.getAsByteBuffer().duplicate(); - } - decompressed = true; - } - return decompressedBuffer.duplicate(); - } - } - - private static final class SingleBlockStream extends DataFileStream { - - private final Schema schema; - private final Codec codec; - private DataBlock block; - - private SingleBlockStream(Schema schema, Codec codec, DataBlock block) throws IOException { - super(new NoOpDatumReader()); - this.schema = schema; - this.codec = codec; - this.block = block; - } - - @Override - public Schema getSchema() { - return schema; - } - - @Override - Codec resolveCodec() { - return codec; - } - - @Override - boolean hasNextBlock() { - return block != null; - } - - @Override - DataBlock nextRawBlock(DataBlock reuse) { - DataBlock result = block; - block = null; - return result; - } - - @Override - public void close() {} - } - private static final class NoOpDatumReader implements DatumReader { @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index 1a89e24bd5ff..38cafb37325f 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -22,7 +22,7 @@ import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; -import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.RawBlock; import org.apache.avro.file.RawBlockReader; import javax.annotation.Nullable; @@ -42,7 +42,7 @@ public final class AvroBlockReader implements Closeable { private final RawBlockReader reader; - private @Nullable RawBlock borrowedRawBlock; + private @Nullable AvroRawBlock borrowedRawBlock; private @Nullable ByteBuffer decompressionBuffer; private long currentBlockRecordCount = -1; @@ -107,11 +107,11 @@ public BorrowedBlock nextBorrowedBlock() throws IOException { * reused again. The block can be skipped without decompression, decompressed lazily, or copied * directly to a compatible Avro writer. */ - public RawBlock nextRawBlock(@Nullable RawBlock reuse) throws IOException { - RawBlockReader.RawBlock block = + public AvroRawBlock nextRawBlock(@Nullable AvroRawBlock reuse) throws IOException { + RawBlock block = replaceAvroRuntimeException( - () -> reader.nextRawBlock(reuse == null ? null : reuse.block)); - RawBlock result = reuse == null ? new RawBlock(block) : reuse.replace(block); + () -> reader.nextRawBlock(reuse == null ? null : reuse.rawBlock())); + AvroRawBlock result = reuse == null ? new AvroRawBlock(block) : reuse.replace(block); currentBlockRecordCount = result.recordCount(); return result; } @@ -172,45 +172,6 @@ public long recordCount() { } } - /** Reusable compressed Avro block. */ - public static final class RawBlock { - - private RawBlockReader.RawBlock block; - - private RawBlock(RawBlockReader.RawBlock block) { - this.block = block; - } - - private RawBlock replace(RawBlockReader.RawBlock block) { - this.block = block; - return this; - } - - public long recordCount() { - return block.recordCount(); - } - - /** Returns the compressed payload size. This block must not have been decompressed. */ - public int compressedSize() { - return block.compressedSize(); - } - - /** - * Lazily decompresses this block, reusing the supplied heap buffer when possible. - * - *

      The returned view remains owned by this block and is invalidated when this holder is - * reused for another block. - */ - public ByteBuffer decompress(@Nullable ByteBuffer reuse) throws IOException { - return block.decompress(reuse); - } - - /** Copies this still-compressed block directly to a compatible Avro writer. */ - public void appendTo(DataFileWriter writer) throws IOException { - RawBlockReader.appendBlock(writer, block); - } - } - @FunctionalInterface private interface IOSupplier { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java similarity index 81% rename from paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java rename to paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java index 195ea9bad776..da55bb7c599b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroWriter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java @@ -16,24 +16,23 @@ * limitations under the License. */ -package org.apache.paimon.format.avro.primitive; +package org.apache.paimon.format.avro; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.PositionOutputStream; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.file.RawBlockReader; import java.io.IOException; import java.nio.ByteBuffer; /** Avro writer which accepts normal rows, encoded records and compressed blocks. */ -public final class PrimitiveAvroWriter implements AutoCloseable { +public final class AvroBlockWriter implements AutoCloseable { private final DataFileWriter writer; private final PositionOutputStream out; - public PrimitiveAvroWriter(DataFileWriter writer, PositionOutputStream out) { + public AvroBlockWriter(DataFileWriter writer, PositionOutputStream out) { this.writer = writer; this.out = out; } @@ -46,8 +45,8 @@ public void addEncoded(ByteBuffer record) throws IOException { writer.appendEncoded(record); } - public void addEncodedBlock(PrimitiveAvroBlock block) throws IOException { - RawBlockReader.appendBlock(writer, block.rawBlock()); + public void addEncodedBlock(AvroRawBlock block) throws IOException { + writer.appendAllFrom(block.asStream(), false); } public boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws IOException { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 11ab121acf8f..24a10801b82e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -26,7 +26,6 @@ import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; -import org.apache.paimon.format.avro.primitive.PrimitiveAvroWriter; import org.apache.paimon.fs.CloseShieldOutputStream; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -100,7 +99,7 @@ public PrimitiveAvroRecordReader createPrimitiveReader( fileIO.newInputStream(path), expectedSchema, projectedRowType); } - public PrimitiveAvroWriter createPrimitiveWriter( + public AvroBlockWriter createBlockWriter( PositionOutputStream out, RowType rowType, String compression) throws IOException { Schema schema = AvroSchemaConverter.convertToSchema(rowType, options.get(AVRO_ROW_NAME_MAPPING)); @@ -109,7 +108,7 @@ public PrimitiveAvroWriter createPrimitiveWriter( writer.setCodec(createCodecFactory(compression)); writer.setFlushOnEveryBlock(false); writer.create(schema, new CloseShieldOutputStream(out)); - return new PrimitiveAvroWriter(writer, out); + return new AvroBlockWriter(writer, out); } @Override diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java similarity index 59% rename from paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java rename to paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java index f7a28032692f..a607423ce9b6 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroBlock.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java @@ -16,23 +16,26 @@ * limitations under the License. */ -package org.apache.paimon.format.avro.primitive; +package org.apache.paimon.format.avro; -import org.apache.avro.file.RawBlockReader.RawBlock; +import org.apache.avro.file.DataFileStream; +import org.apache.avro.file.RawBlock; + +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; -/** Reusable compressed Avro block exposed by {@link PrimitiveAvroRecordReader}. */ -public final class PrimitiveAvroBlock { +/** Reusable compressed block from an Avro object container file. */ +public final class AvroRawBlock { private RawBlock block; - PrimitiveAvroBlock(RawBlock block) { + AvroRawBlock(RawBlock block) { this.block = block; } - PrimitiveAvroBlock replace(RawBlock block) { + AvroRawBlock replace(RawBlock block) { this.block = block; return this; } @@ -45,11 +48,22 @@ public long recordCount() { return block.recordCount(); } + /** Returns the compressed payload size. This block must not have been decompressed. */ public int compressedSize() { return block.compressedSize(); } - ByteBuffer decompress(ByteBuffer reuse) throws IOException { + /** + * Lazily decompresses this block, reusing the supplied heap buffer when possible. + * + *

      The returned view remains owned by this block and is invalidated when this holder is + * reused for another block. + */ + public ByteBuffer decompress(@Nullable ByteBuffer reuse) throws IOException { return block.decompress(reuse); } + + DataFileStream asStream() throws IOException { + return block.asStream(); + } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java index 2e413d4815ad..c8432866993c 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/primitive/PrimitiveAvroRecordReader.java @@ -18,6 +18,8 @@ package org.apache.paimon.format.avro.primitive; +import org.apache.paimon.format.avro.AvroBlockReader; +import org.apache.paimon.format.avro.AvroRawBlock; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeRoot; @@ -25,8 +27,6 @@ import org.apache.paimon.utils.IOUtils; import org.apache.avro.Schema; -import org.apache.avro.file.RawBlockReader; -import org.apache.avro.file.RawBlockReader.RawBlock; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; @@ -46,7 +46,7 @@ */ public final class PrimitiveAvroRecordReader implements AutoCloseable { - private final RawBlockReader stream; + private final AvroBlockReader stream; private final Action rootAction; private final Record record; private final boolean rawBlockCopySupported; @@ -57,8 +57,7 @@ public final class PrimitiveAvroRecordReader implements AutoCloseable { private int blockOffset; private int blockLength; private long blockRemaining; - private RawBlock rawBlock; - private PrimitiveAvroBlock block; + private AvroRawBlock block; private long blockOrdinal = -1; private long blockRecordIndex; private long blockRecordCount; @@ -68,18 +67,18 @@ public final class PrimitiveAvroRecordReader implements AutoCloseable { public PrimitiveAvroRecordReader( InputStream input, Schema expectedEncodedSchema, RowType projectedRowType) throws IOException { - RawBlockReader opened = null; + AvroBlockReader opened = null; try { - opened = new RawBlockReader(input); - if (!hasSameBinaryEncoding(opened.getSchema(), expectedEncodedSchema)) { + opened = new AvroBlockReader(input); + if (!hasSameBinaryEncoding(opened.schema(), expectedEncodedSchema)) { throw new UnsupportedOperationException( "Avro schema is not binary-compatible with the requested encoded row type."); } - this.rawBlockCopySupported = opened.getSchema().equals(expectedEncodedSchema); + this.rawBlockCopySupported = opened.schema().equals(expectedEncodedSchema); Selection root = Selection.from(projectedRowType); this.record = new Record(root.valueCount()); - this.rootAction = action(opened.getSchema(), root); + this.rootAction = action(opened.schema(), root); this.stream = opened; } catch (IOException | RuntimeException | Error failure) { IOUtils.closeQuietly(opened == null ? input : opened); @@ -108,13 +107,13 @@ public boolean rawBlockCopySupported() { /** * Returns whether another compressed block is available. The current block must be consumed. */ - public boolean hasNextRawBlock() { + public boolean hasNextRawBlock() throws IOException { ensureBlockConsumed(); - return stream.hasNextRawBlock(); + return stream.hasNextBlock(); } /** Loads, but does not decompress, the next Avro block. */ - public PrimitiveAvroBlock nextRawBlock() throws IOException { + public AvroRawBlock nextRawBlock() throws IOException { ensureBlockConsumed(); if (!loadNextBlock()) { throw new NoSuchElementException(); @@ -122,7 +121,7 @@ public PrimitiveAvroBlock nextRawBlock() throws IOException { return block; } - public PrimitiveAvroBlock currentRawBlock() { + public AvroRawBlock currentRawBlock() { if (block == null || blockRemaining == 0) { throw new IllegalStateException("No current Avro block."); } @@ -161,11 +160,10 @@ private void decodeNextRecord() throws IOException { } private boolean loadNextBlock() throws IOException { - if (!stream.hasNextRawBlock()) { + if (!stream.hasNextBlock()) { return false; } - rawBlock = stream.nextRawBlock(rawBlock); - block = block == null ? new PrimitiveAvroBlock(rawBlock) : block.replace(rawBlock); + block = stream.nextRawBlock(block); blockRemaining = block.recordCount(); blockRecordCount = blockRemaining; blockRecordIndex = 0; diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 131b65e2f9e9..308548110dd7 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -27,7 +27,6 @@ import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.avro.AvroBlockReader.BorrowedBlock; -import org.apache.paimon.format.avro.AvroBlockReader.RawBlock; import org.apache.paimon.format.avro.primitive.PrimitiveAvroRecordReader; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -420,7 +419,7 @@ void testPrimitiveReaderFactoryReadsLargeZstdBlock() throws IOException { } try (AvroBlockReader blockReader = new AvroBlockReader(fileIO.newInputStream(file))) { - RawBlock block = blockReader.nextRawBlock(null); + AvroRawBlock block = blockReader.nextRawBlock(null); assertThat(block.recordCount()).isEqualTo(1); assertThat(block.compressedSize()).isPositive(); ByteBuffer decoded = block.decompress(null);