diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 70989a7bc76b..84a716637c95 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1023,6 +1023,12 @@ String Partition field name to sort manifest entries by. Validated by schema validation, if not configured, defaults to the first partition field. + +
manifest-sort.run-merge-optimize.enabled
+ true + Boolean + Whether to use streaming run merge for RowID-based manifest sorting. When disabled, the external sorter is used without changing the RowID sort semantics. +
manifest.compression
"zstd" diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index c65da81a80e1..923b5cafadc1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -589,6 +589,15 @@ public InlineElement getDescription() { + " skipped. Set to a larger value to allow more aggressive" + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it."); + public static final ConfigOption MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED = + key("manifest-sort.run-merge-optimize.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to use streaming run merge for RowID-based manifest sorting." + + " When disabled, the external sorter is used without changing" + + " the RowID sort semantics."); + public static final ConfigOption PARTITION_DEFAULT_NAME = key("partition.default-name") .stringType() @@ -3063,6 +3072,10 @@ public long manifestSortMaxRewriteSize() { return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes(); } + public boolean manifestSortRunMergeOptimizeEnabled() { + return options.get(MANIFEST_SORT_RUN_MERGE_OPTIMIZE_ENABLED); + } + public String partitionDefaultName() { return options.get(PARTITION_DEFAULT_NAME); } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java index 09d9ded426a4..274e20abdc0a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayKey.java @@ -47,8 +47,7 @@ byte[] bytes() { public boolean equals(Object obj) { return obj == this || (obj instanceof ByteArrayKey && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes)) - || (obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes())); + || (obj instanceof ByteArrayLookupKey && obj.equals(this)); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java index aaa913ace7ee..023a6b686609 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ByteArrayLookupKey.java @@ -20,8 +20,6 @@ import javax.annotation.Nullable; -import java.util.Arrays; - import static org.apache.paimon.utils.Preconditions.checkArgument; /** @@ -33,6 +31,8 @@ public final class ByteArrayLookupKey { private @Nullable byte[] bytes; + private int offset; + private int length; private int hash; public ByteArrayLookupKey() {} @@ -43,12 +43,26 @@ public ByteArrayLookupKey(byte[] bytes) { public void reset(byte[] bytes) { checkArgument(bytes != null, "Byte array cannot be null."); + reset(bytes, 0, bytes.length); + } + + public void reset(byte[] bytes, int offset, int length) { + checkArgument(bytes != null, "Byte array cannot be null."); + checkArgument(offset >= 0 && length >= 0 && offset <= bytes.length - length); this.bytes = bytes; - this.hash = Arrays.hashCode(bytes); + this.offset = offset; + this.length = length; + int hash = 1; + for (int i = offset; i < offset + length; i++) { + hash = 31 * hash + bytes[i]; + } + this.hash = hash; } public void clear() { bytes = null; + offset = 0; + length = 0; hash = 0; } @@ -62,14 +76,38 @@ public boolean equals(Object obj) { return obj == this || (bytes != null && obj instanceof ByteArrayKey - && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes())) + && equals(((ByteArrayKey) obj).bytes())) || (bytes != null && obj instanceof ByteArrayLookupKey - && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes)); + && equals((ByteArrayLookupKey) obj)); } @Override public int hashCode() { return hash; } + + private boolean equals(byte[] other) { + if (length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other[i]) { + return false; + } + } + return true; + } + + private boolean equals(ByteArrayLookupKey other) { + if (other.bytes == null || length != other.length) { + return false; + } + for (int i = 0; i < length; i++) { + if (bytes[offset + i] != other.bytes[other.offset + i]) { + return false; + } + } + return true; + } } diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java index 89c2db09d426..8248cfcb36cc 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/ByteArrayKeyTest.java @@ -59,6 +59,23 @@ void testReusableMapLookup() { assertThat(lookup.hashCode()).isZero(); } + @Test + void testReusableSliceLookup() { + Map values = new HashMap<>(); + ByteArrayKey key = new ByteArrayKey(new byte[] {1, 2, 3}); + values.put(key, "value"); + ByteArrayLookupKey lookup = new ByteArrayLookupKey(); + + lookup.reset(new byte[] {9, 1, 2, 3, 8}, 1, 3); + assertThat(lookup).isEqualTo(key); + assertThat(key).isEqualTo(lookup); + assertThat(lookup.hashCode()).isEqualTo(key.hashCode()); + assertThat(values.get(lookup)).isEqualTo("value"); + + lookup.clear(); + assertThat(new ByteArrayLookupKey(new byte[] {1, 2, 3})).isNotEqualTo(lookup); + } + @Test void testLookupEqualityLifecycle() { ByteArrayLookupKey first = new ByteArrayLookupKey(new byte[] {1}); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java index e33f2e6f8563..178a4fa25852 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java @@ -213,6 +213,19 @@ public ReusableIdentifier replaceWithPartition(BinaryManifestEntry entry) { return appendEntryFields(entry); } + /** Replaces this encoding with an already serialized identifier. */ + public ReusableIdentifier replace(byte[] value, int offset, int valueLength) { + checkArgument(value != null, "Serialized identifier cannot be null."); + checkArgument( + offset >= 0 && valueLength >= 0 && offset <= value.length - valueLength, + "Identifier byte range is invalid."); + length = 0; + ensureCapacity(valueLength); + System.arraycopy(value, offset, bytes, 0, valueLength); + length = valueLength; + return this; + } + private ReusableIdentifier appendEntryFields(BinaryManifestEntry entry) { putInt(entry.bucket()); BinaryDataFileMeta file = entry.file(); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroBlockReader.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroBlockReader.java new file mode 100644 index 000000000000..144189884709 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestAvroBlockReader.java @@ -0,0 +1,1153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalVector; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.data.variant.Variant; +import org.apache.paimon.format.avro.AvroBlockReader; +import org.apache.paimon.format.avro.AvroFileFormat; +import org.apache.paimon.format.avro.AvroRawBlock; +import org.apache.paimon.format.avro.AvroRecordDecoder; +import org.apache.paimon.format.avro.AvroRecordDecoder.FieldDecoder; +import org.apache.paimon.format.avro.AvroRecordDecoder.FieldType; +import org.apache.paimon.format.avro.AvroRowDatumReader; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.BinaryManifestEntry.Projection; +import org.apache.paimon.memory.MemorySegment; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.IOUtils; + +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DecoderFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** Reader which exposes reusable raw blocks from a Manifest Avro file. */ +public final class ManifestAvroBlockReader implements AutoCloseable { + + private static final String[] TOP_LEVEL_FIELDS = { + ManifestSchemaUtils.FORMAT_IDENTIFIER_FIELD, + ManifestEntry.KIND, + ManifestEntry.PARTITION, + ManifestEntry.BUCKET, + ManifestEntry.TOTAL_BUCKETS, + ManifestEntry.FILE + }; + + private final AvroBlockReader stream; + private final DecoderContext decoderContext; + private final boolean rawBlockCopySupported; + + private long blockOrdinal = -1; + + ManifestAvroBlockReader(InputStream input, AvroFileFormat avroFileFormat) throws IOException { + AvroBlockReader stream = null; + try { + stream = new AvroBlockReader(input); + this.stream = stream; + this.decoderContext = new DecoderContext(stream.createRecordDecoder(), stream.schema()); + this.rawBlockCopySupported = + avroFileFormat.supportsRawBlockCopy( + ManifestEntry.MANIFEST_ROW_TYPE, stream.schema()); + } catch (IOException | RuntimeException | Error failure) { + IOUtils.closeQuietly(stream == null ? input : stream); + throw failure; + } + } + + /** Returns whether another raw Avro block is available. */ + public boolean hasNext() throws IOException { + return stream.hasNextBlock(); + } + + /** Returns the next raw block without decompressing it. */ + public RawBlock next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return new RawBlock( + decoderContext, + rawBlockCopySupported, + stream.nextBorrowedRawBlock(), + ++blockOrdinal); + } + + @Override + public void close() throws IOException { + stream.close(); + } + + /** Borrowed raw block which must be consumed before the enclosing reader advances. */ + public static final class RawBlock { + + private final DecoderContext decoderContext; + private final boolean rawBlockCopySupported; + private final AvroRawBlock block; + private final long blockOrdinal; + private final long blockRecordCount; + + private RawBlock( + DecoderContext decoderContext, + boolean rawBlockCopySupported, + AvroRawBlock block, + long blockOrdinal) { + this.decoderContext = decoderContext; + this.rawBlockCopySupported = rawBlockCopySupported; + this.block = block; + this.blockOrdinal = blockOrdinal; + this.blockRecordCount = block.recordCount(); + } + + /** Lazily decompresses this block and returns an iterator over one reusable row. */ + public RowIterator toRow(Projection projection) throws IOException { + RowType rowType = projection.projectedType(); + ManifestEntryDecoder recordDecoder = decoderContext.recordDecoder(rowType); + + ByteBuffer decompressed = block.decompress(null); + decoderContext.decoder.reset(decompressed); + AvroRowDatumReader materializer = new AvroRowDatumReader(rowType); + materializer.setSchema(decoderContext.encodedSchema); + BlockRow row = recordDecoder.createRow(materializer); + return new RowIterator(blockRecordCount, decoderContext.decoder, recordDecoder, row); + } + + public long blockOrdinal() { + return blockOrdinal; + } + + public long recordCount() { + return blockRecordCount; + } + + public boolean rawBlockCopySupported() { + return rawBlockCopySupported; + } + + public AvroRawBlock encodedBlock() { + return block; + } + } + + /** Decoder state shared by the borrowed blocks produced by one reader. */ + private static final class DecoderContext { + + private final AvroRecordDecoder decoder; + private final Schema encodedSchema; + + private RowType projectedRowType; + private ManifestEntryDecoder recordDecoder; + + private DecoderContext(AvroRecordDecoder decoder, Schema encodedSchema) { + this.decoder = decoder; + this.encodedSchema = encodedSchema; + } + + private ManifestEntryDecoder recordDecoder(RowType rowType) { + if (!rowType.equals(projectedRowType)) { + recordDecoder = new ManifestEntryDecoder(decoder, encodedSchema, rowType); + projectedRowType = rowType; + } + return recordDecoder; + } + } + + /** Iterator over the reusable row decoded from one borrowed block. */ + public static final class RowIterator implements Iterator { + + private final AvroRecordDecoder decoder; + private final ManifestEntryDecoder recordDecoder; + private final BlockRow row; + + private long blockRemaining; + private long blockRecordIndex = -1; + private ByteBuffer encodedRecord; + + private RowIterator( + long recordCount, + AvroRecordDecoder decoder, + ManifestEntryDecoder recordDecoder, + BlockRow row) { + blockRemaining = recordCount; + this.decoder = decoder; + this.recordDecoder = recordDecoder; + this.row = row; + } + + @Override + public boolean hasNext() { + return blockRemaining > 0; + } + + @Override + public InternalRow next() { + if (blockRemaining == 0) { + throw new NoSuchElementException(); + } + try { + row.reset(); + blockRecordIndex++; + int start = decoder.absolutePosition(); + recordDecoder.read(decoder, row); + encodedRecord = decoder.borrowedView(start, decoder.absolutePosition()); + row.replaceEncodedRecord(encodedRecord); + blockRemaining--; + return row; + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to decode projected Manifest Avro record.", e); + } + } + + public long recordIndex() { + checkCurrentRecord(); + return blockRecordIndex; + } + + public ByteBuffer encodedRecord() { + checkCurrentRecord(); + return encodedRecord; + } + + public long longValue(int index) { + checkCurrentRecord(); + return row.longValue(index); + } + + public boolean isNull(int index) { + checkCurrentRecord(); + return row.isNull(index); + } + + public byte[] bytes(int index) { + checkCurrentRecord(); + return row.bytes(index); + } + + public int offset(int index) { + checkCurrentRecord(); + return row.offset(index); + } + + public int length(int index) { + checkCurrentRecord(); + return row.length(index); + } + + private void checkCurrentRecord() { + if (blockRecordIndex < 0 || encodedRecord == null) { + throw new IllegalStateException("No current Manifest Avro record."); + } + } + } + + /** + * Reusable {@link InternalRow} view over values and byte ranges decoded by {@link RowIterator}. + */ + private static final class BlockRow implements InternalRow { + + private final RowType rowType; + private final int[] valueIndexes; + private final boolean[] directFields; + private final BlockRow root; + private final @Nullable BlockRow parent; + private final int parentPosition; + private final int filePosition; + private final @Nullable BlockRow fileRow; + + // The Manifest row and its DataFileMeta row share these reusable slots. Variable-width + // values borrow the decoder's block byte array and only replace their offset and length. + private final long[] longs; + private final int[] offsets; + private final int[] lengths; + private final boolean[] nulls; + private final BinaryString[] stringViews; + private final MemorySegment[] blockSegments; + private final AvroRowDatumReader materializer; + + private byte[] blockBytes; + private InternalRow materializedRow; + private BinaryDecoder materializeDecoder; + private ByteBuffer encodedRecord; + private boolean materialized; + + private RowKind rowKind = RowKind.INSERT; + + private BlockRow( + RowType rowType, + int[] valueIndexes, + boolean[] directFields, + int valueCount, + int filePosition, + @Nullable RowType fileType, + @Nullable int[] fileValueIndexes, + @Nullable boolean[] directFileFields, + AvroRowDatumReader materializer) { + this.rowType = rowType; + this.valueIndexes = valueIndexes; + this.directFields = directFields; + this.root = this; + this.parent = null; + this.parentPosition = -1; + this.filePosition = filePosition; + this.longs = new long[valueCount]; + this.offsets = new int[valueCount]; + this.lengths = new int[valueCount]; + this.nulls = new boolean[valueCount]; + this.stringViews = new BinaryString[valueCount]; + this.blockSegments = new MemorySegment[1]; + this.materializer = materializer; + this.fileRow = + fileType == null + ? null + : new BlockRow( + fileType, + fileValueIndexes, + directFileFields, + this, + this, + filePosition, + materializer); + } + + private BlockRow( + RowType rowType, + int[] valueIndexes, + boolean[] directFields, + BlockRow root, + BlockRow parent, + int parentPosition, + AvroRowDatumReader materializer) { + this.rowType = rowType; + this.valueIndexes = valueIndexes; + this.directFields = directFields; + this.root = root; + this.parent = parent; + this.parentPosition = parentPosition; + this.filePosition = -1; + this.fileRow = null; + this.longs = root.longs; + this.offsets = root.offsets; + this.lengths = root.lengths; + this.nulls = root.nulls; + this.stringViews = root.stringViews; + this.blockSegments = root.blockSegments; + this.materializer = materializer; + } + + private void reset() { + Arrays.fill(nulls, false); + root.materialized = false; + } + + private void replaceEncodedRecord(ByteBuffer encodedRecord) { + root.encodedRecord = encodedRecord; + } + + private boolean directlyDecoded(int pos) { + return directFields[pos]; + } + + private int valueIndex(int pos) { + return valueIndexes[pos]; + } + + private long directLong(int pos) { + return longs[valueIndex(pos)]; + } + + private long longValue(int index) { + return longs[index]; + } + + private boolean isNull(int index) { + return nulls[index]; + } + + private byte[] bytes(int index) { + return root.blockBytes; + } + + private int offset(int index) { + return offsets[index]; + } + + private int length(int index) { + return lengths[index]; + } + + private InternalRow materialized() { + // The manifest run-merge path only uses the direct values above. Keep full Avro + // decoding as a compatibility fallback for uncommon complex InternalRow getters. + if (this != root) { + return parent.materialized().getRow(parentPosition, rowType.getFieldCount()); + } + if (!root.materialized) { + try { + root.materializeDecoder = + DecoderFactory.get() + .binaryDecoder( + root.encodedRecord.array(), + root.encodedRecord.arrayOffset() + + root.encodedRecord.position(), + root.encodedRecord.remaining(), + root.materializeDecoder); + root.materializedRow = + materializer.read(root.materializedRow, root.materializeDecoder); + root.materializedRow.setRowKind(rowKind); + root.materialized = true; + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to materialize projected Manifest Avro record.", e); + } + } + return root.materializedRow; + } + + @Override + public int getFieldCount() { + return rowType.getFieldCount(); + } + + @Override + public RowKind getRowKind() { + return root.rowKind; + } + + @Override + public void setRowKind(RowKind kind) { + root.rowKind = kind; + if (root.materialized) { + root.materializedRow.setRowKind(kind); + } + } + + @Override + public boolean isNullAt(int pos) { + if (fileRow != null && pos == filePosition) { + return false; + } + int index = valueIndex(pos); + return index < 0 ? materialized().isNullAt(pos) : nulls[index]; + } + + @Override + public boolean getBoolean(int pos) { + return directlyDecoded(pos) ? directLong(pos) != 0 : materialized().getBoolean(pos); + } + + @Override + public byte getByte(int pos) { + return directlyDecoded(pos) ? (byte) directLong(pos) : materialized().getByte(pos); + } + + @Override + public short getShort(int pos) { + return directlyDecoded(pos) ? (short) directLong(pos) : materialized().getShort(pos); + } + + @Override + public int getInt(int pos) { + return directlyDecoded(pos) ? (int) directLong(pos) : materialized().getInt(pos); + } + + @Override + public long getLong(int pos) { + return directlyDecoded(pos) ? directLong(pos) : materialized().getLong(pos); + } + + @Override + public float getFloat(int pos) { + return materialized().getFloat(pos); + } + + @Override + public double getDouble(int pos) { + return materialized().getDouble(pos); + } + + @Override + public BinaryString getString(int pos) { + if (!directlyDecoded(pos)) { + return materialized().getString(pos); + } + int index = valueIndex(pos); + BinaryString view = stringViews[index]; + if (view == null) { + view = new BinaryString(blockSegments, offsets[index], lengths[index]); + stringViews[index] = view; + } else { + view.pointTo(blockSegments, offsets[index], lengths[index]); + } + return view; + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + return materialized().getDecimal(pos, precision, scale); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + return materialized().getTimestamp(pos, precision); + } + + @Override + public byte[] getBinary(int pos) { + if (!directlyDecoded(pos)) { + return materialized().getBinary(pos); + } + int index = valueIndex(pos); + return Arrays.copyOfRange( + root.blockBytes, offsets[index], offsets[index] + lengths[index]); + } + + @Override + public Variant getVariant(int pos) { + return materialized().getVariant(pos); + } + + @Override + public Blob getBlob(int pos) { + return materialized().getBlob(pos); + } + + @Override + public InternalArray getArray(int pos) { + return materialized().getArray(pos); + } + + @Override + public InternalVector getVector(int pos) { + return materialized().getVector(pos); + } + + @Override + public InternalMap getMap(int pos) { + return materialized().getMap(pos); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + if (fileRow == null || pos != filePosition || fileRow.getFieldCount() != numFields) { + return materialized().getRow(pos, numFields); + } + return fileRow; + } + } + + private static final class ManifestEntryDecoder { + + private final RowType projectedType; + private final int[] valueIndexes; + private final boolean[] directFields; + private final int valueCount; + private final int versionIndex; + private final int kindIndex; + private final int partitionIndex; + private final int bucketIndex; + private final int totalBucketsIndex; + private final int filePosition; + private final @Nullable RowType projectedFileType; + private final @Nullable int[] fileValueIndexes; + private final @Nullable boolean[] directFileFields; + private final @Nullable DataFileDecoder fileDecoder; + private final @Nullable FieldDecoder fileSkipper; + + private ManifestEntryDecoder( + AvroRecordDecoder decoder, Schema encodedSchema, RowType projectedType) { + validateTopLevelSchema(decoder); + this.projectedType = projectedType; + this.valueIndexes = new int[projectedType.getFieldCount()]; + this.directFields = new boolean[projectedType.getFieldCount()]; + Arrays.fill(valueIndexes, -1); + + int nextIndex = 0; + int nestedFilePosition = -1; + RowType nestedFileType = null; + int[] nestedFileValueIndexes = null; + boolean[] nestedDirectFileFields = null; + for (int position = 0; position < projectedType.getFieldCount(); position++) { + String fieldName = projectedType.getFieldNames().get(position); + if (ManifestEntry.FILE.equals(fieldName)) { + nestedFilePosition = position; + nestedFileType = (RowType) projectedType.getTypeAt(position); + nestedFileValueIndexes = new int[nestedFileType.getFieldCount()]; + nestedDirectFileFields = new boolean[nestedFileType.getFieldCount()]; + for (int nestedPosition = 0; + nestedPosition < nestedFileValueIndexes.length; + nestedPosition++) { + nestedFileValueIndexes[nestedPosition] = nextIndex++; + String nestedName = nestedFileType.getFieldNames().get(nestedPosition); + nestedDirectFileFields[nestedPosition] = + DataFileMeta.FILE_NAME.equals(nestedName) + || DataFileMeta.FILE_SIZE.equals(nestedName) + || DataFileMeta.ROW_COUNT.equals(nestedName) + || DataFileMeta.MIN_KEY.equals(nestedName) + || DataFileMeta.MAX_KEY.equals(nestedName) + || DataFileMeta.MIN_SEQUENCE_NUMBER.equals(nestedName) + || DataFileMeta.MAX_SEQUENCE_NUMBER.equals(nestedName) + || DataFileMeta.SCHEMA_ID.equals(nestedName) + || DataFileMeta.LEVEL.equals(nestedName) + || DataFileMeta.DELETE_ROW_COUNT.equals(nestedName) + || DataFileMeta.EMBEDDED_FILE_INDEX.equals(nestedName) + || DataFileMeta.FILE_SOURCE.equals(nestedName) + || DataFileMeta.EXTERNAL_PATH.equals(nestedName) + || DataFileMeta.FIRST_ROW_ID.equals(nestedName); + } + } else { + valueIndexes[position] = nextIndex++; + directFields[position] = true; + } + } + this.valueCount = nextIndex; + this.filePosition = nestedFilePosition; + this.projectedFileType = nestedFileType; + this.fileValueIndexes = nestedFileValueIndexes; + this.directFileFields = nestedDirectFileFields; + this.versionIndex = valueIndex(ManifestSchemaUtils.FORMAT_IDENTIFIER_FIELD); + this.kindIndex = valueIndex(ManifestEntry.KIND); + this.partitionIndex = valueIndex(ManifestEntry.PARTITION); + this.bucketIndex = valueIndex(ManifestEntry.BUCKET); + this.totalBucketsIndex = valueIndex(ManifestEntry.TOTAL_BUCKETS); + + Schema fileSchema = manifestRecordSchema(encodedSchema).getFields().get(5).schema(); + if (fileSchema.getType() != Schema.Type.RECORD) { + throw mismatch(ManifestEntry.FILE, fileSchema); + } + if (projectedFileType == null) { + this.fileDecoder = null; + this.fileSkipper = decoder.createFieldDecoder(fileSchema, null); + } else { + this.fileDecoder = + new DataFileDecoder( + decoder, fileSchema, projectedFileType, fileValueIndexes); + this.fileSkipper = null; + } + } + + private BlockRow createRow(AvroRowDatumReader materializer) { + return new BlockRow( + projectedType, + valueIndexes, + directFields, + valueCount, + filePosition, + projectedFileType, + fileValueIndexes, + directFileFields, + materializer); + } + + private void read(AvroRecordDecoder decoder, BlockRow row) throws IOException { + if (!decoder.readRecordStart()) { + throw new IOException("Unexpected null or non-record Manifest Avro value."); + } + + int version = decoder.readInt(); + ManifestEntrySerializer.checkFormatIdentifier(version); + if (versionIndex >= 0) { + row.longs[versionIndex] = version; + } + + int kind = decoder.readInt(); + if (kindIndex >= 0) { + row.longs[kindIndex] = kind; + } + + if (partitionIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, partitionIndex, decoder.readBytesView()); + } + + int bucket = decoder.readInt(); + if (bucketIndex >= 0) { + row.longs[bucketIndex] = bucket; + } + + int totalBuckets = decoder.readInt(); + if (totalBucketsIndex >= 0) { + row.longs[totalBucketsIndex] = totalBuckets; + } + + if (fileDecoder == null) { + fileSkipper.skip(decoder); + } else { + fileDecoder.read(decoder, row); + } + } + + private int valueIndex(String fieldName) { + int position = projectedType.getFieldIndex(fieldName); + return position < 0 ? -1 : valueIndexes[position]; + } + + private static void validateTopLevelSchema(AvroRecordDecoder decoder) { + if (decoder.fieldCount() != TOP_LEVEL_FIELDS.length) { + throw new IllegalArgumentException( + String.format( + "Manifest Avro schema has %s top-level fields, expected %s.", + decoder.fieldCount(), TOP_LEVEL_FIELDS.length)); + } + + FieldType[] expectedTypes = { + FieldType.INT, + FieldType.INT, + FieldType.BYTES, + FieldType.INT, + FieldType.INT, + FieldType.RECORD + }; + for (int i = 0; i < TOP_LEVEL_FIELDS.length; i++) { + String actualName = decoder.fieldName(i); + String expectedName = TOP_LEVEL_FIELDS[i]; + if (!expectedName.equals(actualName)) { + throw new IllegalArgumentException( + String.format( + "Unexpected Manifest Avro field at position %s: expected %s but found %s.", + i, expectedName, actualName)); + } + FieldType actualType = decoder.fieldType(i); + if (actualType != expectedTypes[i]) { + throw new IllegalArgumentException( + String.format( + "Unexpected Manifest Avro type for field %s: expected %s but found %s.", + actualName, expectedTypes[i], actualType)); + } + } + } + } + + private static final class DataFileDecoder { + + private static final int MINIMUM_FIELD_COUNT = 13; + + private final int writerFieldCount; + private final int fileNameIndex; + private final int fileSizeIndex; + private final int rowCountIndex; + private final int minKeyIndex; + private final int maxKeyIndex; + private final int keyStatsIndex; + private final int valueStatsIndex; + private final int minSequenceNumberIndex; + private final int maxSequenceNumberIndex; + private final int schemaIdIndex; + private final int levelIndex; + private final int extraFilesIndex; + private final int creationTimeIndex; + private final int deleteRowCountIndex; + private final int embeddedFileIndex; + private final int fileSourceIndex; + private final int valueStatsColsIndex; + private final int externalPathIndex; + private final int firstRowIdIndex; + private final int writeColsIndex; + private final int[] missingValueIndexes; + + private final FieldDecoder keyStatsSkipper; + private final FieldDecoder valueStatsSkipper; + private final FieldDecoder extraFilesSkipper; + private final @Nullable FieldDecoder valueStatsColsSkipper; + private final @Nullable FieldDecoder writeColsSkipper; + + private DataFileDecoder( + AvroRecordDecoder decoder, + Schema fileSchema, + RowType projectedType, + int[] projectedValueIndexes) { + writerFieldCount = fileSchema.getFields().size(); + if (writerFieldCount < MINIMUM_FIELD_COUNT + || writerFieldCount > DataFileMeta.SCHEMA.getFieldCount()) { + throw new IllegalArgumentException( + String.format( + "Unsupported Manifest Avro data file field count %s.", + writerFieldCount)); + } + for (int position = 0; position < writerFieldCount; position++) { + Schema.Field actual = fileSchema.getFields().get(position); + String expected = DataFileMeta.SCHEMA.getField(position).name(); + if (!actual.name().equals(expected)) { + throw new IllegalArgumentException( + String.format( + "Unexpected Manifest Avro data file field at position %s: expected %s but found %s.", + position, expected, actual.name())); + } + } + + fileNameIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.FILE_NAME); + fileSizeIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.FILE_SIZE); + rowCountIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.ROW_COUNT); + minKeyIndex = valueIndex(projectedType, projectedValueIndexes, DataFileMeta.MIN_KEY); + maxKeyIndex = valueIndex(projectedType, projectedValueIndexes, DataFileMeta.MAX_KEY); + keyStatsIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.KEY_STATS); + valueStatsIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.VALUE_STATS); + minSequenceNumberIndex = + valueIndex( + projectedType, projectedValueIndexes, DataFileMeta.MIN_SEQUENCE_NUMBER); + maxSequenceNumberIndex = + valueIndex( + projectedType, projectedValueIndexes, DataFileMeta.MAX_SEQUENCE_NUMBER); + schemaIdIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.SCHEMA_ID); + levelIndex = valueIndex(projectedType, projectedValueIndexes, DataFileMeta.LEVEL); + extraFilesIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.EXTRA_FILES); + creationTimeIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.CREATION_TIME); + deleteRowCountIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.DELETE_ROW_COUNT); + embeddedFileIndex = + valueIndex( + projectedType, projectedValueIndexes, DataFileMeta.EMBEDDED_FILE_INDEX); + fileSourceIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.FILE_SOURCE); + valueStatsColsIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.VALUE_STATS_COLS); + externalPathIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.EXTERNAL_PATH); + firstRowIdIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.FIRST_ROW_ID); + writeColsIndex = + valueIndex(projectedType, projectedValueIndexes, DataFileMeta.WRITE_COLS); + + keyStatsSkipper = + decoder.createFieldDecoder(fileSchema.getFields().get(5).schema(), null); + valueStatsSkipper = + decoder.createFieldDecoder(fileSchema.getFields().get(6).schema(), null); + extraFilesSkipper = + decoder.createFieldDecoder(fileSchema.getFields().get(11).schema(), null); + valueStatsColsSkipper = + writerFieldCount <= 16 + ? null + : decoder.createFieldDecoder( + fileSchema.getFields().get(16).schema().getTypes().get(1), + null); + writeColsSkipper = + writerFieldCount <= 19 + ? null + : decoder.createFieldDecoder( + fileSchema.getFields().get(19).schema().getTypes().get(1), + null); + + int missingCount = 0; + for (int projectedPosition = 0; + projectedPosition < projectedType.getFieldCount(); + projectedPosition++) { + String fieldName = projectedType.getFieldNames().get(projectedPosition); + if (DataFileMeta.SCHEMA.getFieldIndex(fieldName) >= writerFieldCount) { + missingCount++; + } + } + missingValueIndexes = new int[missingCount]; + int missingPosition = 0; + for (int projectedPosition = 0; + projectedPosition < projectedType.getFieldCount(); + projectedPosition++) { + String fieldName = projectedType.getFieldNames().get(projectedPosition); + if (DataFileMeta.SCHEMA.getFieldIndex(fieldName) >= writerFieldCount) { + missingValueIndexes[missingPosition++] = + projectedValueIndexes[projectedPosition]; + } + } + } + + private void read(AvroRecordDecoder decoder, BlockRow row) throws IOException { + for (int index : missingValueIndexes) { + row.nulls[index] = true; + } + + if (fileNameIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, fileNameIndex, decoder.readBytesView()); + } + + long fileSize = decoder.readLong(); + if (fileSizeIndex >= 0) { + row.longs[fileSizeIndex] = fileSize; + } + + long rowCount = decoder.readLong(); + if (rowCountIndex >= 0) { + row.longs[rowCountIndex] = rowCount; + } + + if (minKeyIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, minKeyIndex, decoder.readBytesView()); + } + + if (maxKeyIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, maxKeyIndex, decoder.readBytesView()); + } + + if (keyStatsIndex < 0) { + keyStatsSkipper.skip(decoder); + } else { + int start = decoder.absolutePosition(); + keyStatsSkipper.skip(decoder); + captureRaw(decoder, row, keyStatsIndex, start); + } + + if (valueStatsIndex < 0) { + valueStatsSkipper.skip(decoder); + } else { + int start = decoder.absolutePosition(); + valueStatsSkipper.skip(decoder); + captureRaw(decoder, row, valueStatsIndex, start); + } + + long minSequenceNumber = decoder.readLong(); + if (minSequenceNumberIndex >= 0) { + row.longs[minSequenceNumberIndex] = minSequenceNumber; + } + + long maxSequenceNumber = decoder.readLong(); + if (maxSequenceNumberIndex >= 0) { + row.longs[maxSequenceNumberIndex] = maxSequenceNumber; + } + + long schemaId = decoder.readLong(); + if (schemaIdIndex >= 0) { + row.longs[schemaIdIndex] = schemaId; + } + + int level = decoder.readInt(); + if (levelIndex >= 0) { + row.longs[levelIndex] = level; + } + + if (extraFilesIndex < 0) { + extraFilesSkipper.skip(decoder); + } else { + int start = decoder.absolutePosition(); + extraFilesSkipper.skip(decoder); + captureRaw(decoder, row, extraFilesIndex, start); + } + + int creationTimeStart = creationTimeIndex < 0 ? -1 : decoder.absolutePosition(); + int creationTimeBranch = decoder.readIndex(); + if (creationTimeBranch == 0) { + if (creationTimeIndex >= 0) { + row.nulls[creationTimeIndex] = true; + } + } else if (creationTimeBranch == 1) { + decoder.readLong(); + } else { + throw new IOException( + "Invalid nullable creation time union branch " + creationTimeBranch); + } + if (creationTimeIndex >= 0) { + captureRaw(decoder, row, creationTimeIndex, creationTimeStart); + } + + if (writerFieldCount > 13) { + int branch = decoder.readIndex(); + if (branch == 0) { + if (deleteRowCountIndex >= 0) { + row.nulls[deleteRowCountIndex] = true; + } + } else if (branch == 1) { + long deleteRowCount = decoder.readLong(); + if (deleteRowCountIndex >= 0) { + row.longs[deleteRowCountIndex] = deleteRowCount; + } + } else { + throw new IOException( + "Invalid nullable delete row count union branch " + branch); + } + } + + if (writerFieldCount > 14) { + int branch = decoder.readIndex(); + if (branch == 0) { + if (embeddedFileIndex >= 0) { + row.nulls[embeddedFileIndex] = true; + } + } else if (branch == 1) { + if (embeddedFileIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, embeddedFileIndex, decoder.readBytesView()); + } + } else { + throw new IOException( + "Invalid nullable embedded file index union branch " + branch); + } + } + + if (writerFieldCount > 15) { + int branch = decoder.readIndex(); + if (branch == 0) { + if (fileSourceIndex >= 0) { + row.nulls[fileSourceIndex] = true; + } + } else if (branch == 1) { + int fileSource = decoder.readInt(); + if (fileSourceIndex >= 0) { + row.longs[fileSourceIndex] = fileSource; + } + } else { + throw new IOException("Invalid nullable file source union branch " + branch); + } + } + + if (writerFieldCount > 16) { + int start = valueStatsColsIndex < 0 ? -1 : decoder.absolutePosition(); + int branch = decoder.readIndex(); + if (branch == 0) { + if (valueStatsColsIndex >= 0) { + row.nulls[valueStatsColsIndex] = true; + } + } else if (branch == 1) { + valueStatsColsSkipper.skip(decoder); + } else { + throw new IOException( + "Invalid nullable value stats columns union branch " + branch); + } + if (valueStatsColsIndex >= 0) { + captureRaw(decoder, row, valueStatsColsIndex, start); + } + } + + if (writerFieldCount > 17) { + int branch = decoder.readIndex(); + if (branch == 0) { + if (externalPathIndex >= 0) { + row.nulls[externalPathIndex] = true; + } + } else if (branch == 1) { + if (externalPathIndex < 0) { + decoder.skipBytes(); + } else { + capture(row, externalPathIndex, decoder.readBytesView()); + } + } else { + throw new IOException("Invalid nullable external path union branch " + branch); + } + } + + if (writerFieldCount > 18) { + int branch = decoder.readIndex(); + if (branch == 0) { + if (firstRowIdIndex >= 0) { + row.nulls[firstRowIdIndex] = true; + } + } else if (branch == 1) { + long firstRowId = decoder.readLong(); + if (firstRowIdIndex >= 0) { + row.longs[firstRowIdIndex] = firstRowId; + } + } else { + throw new IOException("Invalid nullable first row id union branch " + branch); + } + } + + if (writerFieldCount > 19) { + int start = writeColsIndex < 0 ? -1 : decoder.absolutePosition(); + int branch = decoder.readIndex(); + if (branch == 0) { + if (writeColsIndex >= 0) { + row.nulls[writeColsIndex] = true; + } + } else if (branch == 1) { + writeColsSkipper.skip(decoder); + } else { + throw new IOException("Invalid nullable write columns union branch " + branch); + } + if (writeColsIndex >= 0) { + captureRaw(decoder, row, writeColsIndex, start); + } + } + } + + private static int valueIndex( + RowType projectedType, int[] projectedValueIndexes, String fieldName) { + int position = projectedType.getFieldIndex(fieldName); + return position < 0 ? -1 : projectedValueIndexes[position]; + } + } + + private static void captureRaw(AvroRecordDecoder decoder, BlockRow row, int index, int start) + throws IOException { + capture(row, index, decoder.borrowedView(start, decoder.absolutePosition())); + } + + private static void capture(BlockRow row, int index, ByteBuffer bytes) { + if (row.root.blockBytes != bytes.array()) { + row.root.blockBytes = bytes.array(); + row.root.blockSegments[0] = MemorySegment.wrap(bytes.array()); + } + row.offsets[index] = bytes.arrayOffset() + bytes.position(); + row.lengths[index] = bytes.remaining(); + } + + private static Schema manifestRecordSchema(Schema schema) { + if (schema.getType() == Schema.Type.RECORD) { + return schema; + } + if (schema.getType() == Schema.Type.UNION) { + Schema record = null; + for (Schema branch : schema.getTypes()) { + if (branch.getType() == Schema.Type.RECORD) { + if (record != null) { + throw new IllegalArgumentException( + "Manifest Avro union contains multiple record branches."); + } + record = branch; + } + } + if (record != null) { + return record; + } + } + throw new IllegalArgumentException("Manifest Avro schema is not a record or record union."); + } + + private static IllegalArgumentException mismatch(String path, Schema schema) { + return new IllegalArgumentException( + "Projected field " + path + " is incompatible with Avro schema " + schema); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 45e0a743785a..bed2e9ffc862 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -19,12 +19,18 @@ package org.apache.paimon.manifest; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsCollector; +import org.apache.paimon.format.avro.AvroBlockWriter; +import org.apache.paimon.format.avro.AvroFileFormat; +import org.apache.paimon.format.avro.AvroRawBlock; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.io.RollingFileWriterImpl; @@ -39,6 +45,7 @@ import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.FileUtils; import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.ObjectsFile; import org.apache.paimon.utils.PathFactory; import org.apache.paimon.utils.SegmentsCache; @@ -47,9 +54,12 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.function.Function; /** @@ -62,6 +72,7 @@ public class ManifestFile extends ObjectsFile { private final SchemaManager schemaManager; private final RowType partitionType; + private final AvroFileFormat avroFileFormat; private final FormatWriterFactory writerFactory; private final long suggestedFileSize; @@ -69,6 +80,7 @@ private ManifestFile( FileIO fileIO, SchemaManager schemaManager, RowType partitionType, + AvroFileFormat avroFileFormat, ManifestEntrySerializer serializer, FormatWriterFactory writerFactory, String compression, @@ -88,6 +100,7 @@ private ManifestFile( cache); this.schemaManager = schemaManager; this.partitionType = partitionType; + this.avroFileFormat = avroFileFormat; this.writerFactory = writerFactory; this.suggestedFileSize = suggestedFileSize; } @@ -209,6 +222,90 @@ private static CloseableIterator createManifestIterator( } } + /** Opens a low-allocation reader for the encoded manifest fields needed by run merge. */ + public ManifestAvroBlockReader scanForRunMerge(String fileName, @Nullable Long fileSize) { + try { + return new ManifestAvroBlockReader( + fileIO.newInputStream(pathFactory.toPath(fileName)), avroFileFormat); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest file " + fileName, e); + } + } + + /** Reusable statistics needed when an encoded manifest record is copied directly. */ + public static final class EncodedManifestEntry { + + private byte kind; + private BinaryRow partition; + private int bucket; + private int level; + private long schemaId; + private long firstRowId; + private long rowCount; + + public EncodedManifestEntry replace( + byte kind, + BinaryRow partition, + int bucket, + int level, + long schemaId, + long firstRowId, + long rowCount) { + this.kind = kind; + this.partition = partition; + this.bucket = bucket; + this.level = level; + this.schemaId = schemaId; + this.firstRowId = firstRowId; + this.rowCount = rowCount; + return this; + } + } + + /** Aggregate statistics for an encoded Avro block copied without decompression. */ + public static final class EncodedManifestBlock { + + private final long addedFiles; + private final long schemaId; + private final int minBucket; + private final int maxBucket; + private final int minLevel; + private final int maxLevel; + private final long minRowId; + private final long maxRowId; + private final @Nullable BinaryRow nullPartition; + private final long nullPartitionCount; + private final @Nullable BinaryRow minNonNullPartition; + private final @Nullable BinaryRow maxNonNullPartition; + + public EncodedManifestBlock( + long addedFiles, + long schemaId, + int minBucket, + int maxBucket, + int minLevel, + int maxLevel, + long minRowId, + long maxRowId, + @Nullable BinaryRow nullPartition, + long nullPartitionCount, + @Nullable BinaryRow minNonNullPartition, + @Nullable BinaryRow maxNonNullPartition) { + this.addedFiles = addedFiles; + this.schemaId = schemaId; + this.minBucket = minBucket; + this.maxBucket = maxBucket; + this.minLevel = minLevel; + this.maxLevel = maxLevel; + this.minRowId = minRowId; + this.maxRowId = maxRowId; + this.nullPartition = nullPartition; + this.nullPartitionCount = nullPartitionCount; + this.minNonNullPartition = minNonNullPartition; + this.maxNonNullPartition = maxNonNullPartition; + } + } + @VisibleForTesting public long suggestedFileSize() { return suggestedFileSize; @@ -283,6 +380,11 @@ public ManifestEntryWriter createManifestEntryWriter(Path manifestPath) { return new ManifestEntryWriter(writerFactory, manifestPath, compression); } + /** Creates the Manifest-specific writer used by the primitive Avro merge path. */ + public PrimitiveManifestRollingWriter createPrimitiveRollingWriter() { + return new PrimitiveManifestRollingWriter(); + } + /** Writer for {@link ManifestEntry}. */ public class ManifestEntryWriter extends SingleFileWriter { @@ -318,6 +420,10 @@ public void write(ManifestEntry entry) throws IOException { super.write(entry); } + collectStats(entry); + } + + private void collectStats(ManifestEntry entry) { switch (entry.kind()) { case ADD: numAddedFiles++; @@ -365,6 +471,376 @@ public ManifestFileMeta result() throws IOException { } } + /** + * Hard-coded rolling writer for Manifest Avro records. + * + *

This deliberately does not extend the generic rolling-writer abstractions: encoded Avro + * records and blocks are an implementation detail of manifest run merging. + */ + public final class PrimitiveManifestRollingWriter implements AutoCloseable { + + private final List results = new ArrayList<>(); + private final List completedPaths = new ArrayList<>(); + private @Nullable PrimitiveManifestFileWriter currentWriter; + private long recordCount; + private boolean closed; + + public void write(ManifestEntry entry) throws IOException { + try { + currentWriter().write(entry); + afterWrite(1, false); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + public void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadata) + throws IOException { + try { + currentWriter().writeEncoded(encodedRecord, metadata); + afterWrite(1, false); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + public void writeEncodedBlock( + AvroRawBlock block, EncodedManifestBlock metadata, long blockRecordCount) + throws IOException { + if (blockRecordCount != block.recordCount()) { + throw new IllegalArgumentException( + String.format( + "Manifest block record count mismatch: expected %s, actual %s.", + blockRecordCount, block.recordCount())); + } + try { + currentWriter().writeEncodedBlock(block, metadata); + afterWrite(blockRecordCount, true); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } + } + + private PrimitiveManifestFileWriter currentWriter() { + if (closed) { + throw new IllegalStateException("Manifest writer has already closed."); + } + if (currentWriter == null) { + currentWriter = new PrimitiveManifestFileWriter(pathFactory.newPath()); + } + return currentWriter; + } + + private void afterWrite(long addedRecords, boolean forceSizeCheck) throws IOException { + recordCount = Math.addExact(recordCount, addedRecords); + if (currentWriter.reachTargetSize( + forceSizeCheck || recordCount % RollingFileWriter.CHECK_ROLLING_RECORD_CNT == 0, + suggestedFileSize)) { + closeCurrentWriter(); + } + } + + private void closeCurrentWriter() throws IOException { + if (currentWriter == null) { + return; + } + currentWriter.close(); + ManifestFileMeta result = currentWriter.result(); + completedPaths.add(currentWriter.path); + results.add(result); + currentWriter = null; + } + + public long recordCount() { + return recordCount; + } + + public List result() { + if (!closed) { + throw new IllegalStateException( + "Cannot access manifest results before closing the writer."); + } + return results; + } + + public void abort() { + if (currentWriter != null) { + currentWriter.abort(); + currentWriter = null; + } + for (Path path : completedPaths) { + fileIO.deleteQuietly(path); + } + completedPaths.clear(); + results.clear(); + closed = true; + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + try { + closeCurrentWriter(); + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } finally { + closed = true; + } + } + } + + /** Single-file counterpart of {@link PrimitiveManifestRollingWriter}. */ + private final class PrimitiveManifestFileWriter { + + private final Path path; + private final SimpleStatsCollector partitionStatsCollector; + private final SimpleStatsConverter partitionStatsSerializer; + private final Map encodedPartitionCounts = new IdentityHashMap<>(); + private final long[] repeatedNullCounts = new long[partitionType.getFieldCount()]; + private @Nullable PositionOutputStream out; + private @Nullable AvroBlockWriter writer; + private @Nullable Long outputBytes; + private long numAddedFiles; + private long numDeletedFiles; + private long schemaId = Long.MIN_VALUE; + private int minBucket = Integer.MAX_VALUE; + private int maxBucket = Integer.MIN_VALUE; + private int minLevel = Integer.MAX_VALUE; + private int maxLevel = Integer.MIN_VALUE; + private @Nullable RowIdStats rowIdStats = new RowIdStats(); + private boolean closed; + + private PrimitiveManifestFileWriter(Path path) { + this.path = path; + this.partitionStatsCollector = new SimpleStatsCollector(partitionType); + this.partitionStatsSerializer = new SimpleStatsConverter(partitionType); + boolean outputCreated = false; + try { + out = fileIO.newOutputStream(path, false); + outputCreated = true; + writer = + avroFileFormat.createBlockWriter( + out, ManifestEntry.MANIFEST_ROW_TYPE, compression); + } catch (IOException failure) { + IOUtils.closeQuietly(writer); + IOUtils.closeQuietly(out); + if (outputCreated) { + fileIO.deleteQuietly(path); + } + throw new UncheckedIOException( + "Failed to create primitive manifest writer for " + path, failure); + } catch (RuntimeException | Error failure) { + IOUtils.closeQuietly(writer); + IOUtils.closeQuietly(out); + if (outputCreated) { + fileIO.deleteQuietly(path); + } + throw failure; + } + } + + private void write(ManifestEntry entry) throws IOException { + ensureOpen(); + writer.addElement( + entry instanceof BinaryManifestEntry + ? ((BinaryManifestEntry) entry).fullRow() + : serializer.toRow(entry)); + collectStats(entry); + } + + private void writeEncoded(ByteBuffer encodedRecord, EncodedManifestEntry metadata) + throws IOException { + ensureOpen(); + writer.addEncoded(encodedRecord); + collectStats(metadata); + addEncodedPartition(metadata.partition, 1); + } + + private void writeEncodedBlock(AvroRawBlock block, EncodedManifestBlock metadata) + throws IOException { + ensureOpen(); + writer.addEncodedBlock(block); + collectStats(metadata); + if (metadata.nullPartitionCount > 0) { + addEncodedPartition(metadata.nullPartition, metadata.nullPartitionCount); + } + if (metadata.minNonNullPartition != null) { + addEncodedPartition(metadata.minNonNullPartition, 1); + if (metadata.maxNonNullPartition != metadata.minNonNullPartition) { + addEncodedPartition(metadata.maxNonNullPartition, 1); + } + } + } + + private void collectStats(ManifestEntry entry) { + switch (entry.kind()) { + case ADD: + numAddedFiles++; + break; + case DELETE: + numDeletedFiles++; + break; + default: + throw new UnsupportedOperationException("Unknown entry kind: " + entry.kind()); + } + schemaId = Math.max(schemaId, entry.file().schemaId()); + minBucket = Math.min(minBucket, entry.bucket()); + maxBucket = Math.max(maxBucket, entry.bucket()); + minLevel = Math.min(minLevel, entry.level()); + maxLevel = Math.max(maxLevel, entry.level()); + if (rowIdStats != null) { + Long firstRowId = entry.file().firstRowId(); + if (firstRowId == null) { + rowIdStats = null; + } else { + rowIdStats.collect(firstRowId, entry.file().rowCount()); + } + } + partitionStatsCollector.collect(entry.partition()); + } + + private void collectStats(EncodedManifestEntry entry) { + switch (FileKind.fromByteValue(entry.kind)) { + case ADD: + numAddedFiles++; + break; + case DELETE: + numDeletedFiles++; + break; + default: + throw new UnsupportedOperationException("Unknown entry kind: " + entry.kind); + } + schemaId = Math.max(schemaId, entry.schemaId); + minBucket = Math.min(minBucket, entry.bucket); + maxBucket = Math.max(maxBucket, entry.bucket); + minLevel = Math.min(minLevel, entry.level); + maxLevel = Math.max(maxLevel, entry.level); + if (rowIdStats != null) { + rowIdStats.collect(entry.firstRowId, entry.rowCount); + } + } + + private void collectStats(EncodedManifestBlock block) { + numAddedFiles = Math.addExact(numAddedFiles, block.addedFiles); + schemaId = Math.max(schemaId, block.schemaId); + minBucket = Math.min(minBucket, block.minBucket); + maxBucket = Math.max(maxBucket, block.maxBucket); + minLevel = Math.min(minLevel, block.minLevel); + maxLevel = Math.max(maxLevel, block.maxLevel); + if (rowIdStats != null) { + rowIdStats.collectRange(block.minRowId, block.maxRowId); + } + } + + private void addEncodedPartition(@Nullable BinaryRow partition, long count) { + if (partition == null || count <= 0) { + return; + } + long[] value = + encodedPartitionCounts.computeIfAbsent(partition, ignored -> new long[1]); + value[0] = Math.addExact(value[0], count); + } + + private SimpleColStats[] partitionStats() { + for (Map.Entry entry : encodedPartitionCounts.entrySet()) { + BinaryRow partition = entry.getKey(); + partitionStatsCollector.collect(partition); + long repeated = entry.getValue()[0] - 1; + if (repeated <= 0) { + continue; + } + for (int field = 0; field < partition.getFieldCount(); field++) { + if (partition.isNullAt(field)) { + repeatedNullCounts[field] = + Math.addExact(repeatedNullCounts[field], repeated); + } + } + } + encodedPartitionCounts.clear(); + SimpleColStats[] stats = partitionStatsCollector.extract(); + for (int field = 0; field < stats.length; field++) { + if (repeatedNullCounts[field] == 0) { + continue; + } + SimpleColStats current = stats[field]; + stats[field] = + new SimpleColStats( + current.min(), + current.max(), + Math.addExact(current.nullCount(), repeatedNullCounts[field])); + } + return stats; + } + + private boolean reachTargetSize(boolean suggestedCheck, long targetSize) + throws IOException { + ensureOpen(); + return writer.reachTargetSize(suggestedCheck, targetSize); + } + + private void ensureOpen() { + if (closed || writer == null) { + throw new IllegalStateException("Manifest writer has already closed."); + } + } + + private void abort() { + IOUtils.closeQuietly(writer); + writer = null; + IOUtils.closeQuietly(out); + out = null; + fileIO.deleteQuietly(path); + closed = true; + } + + private void close() throws IOException { + if (closed) { + return; + } + try { + writer.close(); + writer = null; + out.flush(); + outputBytes = out.getPos(); + out.close(); + out = null; + } catch (IOException | RuntimeException | Error failure) { + abort(); + throw failure; + } finally { + closed = true; + } + } + + private ManifestFileMeta result() { + if (!closed || outputBytes == null) { + throw new IllegalStateException( + "Cannot access manifest result before closing the writer."); + } + return new ManifestFileMeta( + path.getName(), + outputBytes, + numAddedFiles, + numDeletedFiles, + partitionStatsSerializer.toBinaryAllMode(partitionStats()), + numAddedFiles + numDeletedFiles > 0 + ? schemaId + : schemaManager.latest().get().id(), + minBucket, + maxBucket, + minLevel, + maxLevel, + rowIdStats == null ? null : rowIdStats.minRowId, + rowIdStats == null ? null : rowIdStats.maxRowId); + } + } + private static class RowIdStats { private long minRowId = Long.MAX_VALUE; @@ -374,6 +850,11 @@ private void collect(long firstRowId, long rowCount) { minRowId = Math.min(minRowId, firstRowId); maxRowId = Math.max(maxRowId, firstRowId + rowCount - 1); } + + private void collectRange(long minRowId, long maxRowId) { + this.minRowId = Math.min(this.minRowId, minRowId); + this.maxRowId = Math.max(this.maxRowId, maxRowId); + } } /** Creator of {@link ManifestFile}. */ @@ -417,6 +898,7 @@ public ManifestFile create() { fileIO, schemaManager, partitionType, + (AvroFileFormat) fileFormat, new ManifestEntrySerializer(), fileFormat.createWriterFactory(entryType), compression, diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java new file mode 100644 index 000000000000..010cccefe807 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMerge.java @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.ManifestAvroBlockReader; +import org.apache.paimon.manifest.ManifestAvroBlockReader.RawBlock; +import org.apache.paimon.manifest.ManifestAvroBlockReader.RowIterator; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestBlock; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; + +/** Streaming merge of the naturally sorted runs in data-evolution manifest files. */ +final class ManifestEntryRunMerge { + + private static final int FRAGMENTED_RUN_THRESHOLD = 64; + private static final long MAX_IN_MEMORY_FRAGMENTED_ENTRIES = 25_000L; + private static final int MAX_STREAM_CURSORS = 128; + private static final int MAX_STREAM_READ_AMPLIFICATION = 8; + static final int KIND = 0; + static final int PARTITION = 1; + static final int BUCKET = 2; + static final int FILE_NAME = 3; + static final int ROW_COUNT = 4; + static final int LEVEL = 5; + static final int SCHEMA_ID = 6; + static final int FIRST_ROW_ID = 7; + static final int MAX_SEQUENCE_NUMBER = 8; + static final int EXTRA_FILES = 9; + static final int EMBEDDED_FILE_INDEX = 10; + static final int EXTERNAL_PATH = 11; + static final BinaryManifestEntry.Projection ENTRY_LAYOUT = entryLayout(); + + private ManifestEntryRunMerge() {} + + private static BinaryManifestEntry.Projection entryLayout() { + List fields = new ArrayList<>(); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.KIND)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.PARTITION)); + fields.add(ManifestEntry.MANIFEST_ROW_TYPE.getField(ManifestEntry.BUCKET)); + fields.add( + ManifestEntry.MANIFEST_ROW_TYPE + .getField(ManifestEntry.FILE) + .newType( + DataFileMeta.SCHEMA.project( + DataFileMeta.FILE_NAME, + DataFileMeta.ROW_COUNT, + DataFileMeta.LEVEL, + DataFileMeta.SCHEMA_ID, + DataFileMeta.FIRST_ROW_ID, + DataFileMeta.MAX_SEQUENCE_NUMBER, + DataFileMeta.EXTRA_FILES, + DataFileMeta.EMBEDDED_FILE_INDEX, + DataFileMeta.EXTERNAL_PATH))); + return BinaryManifestEntry.Projection.create(new RowType(false, fields)); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge. The caller must + * fall back to the spillable external sorter in that case. + */ + @Nullable + static List sortAndWriteFullEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.Filter filter = + new ManifestEntryRunMergeEntry.Filter(deletedIdentifiers, deletedRowIds); + ManifestEntryRunMergePlan plan = + discoverRuns(section, sortKey, manifestFile, filter, manifestReadParallelism); + if (plan == null) { + return null; + } + return plan.mergeToManifest(sortKey, manifestFile, filter, newFilesForAbort); + } + + /** + * Returns null when the input is too fragmented for a bounded streaming merge or primitive + * manifest reading is unavailable. The caller must fall back to the spillable external sorter. + */ + @Nullable + static Pair, List> sortAndWriteMinorEntries( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + List newFilesForAbort, + @Nullable Integer manifestReadParallelism) + throws Exception { + CompactFileIdentifierSet deletedIdentifiers = new CompactFileIdentifierSet(); + ManifestFileSorter.DeletedRowIdSet deletedRowIds = new ManifestFileSorter.DeletedRowIdSet(); + ManifestEntryRunMergeEntry.Filter.Minor filter = + new ManifestEntryRunMergeEntry.Filter.Minor(deletedIdentifiers, deletedRowIds); + try { + ManifestEntryRunMergePlan plan; + try { + plan = + discoverRuns( + section, sortKey, manifestFile, filter, manifestReadParallelism); + } finally { + deletedRowIds.releaseRangeIndex(); + } + if (plan == null) { + return null; + } + return plan.mergeMinorToManifest( + sortKey, + manifestFile, + filter, + deletedIdentifiers, + deletedRowIds, + newFilesForAbort); + } finally { + deletedIdentifiers.release(); + } + } + + @Nullable + private static ManifestEntryRunMergePlan discoverRuns( + List section, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + @Nullable Integer manifestReadParallelism) + throws Exception { + ManifestEntryRunMergeEntry.PartitionDictionary partitions = + new ManifestEntryRunMergeEntry.PartitionDictionary(sortKey); + List sources = new ArrayList<>(); + int streamCursorCount = 0; + long inMemoryEntries = 0; + List discovered = new ArrayList<>(section.size()); + if (section.size() <= 1 + || manifestReadParallelism == null + || manifestReadParallelism <= 1) { + for (ManifestFileMeta meta : section) { + Discovery.DiscoveredManifest manifest = + discoverManifestRuns(meta, manifestFile, partitions, filter); + if (manifest.requiresExternalSort) { + return null; + } + discovered.add(manifest); + } + } else { + Function> reader = + meta -> { + try { + return Collections.singletonList( + discoverManifestRuns(meta, manifestFile, partitions, filter)); + } catch (Exception e) { + throw new RuntimeException( + "Failed to discover sorted Avro runs in " + meta.fileName(), e); + } + }; + for (Discovery.DiscoveredManifest manifest : + sequentialBatchedExecute(reader, section, manifestReadParallelism)) { + discovered.add(manifest); + } + } + for (int manifestIndex = 0; manifestIndex < section.size(); manifestIndex++) { + ManifestFileMeta meta = section.get(manifestIndex); + Discovery.DiscoveredManifest manifest = discovered.get(manifestIndex); + if (manifest.requiresExternalSort) { + return null; + } + if (manifest.fragmented) { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + inMemoryEntries += entryCount; + if (inMemoryEntries > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return null; + } + sources.add(new ManifestEntryRunMergePlan.Source.FragmentedManifestSpec(meta)); + streamCursorCount++; + } else { + sources.addAll(manifest.runs); + streamCursorCount += manifest.runs.size(); + } + if (streamCursorCount > MAX_STREAM_CURSORS) { + return null; + } + } + partitions.finish(); + for (Discovery.DiscoveredManifest manifest : discovered) { + manifest.finishFiltering(filter); + manifest.updatePartitionRanks(partitions); + } + return new ManifestEntryRunMergePlan(sources, partitions); + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + try (ManifestAvroBlockReader reader = + manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize())) { + return discoverManifestRuns(meta, reader, partitions, filter); + } catch (UnsupportedOperationException unsupported) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + } + + private static Discovery.DiscoveredManifest discoverManifestRuns( + ManifestFileMeta meta, + ManifestAvroBlockReader reader, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) + throws Exception { + List runs = new ArrayList<>(); + List blocks = new ArrayList<>(); + ManifestEntryRunMergeEntry.Key previous = new ManifestEntryRunMergeEntry.Key(); + ManifestEntryRunMergeEntry.Key current = new ManifestEntryRunMergeEntry.Key(); + boolean hasPrevious = false; + long runStart = 0; + long position = 0; + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + boolean fragmented = false; + while (reader.hasNext()) { + RawBlock rawBlock = reader.next(); + RowIterator rows = rawBlock.toRow(ENTRY_LAYOUT); + while (rows.hasNext()) { + rows.next(); + current.replace(rows, partitions); + filter.observe(rows, current); + if (fragmented) { + position++; + continue; + } + if (rows.recordIndex() == 0) { + blocks.add( + new Discovery.BlockInfo( + rawBlock.blockOrdinal(), + position, + rawBlock.rawBlockCopySupported(), + current.stableCopy())); + } + Discovery.BlockInfo block = blocks.get(blocks.size() - 1); + block.collect(rows, current, partitions, filter); + boolean inversion = + hasPrevious && compareDiscoveryKeys(previous, current, partitions) > 0; + if (inversion) { + if (rows.recordIndex() > 0) { + block.sorted = false; + } + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + runStart = position; + if (runs.size() >= FRAGMENTED_RUN_THRESHOLD) { + if (entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES) { + return Discovery.DiscoveredManifest.requiresExternalSort(); + } + fragmented = true; + runs.clear(); + blocks.clear(); + position++; + continue; + } + } + position++; + if (rows.recordIndex() + 1 == rawBlock.recordCount()) { + ManifestEntryRunMergeEntry.Key stableLastKey = current.stableCopy(); + block.finish(position, stableLastKey); + previous.copyFrom(stableLastKey); + } else { + previous.copyFrom(current); + } + hasPrevious = true; + } + } + if (fragmented) { + return Discovery.DiscoveredManifest.fragmented(); + } + if (position > runStart) { + runs.add( + new ManifestEntryRunMergePlan.Source.ManifestRunSpec( + meta, runStart, position, blocks)); + } + if (exceedsStreamingReadAmplification(runs, blocks.size())) { + return entryCount > MAX_IN_MEMORY_FRAGMENTED_ENTRIES + ? Discovery.DiscoveredManifest.requiresExternalSort() + : Discovery.DiscoveredManifest.fragmented(); + } + return Discovery.DiscoveredManifest.runs(runs, blocks); + } + + private static boolean exceedsStreamingReadAmplification( + List runs, int blockCount) { + if (runs.size() <= 1 || blockCount == 0) { + return false; + } + + long prefixBlocksRead = 0; + for (ManifestEntryRunMergePlan.Source.ManifestRunSpec run : runs) { + prefixBlocksRead += run.prefixBlockCount(); + } + return prefixBlocksRead > (long) blockCount * MAX_STREAM_READ_AMPLIFICATION; + } + + private static int compareDiscoveryKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + return compareRemainingKeys( + left, right, partitions.compareIds(left.partitionId, right.partitionId)); + } + + static int compareMergeKeys( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + return compareRemainingKeys( + left, right, Integer.compare(left.partitionRank, right.partitionRank)); + } + + private static int compareRemainingKeys( + ManifestEntryRunMergeEntry.Key left, + ManifestEntryRunMergeEntry.Key right, + int comparison) { + if (comparison == 0) { + comparison = Byte.compare(left.kind, right.kind); + } + if (comparison == 0) { + comparison = Long.compare(left.firstRowId, right.firstRowId); + } + if (comparison == 0) { + comparison = Long.compare(left.rangeEnd, right.rangeEnd); + } + if (comparison == 0) { + comparison = Long.compare(left.reverseSequence, right.reverseSequence); + } + if (comparison == 0) { + comparison = compareBytes(left, right); + } + return comparison; + } + + private static int compareBytes( + ManifestEntryRunMergeEntry.Key left, ManifestEntryRunMergeEntry.Key right) { + int minLength = Math.min(left.fileNameLength, right.fileNameLength); + for (int i = 0; i < minLength; i++) { + int leftByte = left.fileNameBytes[left.fileNameOffset + i] & 0xFF; + int rightByte = right.fileNameBytes[right.fileNameOffset + i] & 0xFF; + if (leftByte != rightByte) { + return leftByte - rightByte; + } + } + return left.fileNameLength - right.fileNameLength; + } + + /** Results and Avro block metadata collected while discovering natural manifest runs. */ + static final class Discovery { + + private Discovery() {} + + static final class DiscoveredManifest { + + final List runs; + final List blocks; + final boolean fragmented; + final boolean requiresExternalSort; + + DiscoveredManifest( + List runs, + List blocks, + boolean fragmented, + boolean requiresExternalSort) { + this.runs = runs; + this.blocks = blocks; + this.fragmented = fragmented; + this.requiresExternalSort = requiresExternalSort; + } + + static DiscoveredManifest runs( + List runs, + List blocks) { + return new DiscoveredManifest(runs, blocks, false, false); + } + + static DiscoveredManifest fragmented() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), true, false); + } + + static DiscoveredManifest requiresExternalSort() { + return new DiscoveredManifest( + Collections.emptyList(), Collections.emptyList(), false, true); + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + for (BlockInfo block : blocks) { + block.updatePartitionRanks(partitions); + } + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + for (BlockInfo block : blocks) { + block.finishFiltering(filter); + } + } + } + + static final class BlockInfo { + + final long ordinal; + final long start; + final ManifestEntryRunMergeEntry.Key firstKey; + boolean eligible; + boolean sorted = true; + long end; + ManifestEntryRunMergeEntry.Key lastKey; + long addedFiles; + long schemaId = Long.MIN_VALUE; + int minBucket = Integer.MAX_VALUE; + int maxBucket = Integer.MIN_VALUE; + int minLevel = Integer.MAX_VALUE; + int maxLevel = Integer.MIN_VALUE; + long minRowId = Long.MAX_VALUE; + long maxRowId = Long.MIN_VALUE; + BinaryRow nullPartition; + long nullPartitionCount; + BinaryRow minNonNullPartition; + BinaryRow maxNonNullPartition; + EncodedManifestBlock metadata; + + BlockInfo( + long ordinal, + long start, + boolean eligible, + ManifestEntryRunMergeEntry.Key firstKey) { + this.ordinal = ordinal; + this.start = start; + this.eligible = eligible; + this.firstKey = firstKey; + } + + void collect( + RowIterator record, + ManifestEntryRunMergeEntry.Key key, + ManifestEntryRunMergeEntry.PartitionDictionary partitions, + ManifestEntryRunMergeEntry.Filter filter) { + BinaryRow partition = partitions.partition(key.partitionId); + eligible &= partition.getFieldCount() == 1 && filter.copyable(record, key); + if (!eligible) { + return; + } + addedFiles++; + schemaId = Math.max(schemaId, record.longValue(SCHEMA_ID)); + int bucket = (int) record.longValue(BUCKET); + minBucket = Math.min(minBucket, bucket); + maxBucket = Math.max(maxBucket, bucket); + int level = (int) record.longValue(LEVEL); + minLevel = Math.min(minLevel, level); + maxLevel = Math.max(maxLevel, level); + minRowId = Math.min(minRowId, key.firstRowId); + maxRowId = Math.max(maxRowId, key.rangeEnd); + if (partition.isNullAt(0)) { + nullPartition = partition; + nullPartitionCount++; + } else { + if (minNonNullPartition == null) { + minNonNullPartition = partition; + } + maxNonNullPartition = partition; + } + } + + void finish(long end, ManifestEntryRunMergeEntry.Key lastKey) { + this.end = end; + this.lastKey = lastKey; + if (eligible && sorted) { + metadata = + new EncodedManifestBlock( + addedFiles, + schemaId, + minBucket, + maxBucket, + minLevel, + maxLevel, + minRowId, + maxRowId, + nullPartition, + nullPartitionCount, + minNonNullPartition, + maxNonNullPartition); + } + } + + boolean copyable(long runStart, long runEnd) { + return metadata != null && start >= runStart && end <= runEnd; + } + + void finishFiltering(ManifestEntryRunMergeEntry.Filter filter) { + if (metadata != null && !filter.copyableAfterDiscovery(minRowId, maxRowId)) { + metadata = null; + } + } + + void updatePartitionRanks(ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + firstKey.partitionRank = partitions.rank(firstKey.partitionId); + lastKey.partitionRank = partitions.rank(lastKey.partitionId); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java new file mode 100644 index 000000000000..d613742e90c0 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergeEntry.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestAvroBlockReader.RowIterator; +import org.apache.paimon.utils.ByteArrayKey; +import org.apache.paimon.utils.ByteArrayLookupKey; +import org.apache.paimon.utils.SerializationUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.paimon.utils.Preconditions.checkState; + +/** Entry-level state shared by manifest run discovery and merge execution. */ +final class ManifestEntryRunMergeEntry { + + private ManifestEntryRunMergeEntry() {} + + static final class Key { + + int partitionId; + int partitionRank; + byte kind; + long firstRowId; + long rangeEnd; + long reverseSequence; + byte[] fileNameBytes; + int fileNameOffset; + int fileNameLength; + + static Key viewOf(BinaryManifestEntry entry, PartitionDictionary partitions) { + Key key = new Key(); + key.replace(entry, partitions); + return key; + } + + void replace(BinaryManifestEntry entry, PartitionDictionary partitions) { + long firstRowId = entry.file().nonNullFirstRowId(); + this.partitionId = partitions.id(entry.partitionBytes()); + this.partitionRank = partitions.rank(partitionId); + this.kind = entry.kind().toByteValue(); + this.firstRowId = firstRowId; + this.rangeEnd = firstRowId + entry.file().rowCount() - 1L; + this.reverseSequence = Long.MAX_VALUE - entry.file().maxSequenceNumber(); + this.fileNameBytes = entry.file().fileNameBinary().toBytes(); + this.fileNameOffset = 0; + this.fileNameLength = fileNameBytes.length; + } + + void replace(RowIterator record, PartitionDictionary partitions) { + checkState( + !record.isNull(ManifestEntryRunMerge.FIRST_ROW_ID), + "First row id should not be null."); + this.partitionId = + partitions.id( + record.bytes(ManifestEntryRunMerge.PARTITION), + record.offset(ManifestEntryRunMerge.PARTITION), + record.length(ManifestEntryRunMerge.PARTITION)); + this.partitionRank = partitions.rank(partitionId); + this.kind = (byte) record.longValue(ManifestEntryRunMerge.KIND); + this.firstRowId = record.longValue(ManifestEntryRunMerge.FIRST_ROW_ID); + this.rangeEnd = firstRowId + record.longValue(ManifestEntryRunMerge.ROW_COUNT) - 1L; + this.reverseSequence = + Long.MAX_VALUE - record.longValue(ManifestEntryRunMerge.MAX_SEQUENCE_NUMBER); + this.fileNameBytes = record.bytes(ManifestEntryRunMerge.FILE_NAME); + this.fileNameOffset = record.offset(ManifestEntryRunMerge.FILE_NAME); + this.fileNameLength = record.length(ManifestEntryRunMerge.FILE_NAME); + } + + void copyFrom(Key key) { + this.partitionId = key.partitionId; + this.partitionRank = key.partitionRank; + this.kind = key.kind; + this.firstRowId = key.firstRowId; + this.rangeEnd = key.rangeEnd; + this.reverseSequence = key.reverseSequence; + this.fileNameBytes = key.fileNameBytes; + this.fileNameOffset = key.fileNameOffset; + this.fileNameLength = key.fileNameLength; + } + + Key stableCopy() { + Key copy = new Key(); + copy.copyFrom(this); + copy.fileNameBytes = + Arrays.copyOfRange( + fileNameBytes, fileNameOffset, fileNameOffset + fileNameLength); + copy.fileNameOffset = 0; + return copy; + } + + void clear() { + fileNameBytes = null; + } + } + + /** Interns variable-width partition bytes once and assigns comparator-compatible ranks. */ + static final class PartitionDictionary { + + final ManifestFileSorter.RowIdEntrySortKey sortKey; + final Map ids = new ConcurrentHashMap<>(); + final ThreadLocal lookup = + ThreadLocal.withInitial(ByteArrayLookupKey::new); + volatile BinaryRow[] partitions = new BinaryRow[16]; + int partitionCount; + int[] ranks; + + PartitionDictionary(ManifestFileSorter.RowIdEntrySortKey sortKey) { + this.sortKey = sortKey; + } + + int id(byte[] bytes) { + return id(bytes, 0, bytes.length); + } + + int id(byte[] bytes, int offset, int length) { + ByteArrayLookupKey lookupKey = lookup.get(); + lookupKey.reset(bytes, offset, length); + try { + Integer existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + synchronized (this) { + existing = ids.get(lookupKey); + if (existing != null) { + return existing; + } + checkState(ranks == null, "Full manifest scan found an unknown partition."); + byte[] canonical = Arrays.copyOfRange(bytes, offset, offset + length); + int id = partitionCount; + if (id == partitions.length) { + partitions = Arrays.copyOf(partitions, partitions.length << 1); + } + partitions[id] = SerializationUtils.deserializeBinaryRow(canonical); + ids.put(new ByteArrayKey(canonical), id); + partitionCount = id + 1; + return id; + } + } finally { + lookupKey.clear(); + } + } + + int compareIds(int left, int right) { + return sortKey.comparePartitions(partitions[left], partitions[right]); + } + + void finish() { + List order = new ArrayList<>(partitionCount); + for (int id = 0; id < partitionCount; id++) { + order.add(id); + } + order.sort((left, right) -> compareIds(left, right)); + ranks = new int[partitionCount]; + int rank = 0; + for (int position = 0; position < order.size(); position++) { + if (position > 0 && compareIds(order.get(position - 1), order.get(position)) != 0) { + rank++; + } + ranks[order.get(position)] = rank; + } + } + + int rank(int id) { + return ranks == null ? 0 : ranks[id]; + } + + BinaryRow partition(int id) { + return partitions[id]; + } + } + + static class Filter { + + final CompactFileIdentifierSet deletedIdentifiers; + final ManifestFileSorter.DeletedRowIdSet deletedRowIds; + final ThreadLocal identifier = + ThreadLocal.withInitial(IdentifierEncoder::new); + + Filter( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + this.deletedIdentifiers = deletedIdentifiers; + this.deletedRowIds = deletedRowIds; + } + + boolean include(BinaryManifestEntry entry) { + return entry.isAdd() && !deletedIdentifiers.contains(entry); + } + + boolean include(RowIterator record, Key key) { + if (key.kind != FileKind.ADD.toByteValue()) { + return false; + } + if (!deletedRowIds.contains(key.firstRowId)) { + return true; + } + + ReusableIdentifier reusable = identifier.get().replace(record); + return !deletedIdentifiers.contains(reusable); + } + + boolean copyable(RowIterator record, Key key) { + return include(record, key); + } + + void observe(RowIterator record, Key key) {} + + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + return true; + } + + ReusableIdentifier identifier(RowIterator record) { + return identifier.get().replace(record); + } + + static final class Minor extends Filter { + + Minor( + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) { + super(deletedIdentifiers, deletedRowIds); + } + + @Override + boolean include(BinaryManifestEntry entry) { + return true; + } + + @Override + boolean include(RowIterator record, Key key) { + return true; + } + + @Override + boolean copyable(RowIterator record, Key key) { + return key.kind == FileKind.ADD.toByteValue(); + } + + @Override + void observe(RowIterator record, Key key) { + if (key.kind != FileKind.DELETE.toByteValue()) { + return; + } + ReusableIdentifier reusable = identifier(record); + synchronized (this) { + deletedIdentifiers.add(reusable); + deletedRowIds.add(key.firstRowId); + } + } + + @Override + boolean copyableAfterDiscovery(long minRowId, long maxRowId) { + // A DELETE preserves the deleted ADD's globally unique first RowID. A range hit may + // be a false positive and only disables block copying; a miss proves the block has + // no deleted ADD. + return !deletedRowIds.intersects(minRowId, maxRowId); + } + } + + private static final class IdentifierEncoder { + + byte[] bytes = new byte[256]; + int length; + final AvroSlice avroSlice = new AvroSlice(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + + ReusableIdentifier replace(RowIterator record) { + length = 0; + putRaw(record, ManifestEntryRunMerge.PARTITION); + putInt((int) record.longValue(ManifestEntryRunMerge.BUCKET)); + putInt((int) record.longValue(ManifestEntryRunMerge.LEVEL)); + putRaw(record, ManifestEntryRunMerge.FILE_NAME); + putStringArray(record, ManifestEntryRunMerge.EXTRA_FILES); + putNullableRaw(record, ManifestEntryRunMerge.EMBEDDED_FILE_INDEX); + putNullableRaw(record, ManifestEntryRunMerge.EXTERNAL_PATH); + return identifier.replace(bytes, 0, length); + } + + void putRaw(RowIterator record, int field) { + int valueLength = record.length(field); + putInt(valueLength); + appendRaw(record.bytes(field), record.offset(field), valueLength); + } + + void putNullableRaw(RowIterator record, int field) { + if (record.isNull(field)) { + putInt(-1); + } else { + putRaw(record, field); + } + } + + void putStringArray(RowIterator record, int field) { + AvroSlice slice = avroSlice.reset(record, field); + long count = 0; + while (true) { + long blockCount = slice.readLong(); + if (blockCount == 0) { + break; + } + if (blockCount < 0) { + blockCount = -blockCount; + slice.readLong(); + } + count = Math.addExact(count, blockCount); + for (long i = 0; i < blockCount; i++) { + slice.skipBytes(); + } + } + checkState(count <= Integer.MAX_VALUE, "Too many extra files in manifest entry."); + putInt((int) count); + + slice.reset(record, field); + while (true) { + long blockCount = slice.readLong(); + if (blockCount == 0) { + break; + } + if (blockCount < 0) { + blockCount = -blockCount; + slice.readLong(); + } + for (long i = 0; i < blockCount; i++) { + int valueLength = slice.readLength(); + putInt(valueLength); + appendRaw(slice.bytes, slice.position, valueLength); + slice.position += valueLength; + } + } + slice.checkConsumed(); + } + + void putInt(int value) { + ensureCapacity(Integer.BYTES); + bytes[length++] = (byte) (value >>> 24); + bytes[length++] = (byte) (value >>> 16); + bytes[length++] = (byte) (value >>> 8); + bytes[length++] = (byte) value; + } + + void appendRaw(byte[] value, int offset, int valueLength) { + checkState( + offset >= 0 && valueLength >= 0 && offset <= value.length - valueLength, + "Identifier byte range is invalid."); + ensureCapacity(valueLength); + System.arraycopy(value, offset, bytes, length, valueLength); + length += valueLength; + } + + void ensureCapacity(int additional) { + int required = Math.addExact(length, additional); + if (required <= bytes.length) { + return; + } + int grown = Math.max(required, bytes.length + (bytes.length >>> 1)); + bytes = Arrays.copyOf(bytes, grown); + } + } + + private static final class AvroSlice { + + byte[] bytes; + int position; + int end; + + AvroSlice reset(RowIterator record, int field) { + bytes = record.bytes(field); + position = record.offset(field); + end = position + record.length(field); + return this; + } + + long readLong() { + long raw = 0; + int shift = 0; + while (position < end && shift < 64) { + int value = bytes[position++] & 0xFF; + raw |= (long) (value & 0x7F) << shift; + if ((value & 0x80) == 0) { + return (raw >>> 1) ^ -(raw & 1L); + } + shift += 7; + } + throw new IllegalStateException("Invalid Avro variable-length integer."); + } + + int readLength() { + long valueLength = readLong(); + checkState( + valueLength >= 0 + && valueLength <= Integer.MAX_VALUE + && valueLength <= end - position, + "Invalid Avro byte sequence length %s.", + valueLength); + return (int) valueLength; + } + + void skipBytes() { + int valueLength = readLength(); + position += valueLength; + } + + void checkConsumed() { + checkState(position == end, "Manifest field contains trailing Avro bytes."); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java new file mode 100644 index 000000000000..2f3dc22dd526 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryRunMergePlan.java @@ -0,0 +1,808 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.format.avro.AvroRawBlock; +import org.apache.paimon.manifest.BinaryManifestEntry; +import org.apache.paimon.manifest.CompactFileIdentifierSet; +import org.apache.paimon.manifest.FileEntry.ReusableIdentifier; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestAvroBlockReader; +import org.apache.paimon.manifest.ManifestAvroBlockReader.RawBlock; +import org.apache.paimon.manifest.ManifestAvroBlockReader.RowIterator; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestBlock; +import org.apache.paimon.manifest.ManifestFile.EncodedManifestEntry; +import org.apache.paimon.manifest.ManifestFile.PrimitiveManifestRollingWriter; +import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.paimon.utils.Preconditions.checkState; + +final class ManifestEntryRunMergePlan { + + final List sources; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + + ManifestEntryRunMergePlan( + List sources, ManifestEntryRunMergeEntry.PartitionDictionary partitions) { + this.sources = sources; + this.partitions = partitions; + } + + List mergeToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Collections.emptyList(); + } + List files = writeSelected(selectionTree, manifestFile); + newFilesForAbort.addAll(files); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + Pair, List> mergeMinorToManifest( + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestFile manifestFile, + ManifestEntryRunMergeEntry.Filter filter, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds, + List newFilesForAbort) + throws Exception { + List cursors = new ArrayList<>(sources.size()); + Exception failure = null; + try { + for (Source.Spec source : sources) { + Cursor cursor = source.open(manifestFile, sortKey, filter, partitions); + cursors.add(cursor); + cursor.advance(); + } + SelectionTree selectionTree = new SelectionTree(cursors); + if (selectionTree.winner() < 0) { + return Pair.of(Collections.emptyList(), Collections.emptyList()); + } + Pair, List> files = + writeMinorSelected( + selectionTree, manifestFile, deletedIdentifiers, deletedRowIds); + newFilesForAbort.addAll(files.getLeft()); + newFilesForAbort.addAll(files.getRight()); + return files; + } catch (Exception e) { + failure = e; + throw e; + } finally { + try { + closeCursors(cursors); + } catch (Exception closeFailure) { + if (failure == null) { + throw closeFailure; + } + failure.addSuppressed(closeFailure); + } + } + } + + static List writeSelected( + SelectionTree selectionTree, ManifestFile manifestFile) throws Exception { + PrimitiveManifestRollingWriter writer = manifestFile.createPrimitiveRollingWriter(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + writer.writeEncodedBlock( + cursor.encodedBlock(), + cursor.blockMetadata(), + cursor.blockRecordCount()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + cursor.materializeCurrent(); + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + selectionTree.update(winner, cursor.advance()); + } + } catch (Exception e) { + failure = e; + } finally { + if (failure != null) { + writer.abort(); + throw failure; + } + writer.close(); + } + return writer.result(); + } + + private static Pair, List> writeMinorSelected( + SelectionTree selectionTree, + ManifestFile manifestFile, + CompactFileIdentifierSet deletedIdentifiers, + ManifestFileSorter.DeletedRowIdSet deletedRowIds) + throws Exception { + PrimitiveManifestRollingWriter addWriter = manifestFile.createPrimitiveRollingWriter(); + PrimitiveManifestRollingWriter deleteWriter = manifestFile.createPrimitiveRollingWriter(); + CompactFileIdentifierSet matchedEntries = new CompactFileIdentifierSet(); + CompactFileIdentifierSet emittedDeletes = new CompactFileIdentifierSet(); + Exception failure = null; + try { + int winner; + while ((winner = selectionTree.winner()) >= 0) { + Cursor cursor = selectionTree.cursor(winner); + if (cursor.hasCopyableBlock() + && selectionTree.blockPrecedesOthers(winner, cursor.blockLastKey())) { + addWriter.writeEncodedBlock( + cursor.encodedBlock(), + cursor.blockMetadata(), + cursor.blockRecordCount()); + selectionTree.update(winner, cursor.advanceAfterBlock()); + continue; + } + + cursor.materializeCurrent(); + if (cursor.key().kind == FileKind.ADD.toByteValue()) { + if (!deletedRowIds.contains(cursor.key().firstRowId)) { + writeCurrent(addWriter, cursor); + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (deletedIdentifiers.contains(identifier)) { + matchedEntries.add(identifier); + } else { + writeCurrent(addWriter, cursor); + } + } + } else { + ReusableIdentifier identifier = cursor.identifier(); + if (!matchedEntries.contains(identifier) + && !emittedDeletes.contains(identifier)) { + emittedDeletes.add(identifier); + writeCurrent(deleteWriter, cursor); + } + } + selectionTree.update(winner, cursor.advance()); + } + addWriter.close(); + deleteWriter.close(); + } catch (Exception e) { + failure = e; + } finally { + matchedEntries.release(); + emittedDeletes.release(); + if (failure != null) { + addWriter.abort(); + deleteWriter.abort(); + throw failure; + } + } + return Pair.of(addWriter.result(), deleteWriter.result()); + } + + private static void writeCurrent(PrimitiveManifestRollingWriter writer, Cursor cursor) + throws Exception { + ByteBuffer encodedRecord = cursor.encodedRecord(); + if (encodedRecord == null) { + writer.write(cursor.current()); + } else { + writer.writeEncoded(encodedRecord, cursor.metadata()); + } + } + + static void closeCursors(List cursors) throws Exception { + Exception failure = null; + for (Cursor cursor : cursors) { + try { + cursor.close(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + + /** Describes the manifest inputs which become cursors when this plan starts executing. */ + static final class Source { + + private Source() {} + + interface Spec { + + Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception; + } + + static final class ManifestRunSpec implements Spec { + + final ManifestFileMeta meta; + final long start; + final long end; + final List blocks; + + ManifestRunSpec( + ManifestFileMeta meta, + long start, + long end, + List blocks) { + this.meta = meta; + this.start = start; + this.end = end; + this.blocks = blocks; + } + + long prefixBlockCount() { + long lastBlockOrdinal = -1; + for (ManifestEntryRunMerge.Discovery.BlockInfo block : blocks) { + if (block.start >= end) { + break; + } + if (block.end > start) { + lastBlockOrdinal = block.ordinal; + } + } + checkState(lastBlockOrdinal >= 0, "Manifest run does not contain an Avro block."); + return lastBlockOrdinal + 1; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new PrimitiveManifestRunCursor( + manifestFile, meta, start, end, blocks, filter, partitions); + } + } + + static final class FragmentedManifestSpec implements Spec { + + final ManifestFileMeta meta; + + FragmentedManifestSpec(ManifestFileMeta meta) { + this.meta = meta; + } + + @Override + public Cursor open( + ManifestFile manifestFile, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + return new InMemoryManifestCursor(manifestFile, meta, sortKey, filter, partitions); + } + } + } + + interface Cursor extends AutoCloseable { + + boolean advance() throws Exception; + + boolean hasCurrent(); + + @Nullable + BinaryManifestEntry current(); + + @Nullable + EncodedManifestEntry metadata(); + + ManifestEntryRunMergeEntry.Key key(); + + @Nullable + ByteBuffer encodedRecord(); + + ReusableIdentifier identifier(); + + default boolean hasCopyableBlock() { + return false; + } + + default ManifestEntryRunMergeEntry.Key blockLastKey() { + throw new UnsupportedOperationException(); + } + + default AvroRawBlock encodedBlock() { + throw new UnsupportedOperationException(); + } + + default EncodedManifestBlock blockMetadata() { + throw new UnsupportedOperationException(); + } + + default long blockRecordCount() { + throw new UnsupportedOperationException(); + } + + default boolean advanceAfterBlock() throws Exception { + throw new UnsupportedOperationException(); + } + + default void materializeCurrent() throws Exception {} + + @Override + void close() throws Exception; + } + + static final class PrimitiveManifestRunCursor implements Cursor { + + final ManifestAvroBlockReader reader; + final ManifestEntryRunMergeEntry.Filter filter; + final ManifestEntryRunMergeEntry.PartitionDictionary partitions; + final ManifestEntryRunMergeEntry.Key key = new ManifestEntryRunMergeEntry.Key(); + final EncodedManifestEntry metadata = new EncodedManifestEntry(); + final List blocks; + final long runStart; + final long runEnd; + int blockIndex; + long nextReaderBlockOrdinal; + long decodedRemaining; + boolean rawBlock; + boolean current; + @Nullable RawBlock currentRawBlock; + @Nullable RowIterator currentRows; + @Nullable ManifestEntryRunMerge.Discovery.BlockInfo currentBlock; + boolean closed; + + PrimitiveManifestRunCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + long start, + long end, + List blocks, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + this.reader = manifestFile.scanForRunMerge(meta.fileName(), meta.fileSize()); + this.filter = filter; + this.partitions = partitions; + this.blocks = blocks; + this.runStart = start; + this.runEnd = end; + try { + while (blockIndex < blocks.size() && blocks.get(blockIndex).end <= start) { + blockIndex++; + } + checkState( + blockIndex < blocks.size(), + "Manifest run starts after the end of the file."); + } catch (Exception e) { + try { + reader.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + } + + @Override + public boolean advance() throws Exception { + current = false; + while (true) { + if (decodedRemaining == 0) { + if (!prepareNextBlock()) { + key.clear(); + close(); + return false; + } + if (rawBlock) { + return true; + } + } + checkState( + currentRows != null && currentRows.hasNext(), + "Manifest block ends before its discovered boundary."); + currentRows.next(); + decodedRemaining--; + key.replace(currentRows, partitions); + if (filter.include(currentRows, key)) { + current = true; + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + (int) currentRows.longValue(ManifestEntryRunMerge.BUCKET), + (int) currentRows.longValue(ManifestEntryRunMerge.LEVEL), + currentRows.longValue(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + currentRows.longValue(ManifestEntryRunMerge.ROW_COUNT)); + return true; + } + } + } + + boolean prepareNextBlock() throws Exception { + rawBlock = false; + current = false; + currentRows = null; + while (blockIndex < blocks.size()) { + ManifestEntryRunMerge.Discovery.BlockInfo info = blocks.get(blockIndex); + if (info.start >= runEnd) { + return false; + } + while (nextReaderBlockOrdinal < info.ordinal) { + checkState(reader.hasNext(), "Manifest block ordinal is missing."); + reader.next(); + nextReaderBlockOrdinal++; + } + checkState(reader.hasNext(), "Manifest run ends after the end of the file."); + currentRawBlock = reader.next(); + nextReaderBlockOrdinal++; + currentBlock = info; + if (info.copyable(runStart, runEnd)) { + rawBlock = true; + key.copyFrom(info.firstKey); + return true; + } + + long overlapStart = Math.max(runStart, info.start); + long overlapEnd = Math.min(runEnd, info.end); + long prefix = overlapStart - info.start; + currentRows = currentRawBlock.toRow(ManifestEntryRunMerge.ENTRY_LAYOUT); + for (long i = 0; i < prefix; i++) { + checkState( + currentRows.hasNext(), + "Manifest run starts after the end of its block."); + currentRows.next(); + } + decodedRemaining = overlapEnd - overlapStart; + blockIndex++; + if (decodedRemaining > 0) { + return true; + } + } + return false; + } + + @Override + public boolean hasCurrent() { + return current || rawBlock; + } + + @Override + public BinaryManifestEntry current() { + return null; + } + + @Override + public EncodedManifestEntry metadata() { + return metadata; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return key; + } + + @Override + public ByteBuffer encodedRecord() { + return current ? currentRows.encodedRecord() : null; + } + + @Override + public ReusableIdentifier identifier() { + checkState(current, "Manifest entry has not been materialized."); + return filter.identifier(currentRows); + } + + @Override + public boolean hasCopyableBlock() { + return rawBlock; + } + + @Override + public ManifestEntryRunMergeEntry.Key blockLastKey() { + return currentBlock.lastKey; + } + + @Override + public AvroRawBlock encodedBlock() { + return currentRawBlock.encodedBlock(); + } + + @Override + public EncodedManifestBlock blockMetadata() { + return currentBlock.metadata; + } + + @Override + public long blockRecordCount() { + return currentBlock.end - currentBlock.start; + } + + @Override + public boolean advanceAfterBlock() throws Exception { + checkState(rawBlock, "There is no raw block to advance."); + rawBlock = false; + currentRawBlock = null; + blockIndex++; + return advance(); + } + + @Override + public void materializeCurrent() throws Exception { + if (!rawBlock) { + return; + } + rawBlock = false; + decodedRemaining = currentBlock.end - currentBlock.start; + checkState(decodedRemaining > 0, "Raw Avro block is empty."); + currentRows = currentRawBlock.toRow(ManifestEntryRunMerge.ENTRY_LAYOUT); + checkState(currentRows.hasNext(), "Manifest block cannot be decompressed."); + currentRows.next(); + decodedRemaining--; + key.replace(currentRows, partitions); + checkState( + filter.include(currentRows, key), + "Copyable manifest block contains a filtered entry."); + current = true; + metadata.replace( + key.kind, + partitions.partition(key.partitionId), + (int) currentRows.longValue(ManifestEntryRunMerge.BUCKET), + (int) currentRows.longValue(ManifestEntryRunMerge.LEVEL), + currentRows.longValue(ManifestEntryRunMerge.SCHEMA_ID), + key.firstRowId, + currentRows.longValue(ManifestEntryRunMerge.ROW_COUNT)); + blockIndex++; + } + + @Override + public void close() throws Exception { + if (closed) { + return; + } + closed = true; + current = false; + currentRawBlock = null; + currentRows = null; + currentBlock = null; + rawBlock = false; + key.clear(); + reader.close(); + } + } + + static final class InMemoryManifestCursor implements Cursor { + + final List entries; + final BinaryManifestEntry current = BinaryManifestEntry.fullProjection().createEntry(); + final ReusableIdentifier identifier = new ReusableIdentifier(); + int position = -1; + + InMemoryManifestCursor( + ManifestFile manifestFile, + ManifestFileMeta meta, + ManifestFileSorter.RowIdEntrySortKey sortKey, + ManifestEntryRunMergeEntry.Filter filter, + ManifestEntryRunMergeEntry.PartitionDictionary partitions) + throws Exception { + long entryCount = meta.numAddedFiles() + meta.numDeletedFiles(); + this.entries = new ArrayList<>((int) entryCount); + InternalRowSerializer serializer = + new InternalRowSerializer(ManifestEntry.MANIFEST_ROW_TYPE); + BinaryManifestEntry view = BinaryManifestEntry.fullProjection().createEntry(); + try (CloseableIterator iterator = + manifestFile.scan( + meta.fileName(), + meta.fileSize(), + BinaryManifestEntry.fullProjection())) { + while (iterator.hasNext()) { + BinaryManifestEntry entry = iterator.next(); + if (!filter.include(entry)) { + continue; + } + BinaryRow row = serializer.toBinaryRow(entry.fullRow()).copy(); + entries.add( + new StoredEntry( + row, + ManifestEntryRunMergeEntry.Key.viewOf( + view.replace(row), partitions))); + } + } + entries.sort( + (left, right) -> ManifestEntryRunMerge.compareMergeKeys(left.key, right.key)); + view.clear(); + } + + @Override + public boolean advance() { + position++; + if (position >= entries.size()) { + current.clear(); + return false; + } + StoredEntry stored = entries.get(position); + current.replace(stored.row); + return true; + } + + @Override + public boolean hasCurrent() { + return position >= 0 && position < entries.size(); + } + + @Override + public BinaryManifestEntry current() { + return current; + } + + @Override + public EncodedManifestEntry metadata() { + return null; + } + + @Override + public ManifestEntryRunMergeEntry.Key key() { + return entries.get(position).key; + } + + @Override + public ByteBuffer encodedRecord() { + return null; + } + + @Override + public ReusableIdentifier identifier() { + return identifier.replaceWithPartition(current); + } + + @Override + public void close() { + current.clear(); + identifier.release(); + entries.clear(); + position = -1; + } + } + + private static final class StoredEntry { + + final BinaryRow row; + final ManifestEntryRunMergeEntry.Key key; + + StoredEntry(BinaryRow row, ManifestEntryRunMergeEntry.Key key) { + this.row = row; + this.key = key; + } + } + + /** Fixed-size tournament tree which selects a cursor with one comparison per tree level. */ + private static final class SelectionTree { + + final List cursors; + final int leafBase; + final int[] winners; + + SelectionTree(List cursors) { + this.cursors = cursors; + int base = 1; + while (base < cursors.size()) { + base <<= 1; + } + this.leafBase = base; + this.winners = new int[leafBase << 1]; + Arrays.fill(winners, -1); + for (int cursor = 0; cursor < cursors.size(); cursor++) { + if (cursors.get(cursor).hasCurrent()) { + winners[leafBase + cursor] = cursor; + } + } + for (int node = leafBase - 1; node > 0; node--) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int winner() { + return winners[1]; + } + + Cursor cursor(int index) { + return cursors.get(index); + } + + void update(int cursor, boolean hasCurrent) { + int node = leafBase + cursor; + winners[node] = hasCurrent ? cursor : -1; + while ((node >>= 1) > 0) { + winners[node] = select(winners[node << 1], winners[(node << 1) + 1]); + } + } + + int select(int left, int right) { + if (left < 0) { + return right; + } + if (right < 0) { + return left; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + cursors.get(left).key(), cursors.get(right).key()); + return comparison < 0 || (comparison == 0 && left < right) ? left : right; + } + + boolean blockPrecedesOthers(int cursor, ManifestEntryRunMergeEntry.Key blockLastKey) { + for (int other = 0; other < cursors.size(); other++) { + if (other == cursor || !cursors.get(other).hasCurrent()) { + continue; + } + int comparison = + ManifestEntryRunMerge.compareMergeKeys( + blockLastKey, cursors.get(other).key()); + if (comparison > 0 || (comparison == 0 && cursor > other)) { + return false; + } + } + return true; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java index bee9ddc18792..d08b876ff025 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java @@ -70,9 +70,11 @@ public class ManifestFileSorter { /** Context object that carries shared state across compaction methods. */ static class CompactionContext { final boolean fullCompaction; + final boolean runMergeOptimizeEnabled; final ManifestSortKey sortKey; final ManifestEntryExternalSort.ExternalSortConfig externalSortConfig; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -81,38 +83,43 @@ static class CompactionContext { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; final List levelRuns; final List pickedRuns; CompactionContext( boolean fullCompaction, + boolean runMergeOptimizeEnabled, ManifestSortKey sortKey, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort, + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles, List levelRuns, List pickedRuns) { this.fullCompaction = fullCompaction; + this.runMergeOptimizeEnabled = runMergeOptimizeEnabled; this.sortKey = sortKey; this.externalSortConfig = externalSortConfig; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; this.levelRuns = levelRuns; this.pickedRuns = pickedRuns; } /** Check whether the given manifest file is marked for unsorted compaction. */ - boolean isMarkedForUnsortedCompaction(ManifestFileMeta file) { - return compactWithoutSort.containsKey(file); + boolean isMarkedForDefaultCompaction(ManifestFileMeta file) { + return defaultCompactFiles.containsKey(file); } } /** Result of classifying manifest files. */ - private static class ClassifyResult { + private static class ManifestClassification { final List lsmFiles; final CompactFileIdentifierSet deleteEntries; + final DeletedRowIdSet deletedRowIds; /** * Manifest files that need unsorted compaction. * @@ -121,29 +128,153 @@ private static class ClassifyResult { *

Value: true if fullCompaction is true and the file overlaps with delete partitions. It * means the file needs to eliminate delete entries file */ - final Map compactWithoutSort; + final Map defaultCompactFiles; - ClassifyResult( + ManifestClassification( List lsmFiles, CompactFileIdentifierSet deleteEntries, - Map compactWithoutSort) { + DeletedRowIdSet deletedRowIds, + Map defaultCompactFiles) { this.lsmFiles = lsmFiles; this.deleteEntries = deleteEntries; - this.compactWithoutSort = compactWithoutSort; + this.deletedRowIds = deletedRowIds; + this.defaultCompactFiles = defaultCompactFiles; } } /** Binary identifiers and partition values collected from DELETE entries. */ private static class DeletedEntryInfo { final CompactFileIdentifierSet identifiers; + final DeletedRowIdSet rowIds; final Set partitions; - private DeletedEntryInfo(CompactFileIdentifierSet identifiers, Set partitions) { + private DeletedEntryInfo( + CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, + Set partitions) { this.identifiers = identifiers; + this.rowIds = rowIds; this.partitions = partitions; } } + /** Primitive set used by RowID full compaction to avoid rebuilding file identifiers. */ + static final class DeletedRowIdSet { + + private static final long EMPTY = Long.MIN_VALUE; + private long[] table = emptyTable(16); + private int size; + private boolean containsMinValue; + private @Nullable long[] sortedRowIds; + + void add(long value) { + if (value == EMPTY) { + if (!containsMinValue) { + containsMinValue = true; + size++; + sortedRowIds = null; + } + return; + } + if ((size + 1) * 2 > table.length) { + grow(); + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return; + } + slot = (slot + 1) & (table.length - 1); + } + table[slot] = value; + size++; + sortedRowIds = null; + } + + boolean contains(long value) { + if (value == EMPTY) { + return containsMinValue; + } + int slot = slot(value, table.length); + while (table[slot] != EMPTY) { + if (table[slot] == value) { + return true; + } + slot = (slot + 1) & (table.length - 1); + } + return false; + } + + boolean intersects(long minInclusive, long maxInclusive) { + if (minInclusive > maxInclusive) { + return true; + } + long[] values = sortedRowIds(); + int position = java.util.Arrays.binarySearch(values, minInclusive); + if (position < 0) { + position = -position - 1; + } + return position < values.length && values[position] <= maxInclusive; + } + + private long[] sortedRowIds() { + if (sortedRowIds != null) { + return sortedRowIds; + } + long[] values = new long[size]; + int position = 0; + if (containsMinValue) { + values[position++] = EMPTY; + } + for (long value : table) { + if (value != EMPTY) { + values[position++] = value; + } + } + if (position != size) { + throw new IllegalStateException("Failed to snapshot deleted RowID set."); + } + java.util.Arrays.sort(values); + sortedRowIds = values; + return values; + } + + void releaseRangeIndex() { + sortedRowIds = null; + } + + private void grow() { + long[] previous = table; + if (previous.length >= (1 << 30)) { + throw new IllegalStateException("Too many deleted RowIDs in one manifest group."); + } + table = emptyTable(previous.length << 1); + int previousSize = size; + size = containsMinValue ? 1 : 0; + for (long value : previous) { + if (value != EMPTY) { + add(value); + } + } + if (size != previousSize) { + throw new IllegalStateException("Failed to grow deleted RowID set."); + } + } + + private static int slot(long value, int length) { + value ^= value >>> 33; + value *= 0xff51afd7ed558ccdL; + value ^= value >>> 33; + return ((int) value) & (length - 1); + } + + private static long[] emptyTable(int length) { + long[] table = new long[length]; + java.util.Arrays.fill(table, EMPTY); + return table; + } + } + /** * Try to sort-rewrite the merged manifest list by a configured partition field. If the sort * field cannot be resolved, the input is returned as-is. @@ -160,6 +291,7 @@ static List trySortCompaction( @Nullable IOManager ioManager) throws Exception { String sortPartitionField = options.manifestSortPartitionField(); + boolean runMergeOptimizeEnabled = options.manifestSortRunMergeOptimizeEnabled(); long suggestedMetaSize = options.manifestTargetSize().getBytes(); int suggestedMinMetaCount = options.manifestMergeMinCount(); long fullCompactionThreshold = options.manifestFullCompactionThresholdSize().getBytes(); @@ -178,6 +310,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, fullCompactionThreshold, @@ -196,6 +329,7 @@ static List trySortCompaction( partitionType, sortPartitionField, options.dataEvolutionEnabled(), + runMergeOptimizeEnabled, suggestedMetaSize, suggestedMinMetaCount, maxRewriteSize, @@ -218,6 +352,7 @@ private static Optional> tryFullCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long fullCompactionThreshold, @@ -246,6 +381,7 @@ private static Optional> tryFullCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -254,19 +390,19 @@ private static Optional> tryFullCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort full compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort full compact skipped: no runs picked and no defaultCompactFiles."); return Optional.empty(); } LOG.info( "Manifest sort full compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 3: Collect reused files (not picked) and picked files Set pickedSet = new HashSet<>(pickedRuns); @@ -280,7 +416,7 @@ private static Optional> tryFullCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Split into sections and merge small adjacent sections List

sections = splitIntoSections(pickedFiles, ctx); @@ -324,6 +460,7 @@ private static List tryMinorCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int suggestedMinMetaCount, long maxRewriteSize, @@ -341,6 +478,7 @@ private static List tryMinorCompaction( partitionType, sortPartitionField, dataEvolutionEnabled, + runMergeOptimizeEnabled, suggestedMetaSize, maxSizeAmplificationPercent, sortedRunSizeRatio, @@ -349,19 +487,19 @@ private static List tryMinorCompaction( List levelRuns = ctx.levelRuns; List pickedRuns = ctx.pickedRuns; - if (pickedRuns.isEmpty() && ctx.compactWithoutSort.isEmpty()) { + if (pickedRuns.isEmpty() && ctx.defaultCompactFiles.isEmpty()) { LOG.debug( - "Manifest sort minor compact skipped: no runs picked and no compactWithoutSort files."); + "Manifest sort minor compact skipped: no runs picked and no defaultCompactFiles."); return input; } LOG.info( "Manifest sort minor compact: input={} files, lsm={} runs, picked={} runs, " - + "compactWithoutSort={} files.", + + "defaultCompactFiles={}.", input.size(), levelRuns.size(), pickedRuns.size(), - ctx.compactWithoutSort.size()); + ctx.defaultCompactFiles.size()); // Step 2: Build fileName -> index mapping and initialize 2D result Map fileNameToIndex = new HashMap<>(); @@ -388,7 +526,7 @@ private static List tryMinorCompaction( for (ManifestAdjacentSortedRun run : pickedRuns) { pickedFiles.addAll(run.files()); } - pickedFiles.addAll(ctx.compactWithoutSort.keySet()); + pickedFiles.addAll(ctx.defaultCompactFiles.keySet()); // Step 4: Compute index range int minIdx = Integer.MAX_VALUE; @@ -450,26 +588,29 @@ private static CompactionContext prepareCompaction( RowType partitionType, String sortPartitionField, boolean dataEvolutionEnabled, + boolean runMergeOptimizeEnabled, long suggestedMetaSize, int maxSizeAmplificationPercent, int sortedRunSizeRatio, ManifestEntryExternalSort.ExternalSortConfig externalSortConfig, @Nullable Integer manifestReadParallelism) { + boolean rowIdSort = dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input); + boolean useRunMergeOptimize = rowIdSort && runMergeOptimizeEnabled; // Step 1: Resolve sort key. Data evolution tables prefer RowID ranges when available. - ManifestSortKey sortKey = - createSortKey(dataEvolutionEnabled, input, sortPartitionField, partitionType); + ManifestSortKey sortKey = createSortKey(rowIdSort, sortPartitionField, partitionType); // Step 2: Classify manifests into LSM files and collect delete entries. - ClassifyResult classifyResult = + ManifestClassification classification = classifyManifests( input, fullCompaction, manifestFile, partitionType, suggestedMetaSize, + useRunMergeOptimize, manifestReadParallelism); - List lsmFiles = classifyResult.lsmFiles; + List lsmFiles = classification.lsmFiles; // Step 3: Build level-sorted runs from LSM files based on partition order. List levelRuns = @@ -482,10 +623,12 @@ private static CompactionContext prepareCompaction( return new CompactionContext( fullCompaction, + useRunMergeOptimize, sortKey, externalSortConfig, - classifyResult.deleteEntries, - classifyResult.compactWithoutSort, + classification.deleteEntries, + classification.deletedRowIds, + classification.defaultCompactFiles, levelRuns, pickedRuns); } @@ -494,30 +637,34 @@ private static CompactionContext prepareCompaction( * Classify manifest files into default-compaction group and LSM group. * *

Full compaction: small files and files overlapping delete partitions go into - * compactWithoutSort; the rest are returned as lsmFiles. + * defaultCompactFiles; the rest are returned as lsmFiles. * - *

Non-full compaction: small files go to compactWithoutSort for minor-style merge; the rest + *

Non-full compaction: small files go to defaultCompactFiles for minor-style merge; the rest * are returned as lsmFiles. * - * @return ClassifyResult containing lsmFiles, deleteEntries, and compactWithoutSort + * @return classification containing lsmFiles, deleteEntries, and defaultCompactFiles */ - private static ClassifyResult classifyManifests( + private static ManifestClassification classifyManifests( List input, boolean fullCompaction, ManifestFile manifestFile, RowType partitionType, long suggestedMetaSize, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { // Initialize classification containers and read delete entries - Map compactWithoutSort = new LinkedHashMap<>(); + Map defaultCompactFiles = new LinkedHashMap<>(); List lsmFiles = new LinkedList<>(input); CompactFileIdentifierSet classifiedDeleteEntries = new CompactFileIdentifierSet(); + DeletedRowIdSet deletedRowIds = new DeletedRowIdSet(); Set deletePartitions = Collections.emptySet(); PartitionPredicate predicate = null; if (fullCompaction) { DeletedEntryInfo deletedEntries = - readDeletedEntries(manifestFile, input, manifestReadParallelism); + readDeletedEntries( + manifestFile, input, runMergeOptimizeEnabled, manifestReadParallelism); classifiedDeleteEntries = deletedEntries.identifiers; + deletedRowIds = deletedEntries.rowIds; deletePartitions = deletedEntries.partitions; // Build partition predicate from delete entries for overlap detection. @@ -546,18 +693,21 @@ private static ClassifyResult classifyManifests( file.partitionStats().nullCounts()); if (small || inDeleteRange) { iterator.remove(); - compactWithoutSort.put(file, inDeleteRange); + defaultCompactFiles.put(file, inDeleteRange); } } - return new ClassifyResult(lsmFiles, classifiedDeleteEntries, compactWithoutSort); + return new ManifestClassification( + lsmFiles, classifiedDeleteEntries, deletedRowIds, defaultCompactFiles); } private static DeletedEntryInfo readDeletedEntries( ManifestFile manifestFile, List manifestFiles, + boolean runMergeOptimizeEnabled, @Nullable Integer manifestReadParallelism) { CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet(); + DeletedRowIdSet rowIds = new DeletedRowIdSet(); Set partitions = new HashSet<>(); List filesWithDeletes = new ArrayList<>(); for (ManifestFileMeta meta : manifestFiles) { @@ -569,12 +719,26 @@ private static DeletedEntryInfo readDeletedEntries( if (filesWithDeletes.size() <= 1 || (manifestReadParallelism != null && manifestReadParallelism <= 1)) { for (ManifestFileMeta meta : filesWithDeletes) { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, false); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + false); } } else { Function> reader = meta -> { - collectDeletedEntries(meta, manifestFile, identifiers, partitions, true); + collectDeletedEntries( + meta, + manifestFile, + identifiers, + rowIds, + partitions, + runMergeOptimizeEnabled, + true); return Collections.singletonList(Boolean.TRUE); }; for (Boolean ignored : @@ -582,14 +746,16 @@ private static DeletedEntryInfo readDeletedEntries( // Iteration waits for each bounded batch of parallel reads. } } - return new DeletedEntryInfo(identifiers, partitions); + return new DeletedEntryInfo(identifiers, rowIds, partitions); } private static void collectDeletedEntries( ManifestFileMeta meta, ManifestFile manifestFile, CompactFileIdentifierSet identifiers, + DeletedRowIdSet rowIds, Set partitions, + boolean runMergeOptimizeEnabled, boolean synchronize) { try (CloseableIterator entries = manifestFile.scan( @@ -605,10 +771,16 @@ private static void collectDeletedEntries( if (synchronize) { synchronized (identifiers) { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } else { identifiers.add(entry); + if (runMergeOptimizeEnabled) { + rowIds.add(entry.file().nonNullFirstRowId()); + } partitions.add(partition); } } @@ -693,7 +865,7 @@ static List buildLevelSortedRuns( /** * Split picked files into sections. Files with overlapping sort-key intervals go into the same - * section. Each section is built with pre-computed totalSize and hasUnsortedCompactMeta. + * section. Each section is built with pre-computed totalSize and hasDefaultCompactFile. */ static List

splitIntoSections( List pickedFiles, CompactionContext ctx) { @@ -714,7 +886,7 @@ static List
splitIntoSections( currentSectionFiles.add(first); currentSectionTotalSize += first.fileSize(); - boolean currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(first); + boolean currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(first); ManifestFileMeta sectionMaxFile = first; for (int i = 1; i < pickedFiles.size(); i++) { @@ -726,20 +898,20 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); // start a new section currentSectionFiles = new ArrayList<>(); currentSectionTotalSize = 0; currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - currentSectionHasUnsortedCompactMeta = ctx.isMarkedForUnsortedCompaction(file); + currentSectionHasDefaultCompactFile = ctx.isMarkedForDefaultCompaction(file); sectionMaxFile = file; } else { currentSectionFiles.add(file); currentSectionTotalSize += file.fileSize(); - if (!currentSectionHasUnsortedCompactMeta - && ctx.isMarkedForUnsortedCompaction(file)) { - currentSectionHasUnsortedCompactMeta = true; + if (!currentSectionHasDefaultCompactFile + && ctx.isMarkedForDefaultCompaction(file)) { + currentSectionHasDefaultCompactFile = true; } if (sortKey.compareMax(file, sectionMaxFile) > 0) { sectionMaxFile = file; @@ -750,7 +922,7 @@ static List
splitIntoSections( new Section( currentSectionFiles, currentSectionTotalSize, - currentSectionHasUnsortedCompactMeta)); + currentSectionHasDefaultCompactFile)); return sections; } @@ -793,7 +965,7 @@ private static List
mergeSmallAdjacentSections( *
  • First overflow: The current section is split. The rewritable part is sorted and * rewritten. The remaining part is appended back to the sections queue for later * processing. - *
  • Subsequent overflows: If the section has files in compactWithoutSort (needs unsorted + *
  • Subsequent overflows: If the section has files in defaultCompactFiles (needs default * compaction), unsortedCompactSection is called to process it in smaller chunks. * Otherwise, the section is skipped. * @@ -904,9 +1076,9 @@ private static Section splitSectionAndRewriteHead( List tailFiles = new ArrayList<>(); long headSize = 0; long tailSize = 0; - // Whether tail section has files in compactWithoutSort, if true, the section need to + // Whether the tail section has files in defaultCompactFiles. If so, the section needs to // be rewritten. - boolean tailHasUnsortedCompactMeta = false; + boolean tailHasDefaultCompactFile = false; for (ManifestFileMeta file : section.files) { // Rewrite budget is enforced at manifest-file granularity. Include the first file that @@ -918,8 +1090,8 @@ private static Section splitSectionAndRewriteHead( } else { tailFiles.add(file); tailSize += file.fileSize(); - if (ctx.isMarkedForUnsortedCompaction(file)) { - tailHasUnsortedCompactMeta = true; + if (ctx.isMarkedForDefaultCompaction(file)) { + tailHasDefaultCompactFile = true; } } } @@ -929,7 +1101,7 @@ private static Section splitSectionAndRewriteHead( if (tailFiles.isEmpty()) { return null; } - return new Section(tailFiles, tailSize, tailHasUnsortedCompactMeta); + return new Section(tailFiles, tailSize, tailHasDefaultCompactFile); } /** @@ -947,7 +1119,7 @@ private static void rewriteSectionBeyondBudget( int suggestedMinMetaCount, @Nullable Integer manifestReadParallelism) throws Exception { - if (section.hasUnsortedCompactMeta) { + if (section.hasDefaultCompactFile) { unsortedCompactSection( section.files, output, @@ -967,8 +1139,8 @@ private static void rewriteSectionBeyondBudget( * *

    Semantics difference from old minor merge: In the old ManifestFileMerger path, the * trailing candidates are kept unchanged when their count is below manifest.merge-min-count. In - * this sort path, unsortedCompactSection is triggered when compactWithoutSort is non-empty, - * regardless of the manifest count. This is because files in compactWithoutSort either: + * this sort path, unsortedCompactSection is triggered when defaultCompactFiles is non-empty, + * regardless of the manifest count. This is because files in defaultCompactFiles either: * *

      *
    • Are small files needing consolidation @@ -1038,7 +1210,7 @@ private static void rewriteSection( @Nullable Integer manifestReadParallelism) throws Exception { // Skip rewrite for single file not in delete-range. - if (section.size() == 1 && !ctx.compactWithoutSort.getOrDefault(section.get(0), false)) { + if (section.size() == 1 && !ctx.defaultCompactFiles.getOrDefault(section.get(0), false)) { output.addUnchanged(section.get(0)); return; } @@ -1064,24 +1236,38 @@ private static void rewriteFull( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - List sorted = - ManifestEntryExternalSort.sortAndWriteFullEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - ctx.deleteEntries, - manifestReadParallelism); + List sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteFullEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + ctx.deletedRowIds, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteFullEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + ctx.deleteEntries, + manifestReadParallelism); + } if (!sorted.isEmpty()) { output.addSortedFiles(sorted); } } /** - * Minor compaction path: collect DELETE entries in memory while external-sorting all entries, - * then write surviving ADD entries from the sorted stream and remaining DELETE entries from - * memory. + * Minor compaction path: collect DELETE identities, merge the existing sorted runs, and write + * surviving ADD entries and unmatched DELETE entries separately. Falls back to external sort + * when the input is not suitable for run merge. */ private static void rewriteMinor( List section, @@ -1091,14 +1277,26 @@ private static void rewriteMinor( ManifestFile manifestFile, @Nullable Integer manifestReadParallelism) throws Exception { - Pair, List> sorted = - ManifestEntryExternalSort.sortAndWriteMinorEntries( - section, - ctx.sortKey, - ctx.externalSortConfig, - manifestFile, - sortNewFiles, - manifestReadParallelism); + Pair, List> sorted = null; + if (ctx.runMergeOptimizeEnabled) { + sorted = + ManifestEntryRunMerge.sortAndWriteMinorEntries( + section, + (RowIdEntrySortKey) ctx.sortKey, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } + if (sorted == null) { + sorted = + ManifestEntryExternalSort.sortAndWriteMinorEntries( + section, + ctx.sortKey, + ctx.externalSortConfig, + manifestFile, + sortNewFiles, + manifestReadParallelism); + } if (!sorted.getLeft().isEmpty()) { output.addSortedFiles(sorted.getLeft()); @@ -1119,11 +1317,8 @@ private static boolean containsNoDeleteEntries(List section) { } private static ManifestSortKey createSortKey( - boolean dataEvolutionEnabled, - List input, - String sortPartitionField, - RowType partitionType) { - if (dataEvolutionEnabled && ManifestFileMeta.allContainsRowId(input)) { + boolean rowIdSort, String sortPartitionField, RowType partitionType) { + if (rowIdSort) { // RowID sorting uses the configured partition field as the primary key when specified, // otherwise it uses the full partition row to preserve partition locality. It then // orders files by RowID. @@ -1202,6 +1397,11 @@ void replaceExternalSortRow( InternalRow binaryManifestRow(BinaryRow row); } + interface RowIdEntrySortKey extends ManifestSortKey { + + int comparePartitions(BinaryRow left, BinaryRow right); + } + private static class PartitionSortKey implements ManifestSortKey { private final RecordComparator fieldComparator; @@ -1273,7 +1473,7 @@ public InternalRow binaryManifestRow(BinaryRow row) { } } - private static class RowIdSortKey implements ManifestSortKey { + private static class RowIdSortKey implements RowIdEntrySortKey { @Nullable private final RecordComparator partitionComparator; private final InternalRow.FieldGetter[] partitionFieldGetters; @@ -1288,21 +1488,8 @@ private RowIdSortKey( this.partitionComparator = partitionComparator; this.partitionFieldGetters = createPartitionFieldGetters(partitionType, partitionSortFields); - - List fieldTypes = new ArrayList<>(); - for (int partitionSortField : partitionSortFields) { - fieldTypes.add(partitionType.getTypeAt(partitionSortField)); - } - // ADD must precede DELETE for the same partition. Minor compaction streams the sorted - // rows once and uses this ordering to eliminate a matching pair without retaining all - // ADD identifiers. - fieldTypes.add(DataTypes.TINYINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.BIGINT()); - fieldTypes.add(DataTypes.STRING()); - fieldTypes.add(ManifestEntry.MANIFEST_ROW_TYPE); - this.externalSortRowType = DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + this.externalSortRowType = + createRowIdExternalSortRowType(partitionType, partitionSortFields); this.sortFieldNum = externalSortRowType.getFieldCount() - 1; this.externalSortKeyFields = createSequentialFields(sortFieldNum); } @@ -1336,7 +1523,7 @@ public boolean isAfterMax(ManifestFileMeta file, ManifestFileMeta maxFile) { return c > 0; } } - return Long.compare(nonNullMinRowId(file), nonNullMaxRowId(maxFile)) > 0; + return nonNullMinRowId(file) > nonNullMaxRowId(maxFile); } @Override @@ -1373,6 +1560,11 @@ public InternalRow binaryManifestRow(BinaryRow row) { return row.getRow(sortFieldNum, ManifestEntry.MANIFEST_ROW_TYPE.getFieldCount()); } + @Override + public int comparePartitions(BinaryRow left, BinaryRow right) { + return partitionComparator == null ? 0 : partitionComparator.compare(left, right); + } + private int comparePartitionMin(ManifestFileMeta a, ManifestFileMeta b) { if (partitionComparator == null) { return 0; @@ -1412,6 +1604,26 @@ private static long rowIdRangeEnd(ManifestEntry entry) { } } + private static RowType createRowIdExternalSortRowType( + RowType partitionType, int[] partitionSortFields) { + List fieldTypes = new ArrayList<>(partitionSortFields.length + 6); + for (int partitionSortField : partitionSortFields) { + fieldTypes.add(partitionType.getTypeAt(partitionSortField)); + } + // ADD must precede DELETE for the same partition. Minor compaction streams the sorted rows + // once and uses this ordering to eliminate a matching pair without retaining all ADD + // identifiers. + Collections.addAll( + fieldTypes, + DataTypes.TINYINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.STRING(), + ManifestEntry.MANIFEST_ROW_TYPE); + return DataTypes.ROW(fieldTypes.toArray(new DataType[0])); + } + private static int[] createSequentialFields(int fieldCount) { int[] fields = new int[fieldCount]; for (int i = 0; i < fieldCount; i++) { @@ -1530,12 +1742,12 @@ public void addDeleteFiles(List files) { static class Section { final List files; final long totalSize; - final boolean hasUnsortedCompactMeta; + final boolean hasDefaultCompactFile; - Section(List files, long totalSize, boolean hasUnsortedCompactMeta) { + Section(List files, long totalSize, boolean hasDefaultCompactFile) { this.files = files; this.totalSize = totalSize; - this.hasUnsortedCompactMeta = hasUnsortedCompactMeta; + this.hasDefaultCompactFile = hasDefaultCompactFile; } /** Create a merged section from two sections. */ @@ -1545,7 +1757,7 @@ static Section merge(Section a, Section b) { return new Section( merged, a.totalSize + b.totalSize, - a.hasUnsortedCompactMeta || b.hasUnsortedCompactMeta); + a.hasDefaultCompactFile || b.hasDefaultCompactFile); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 7dff697e8fc6..9fa1edb7615e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -59,10 +59,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -70,6 +72,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -883,6 +886,28 @@ private void beforeFirstRead() throws IOException { } } + private static class CountingReadFileIO extends LocalFileIO { + + private final Map readCounts = new ConcurrentHashMap<>(); + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + readCounts + .computeIfAbsent(path.getName(), ignored -> new AtomicInteger()) + .incrementAndGet(); + return super.newInputStream(path); + } + + private int readCount(String fileName) { + AtomicInteger count = readCounts.get(fileName); + return count == null ? 0 : count.get(); + } + + private void resetReadCounts() { + readCounts.clear(); + } + } + // ==================== Manifest Sort Tests ==================== /** @@ -1263,6 +1288,386 @@ public void testDataEvolutionManifestSortByPartitionAndRowId() { } } + @Test + public void testDisablingRunMergeOptimizePreservesDataEvolutionRowIdSort() { + assertThat( + CoreOptions.fromMap(Collections.emptyMap()) + .manifestSortRunMergeOptimizeEnabled()) + .isTrue(); + + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "row-30", 0, 30, 5), + makeRowIdEntry(true, "row-10", 0, 10, 5)), + makeManifest( + makeRowIdEntry(true, "row-20", 0, 20, 5), + makeRowIdEntry(true, "row-0", 0, 0, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("manifest-sort.run-merge-optimize.enabled", "false"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + CoreOptions coreOptions = CoreOptions.fromMap(testOptions.toMap()); + + assertThat(coreOptions.manifestSortRunMergeOptimizeEnabled()).isFalse(); + + List merged = + ManifestFileMerger.merge(input, manifestFile, getPartitionType(), coreOptions); + + assertEquivalentEntries(input, merged); + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("row-0", "row-10", "row-20", "row-30"); + } + + @Test + public void testDataEvolutionManifestRunMergeSecondaryKeys() { + List firstManifest = new ArrayList<>(); + firstManifest.add(makeRowIdEntry(true, "range-short", 0, 100, 5, 1)); + firstManifest.add(makeRowIdEntry(true, "sequence-newer", 0, 100, 10, 9)); + for (int i = 19; i >= 10; i--) { + firstManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List secondManifest = new ArrayList<>(); + for (int i = 9; i >= 0; i--) { + secondManifest.add(makeRowIdEntry(true, String.format("tie-%02d", i), 0, 100, 10, 5)); + } + + List input = new ArrayList<>(); + input.add(makeManifest(firstManifest.toArray(new ManifestEntry[0]))); + input.add(makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + List expected = new ArrayList<>(); + expected.add("range-short"); + expected.add("sequence-newer"); + for (int i = 0; i < 20; i++) { + expected.add(String.format("tie-%02d", i)); + } + assertThat(readEntries(merged).stream().map(e -> e.file().fileName())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesExactDeleteIdentifier() { + List input = + Arrays.asList( + makeManifest( + makeRowIdEntry(true, "deleted", 0, 100, 5), + makeRowIdEntry(true, "same-row-id-survivor", 0, 100, 5)), + makeManifest(makeRowIdEntry(false, "deleted", 0, 100, 5))); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + testOptions.set("scan.manifest.parallelism", "2"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().fileName())) + .containsExactly("same-row-id-survivor"); + } + + @Test + public void testDataEvolutionManifestRunMergeUsesRawDeleteIdentityFields() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry survivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + List input = + Arrays.asList(makeManifest(deleted, survivor), makeManifest(delete)); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged)).singleElement().isEqualTo(survivor); + } + + @Test + public void testDataEvolutionManifestRunMergeManyPartitions() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (int partition = 39; partition >= 0; partition--) { + ManifestEntry entry = + makeRowIdEntry( + true, + String.format("partition-%02d", partition), + partition, + partition * 10L, + 5); + (partition >= 20 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(e -> e.partition().getInt(0))) + .containsExactlyElementsOf( + IntStream.range(0, 40).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFragmentedSmallManifests() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 199; firstRowId >= 0; firstRowId--) { + ManifestEntry entry = + makeRowIdEntry(true, String.format("row-%03d", firstRowId), 0, firstRowId, 1); + (firstRowId >= 100 ? firstManifest : secondManifest).add(entry); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 200).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionManifestRunMergeFallsBackForLargeFragmentedManifest() { + List firstManifest = new ArrayList<>(); + List secondManifest = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 12_500; firstRowId--) { + firstManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + for (long firstRowId = 12_499; firstRowId >= 0; firstRowId--) { + secondManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(firstManifest.toArray(new ManifestEntry[0])), + makeManifest(secondManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList())); + } + + @Test + public void testDataEvolutionMinorRunMergeFallsBackForLargeFragmentedManifest() { + List fragmentedEntries = new ArrayList<>(); + for (long firstRowId = 25_000; firstRowId >= 0; firstRowId--) { + fragmentedEntries.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 0, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])), + makeManifest( + makeRowIdEntry(false, "row-12500", 0, 12_500, 1), + makeRowIdEntry(true, "row-30000", 0, 30_000, 1))); + + List expected = + LongStream.rangeClosed(0, 25_000).boxed().collect(Collectors.toList()); + expected.remove(Long.valueOf(12_500L)); + expected.add(30_000L); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat(runMergeResult.stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expected); + } + + @Test + public void testDataEvolutionManifestRunMergeLimitsReadAmplification() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + + int runCount = 20; + int entriesPerRun = 600; + List fragmentedEntries = new ArrayList<>(); + for (int run = runCount - 1; run >= 0; run--) { + long runStart = (long) run * entriesPerRun; + for (int entry = 0; entry < entriesPerRun; entry++) { + long firstRowId = runStart + entry; + fragmentedEntries.add( + makeRowIdEntry( + true, + String.format("fragmented-%05d", firstRowId), + 0, + firstRowId, + 1)); + } + } + + ManifestFileMeta fragmented = makeManifest(fragmentedEntries.toArray(new ManifestEntry[0])); + ManifestFileMeta overlap = + makeManifest(makeRowIdEntry(true, "overlap", 0, entriesPerRun / 2L, 1)); + fileIO.resetReadCounts(); + + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + Arrays.asList(fragmented, overlap), + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(fileIO.readCount(fragmented.fileName())).isEqualTo(2); + List expectedRowIds = + LongStream.range(0, (long) runCount * entriesPerRun) + .boxed() + .collect(Collectors.toList()); + expectedRowIds.add(entriesPerRun / 2L); + Collections.sort(expectedRowIds); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf(expectedRowIds); + } + + @Test + public void testDataEvolutionManifestRunMergePreservesBlockStats() { + List spanningManifest = new ArrayList<>(); + List middleManifest = new ArrayList<>(); + for (long firstRowId = 0; firstRowId < 5_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + for (long firstRowId = 10_000; firstRowId < 15_000; firstRowId++) { + spanningManifest.add( + makeRowIdEntry(true, String.format("row-%05d", firstRowId), 7, firstRowId, 1)); + } + for (long firstRowId = 5_000; firstRowId < 10_000; firstRowId++) { + middleManifest.add( + makeRowIdEntry( + true, String.format("row-%05d", firstRowId), null, firstRowId, 1)); + } + + List input = + Arrays.asList( + makeManifest(spanningManifest.toArray(new ManifestEntry[0])), + makeManifest(middleManifest.toArray(new ManifestEntry[0]))); + Options testOptions = new Options(); + testOptions.set("manifest-sort.enabled", "true"); + testOptions.set("data-evolution.enabled", "true"); + testOptions.set("manifest.full-compaction-threshold-size", "1B"); + + List merged = + ManifestFileMerger.merge( + input, + manifestFile, + getPartitionType(), + CoreOptions.fromMap(testOptions.toMap())); + + assertThat(merged).hasSize(1); + ManifestFileMeta output = merged.get(0); + assertThat(output.numAddedFiles()).isEqualTo(15_000); + assertThat(output.numDeletedFiles()).isZero(); + assertThat(output.minRowId()).isZero(); + assertThat(output.maxRowId()).isEqualTo(14_999); + assertThat(output.partitionStats().minValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().maxValues().getInt(0)).isEqualTo(7); + assertThat(output.partitionStats().nullCounts().getLong(0)).isEqualTo(10_000); + assertThat(readEntries(merged).stream().map(entry -> entry.file().nonNullFirstRowId())) + .containsExactlyElementsOf( + LongStream.range(0, 15_000).boxed().collect(Collectors.toList())); + } + @Test public void testDataEvolutionManifestSortUsesConfiguredPartitionFieldBeforeRowId() { RowType multiPartitionType = RowType.of(new IntType(), new IntType(), new IntType()); @@ -1394,6 +1799,101 @@ public void testDataEvolutionMinorManifestSortPreservesUnmatchedDeleteEntries() .containsExactly("ADD-new-row20", "ADD-survivor-row30", "DELETE-old-row10"); } + @Test + public void testDataEvolutionMinorRunMergeMatchesExternalSort() { + ManifestEntry deleted = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry sameRowIdSurvivor = + makeRowIdEntry( + true, + "same-file-name", + 0, + 100, + 5, + 0, + Collections.singletonList("other-extra"), + new byte[] {3, 4}, + "external-b"); + ManifestEntry delete = + makeRowIdEntry( + false, + "same-file-name", + 0, + 100, + 5, + 0, + Arrays.asList("extra-a", "extra-b"), + new byte[] {1, 2}, + "external-a"); + ManifestEntry unmatchedDelete = makeRowIdEntry(false, "old-row-200", 0, 200, 5); + List input = + Arrays.asList( + makeManifest( + deleted, + sameRowIdSurvivor, + makeRowIdEntry(true, "survivor-row-300", 0, 300, 5)), + makeManifest(delete, unmatchedDelete, unmatchedDelete)); + + List externalResult = readEntries(mergeMinorManifestEntries(input, false)); + List runMergeResult = readEntries(mergeMinorManifestEntries(input, true)); + + assertThat(runMergeResult).containsExactlyElementsOf(externalResult); + assertThat( + runMergeResult.stream() + .map(entry -> entry.kind() + "-" + entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly( + "ADD-same-file-name", "ADD-survivor-row-300", "DELETE-old-row-200"); + assertThat(runMergeResult.get(0)).isEqualTo(sameRowIdSurvivor); + } + + @Test + public void testDataEvolutionMinorRunMergeCollectsDeletesDuringDiscovery() { + CountingReadFileIO fileIO = new CountingReadFileIO(); + manifestFile = createManifestFile(tempDir.toString(), fileIO); + ManifestFileMeta base = + makeManifest( + makeRowIdEntry(true, "deleted-row-10", 0, 10, 5), + makeRowIdEntry(true, "survivor-row-20", 0, 20, 5)); + ManifestFileMeta delta = + makeManifest( + makeRowIdEntry(true, "survivor-row-30", 0, 30, 5), + makeRowIdEntry(false, "deleted-row-10", 0, 10, 5)); + fileIO.resetReadCounts(); + + List merged = mergeMinorManifestEntries(Arrays.asList(base, delta), true); + + assertThat(fileIO.readCount(base.fileName())).isEqualTo(2); + assertThat(fileIO.readCount(delta.fileName())).isEqualTo(2); + assertThat( + readEntries(merged).stream() + .map(entry -> entry.file().fileName()) + .collect(Collectors.toList())) + .containsExactly("survivor-row-20", "survivor-row-30"); + } + + private List mergeMinorManifestEntries( + List input, boolean runMergeOptimizeEnabled) { + Options options = new Options(); + options.set("manifest-sort.enabled", "true"); + options.set( + "manifest-sort.run-merge-optimize.enabled", + Boolean.toString(runMergeOptimizeEnabled)); + options.set("data-evolution.enabled", "true"); + options.set("manifest.full-compaction-threshold-size", Long.MAX_VALUE + "B"); + return ManifestFileMerger.merge( + input, manifestFile, getPartitionType(), CoreOptions.fromMap(options.toMap())); + } + /** * Test manifest sort with a multi-field partition type. * @@ -1909,20 +2409,46 @@ private List readFileNames( /** Create a ManifestEntry with row ID metadata for data evolution manifest sort tests. */ private ManifestEntry makeRowIdEntry( - boolean isAdd, String fileName, int partition, long firstRowId, long rowCount) { + boolean isAdd, String fileName, Integer partition, long firstRowId, long rowCount) { return makeRowIdEntry(isAdd, fileName, partition, firstRowId, rowCount, 0); } private ManifestEntry makeRowIdEntry( boolean isAdd, String fileName, - int partition, + Integer partition, long firstRowId, long rowCount, long sequenceNumber) { + return makeRowIdEntry( + isAdd, + fileName, + partition, + firstRowId, + rowCount, + sequenceNumber, + Collections.emptyList(), + null, + null); + } + + private ManifestEntry makeRowIdEntry( + boolean isAdd, + String fileName, + Integer partition, + long firstRowId, + long rowCount, + long sequenceNumber, + List extraFiles, + byte[] embeddedIndex, + String externalPath) { BinaryRow binaryRow = new BinaryRow(1); BinaryRowWriter writer = new BinaryRowWriter(binaryRow); - writer.writeInt(0, partition); + if (partition == null) { + writer.setNullAt(0); + } else { + writer.writeInt(0, partition); + } writer.complete(); return ManifestEntry.create( @@ -1942,13 +2468,13 @@ private ManifestEntry makeRowIdEntry( sequenceNumber, 0, 0, - Collections.emptyList(), + extraFiles, Timestamp.fromEpochMillis(200000), 0L, - null, + embeddedIndex, FileSource.APPEND, null, - null, + externalPath, firstRowId, Collections.singletonList("f0"))); } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index c1e1898a3648..c8c2b1bb0bb2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.avro.AvroFileFormat; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; @@ -63,7 +64,8 @@ public class ManifestFileTest { private final ManifestTestDataGenerator gen = ManifestTestDataGenerator.builder().build(); - private final FileFormat avro = FileFormat.fromIdentifier("avro", new Options()); + private final AvroFileFormat avro = + (AvroFileFormat) FileFormat.fromIdentifier("avro", new Options()); @TempDir java.nio.file.Path tempDir; @@ -297,6 +299,22 @@ void testAvroReaderReadsLegacyDataFileMetaWithFewerFields() throws Exception { assertThat(actual.fileName()).isEqualTo(source.fileName()); assertThat(actual.file().firstRowId()).isNull(); assertThat(actual.file().writeCols()).isNull(); + + BinaryManifestEntry.Projection projection = BinaryManifestEntry.ROW_RANGE_PROJECTION; + BinaryManifestEntry binaryEntry = projection.createEntry(); + try (ManifestAvroBlockReader reader = + new ManifestAvroBlockReader(fileIO.newInputStream(path), avro)) { + assertThat(reader.hasNext()).isTrue(); + ManifestAvroBlockReader.RawBlock block = reader.next(); + assertThat(block.rawBlockCopySupported()).isFalse(); + ManifestAvroBlockReader.RowIterator rows = block.toRow(projection); + assertThat(rows.hasNext()).isTrue(); + binaryEntry.replace(rows.next()); + assertThat(binaryEntry.rowCount()).isEqualTo(source.rowCount()); + assertThat(binaryEntry.firstRowId()).isNull(); + assertThat(rows.hasNext()).isFalse(); + assertThat(reader.hasNext()).isFalse(); + } } @Test @@ -516,6 +534,111 @@ void testScanProjectedManifestEntriesCanBeRetained() throws Exception { entries.stream().map(ManifestEntry::rowCount).collect(Collectors.toList())); } + @Test + void testBlockReaderConvertsRawBlocksToProjectedRows() throws Exception { + List entries = Arrays.asList(gen.next(), gen.next(), gen.next()); + ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestFileMeta manifest = writeSingleManifest(manifestFile, entries); + BinaryManifestEntry.Projection projection = + projection(DataFileMeta.FILE_NAME, DataFileMeta.ROW_COUNT); + BinaryManifestEntry actual = projection.createEntry(); + InternalRow reusedRow = null; + InternalRow reusedFileRow = null; + int position = 0; + + try (ManifestAvroBlockReader reader = + manifestFile.scanForRunMerge(manifest.fileName(), manifest.fileSize())) { + while (reader.hasNext()) { + ManifestAvroBlockReader.RawBlock block = reader.next(); + assertThat(block.rawBlockCopySupported()).isTrue(); + ManifestAvroBlockReader.RowIterator rows = block.toRow(projection); + while (rows.hasNext()) { + InternalRow row = rows.next(); + InternalRow fileRow = row.getRow(2, 2); + if (reusedRow != null) { + assertThat(row).isSameAs(reusedRow); + assertThat(fileRow).isSameAs(reusedFileRow); + } + reusedRow = row; + reusedFileRow = fileRow; + actual.replace(row); + ManifestEntry expected = entries.get(position++); + assertThat(actual.kind()).isEqualTo(expected.kind()); + assertThat(actual.partition()).isEqualTo(expected.partition()); + assertThat(actual.fileName()).isEqualTo(expected.fileName()); + assertThat(actual.rowCount()).isEqualTo(expected.rowCount()); + } + } + } + + assertThat(position).isEqualTo(entries.size()); + } + + @Test + void testBlockReaderSupportsReorderedProjection() throws Exception { + List entries = Arrays.asList(gen.next(), gen.next(), gen.next()); + ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestFileMeta manifest = writeSingleManifest(manifestFile, entries); + RowType manifestType = ManifestEntry.MANIFEST_ROW_TYPE; + RowType fileType = + DataFileMeta.SCHEMA.project(DataFileMeta.ROW_COUNT, DataFileMeta.FILE_NAME); + RowType projectedType = + new RowType( + false, + Arrays.asList( + manifestType.getField(ManifestEntry.FILE).newType(fileType), + manifestType.getField(ManifestEntry.PARTITION), + manifestType.getField(ManifestEntry.KIND))); + BinaryManifestEntry.Projection projection = + BinaryManifestEntry.Projection.create(projectedType); + BinaryManifestEntry actual = projection.createEntry(); + int position = 0; + + try (ManifestAvroBlockReader reader = + manifestFile.scanForRunMerge(manifest.fileName(), manifest.fileSize())) { + while (reader.hasNext()) { + ManifestAvroBlockReader.RowIterator rows = reader.next().toRow(projection); + while (rows.hasNext()) { + InternalRow row = rows.next(); + assertThat(row.getRow(0, 2).getLong(0)) + .isEqualTo(entries.get(position).rowCount()); + actual.replace(row); + assertThat(actual.fileName()).isEqualTo(entries.get(position).fileName()); + assertThat(actual.partition()).isEqualTo(entries.get(position).partition()); + assertThat(actual.kind()).isEqualTo(entries.get(position).kind()); + position++; + } + } + } + + assertThat(position).isEqualTo(entries.size()); + } + + @Test + void testBlockReaderSupportsFullManifestProjection() throws Exception { + List entries = Arrays.asList(gen.next(), gen.next(), gen.next()); + ManifestFile manifestFile = createManifestFile(tempDir.toString(), Long.MAX_VALUE); + ManifestFileMeta manifest = writeSingleManifest(manifestFile, entries); + BinaryManifestEntry.Projection projection = BinaryManifestEntry.fullProjection(); + BinaryManifestEntry binaryEntry = projection.createEntry(); + ManifestEntrySerializer serializer = new ManifestEntrySerializer(); + int position = 0; + + try (ManifestAvroBlockReader reader = + manifestFile.scanForRunMerge(manifest.fileName(), manifest.fileSize())) { + while (reader.hasNext()) { + ManifestAvroBlockReader.RowIterator rows = reader.next().toRow(projection); + while (rows.hasNext()) { + binaryEntry.replace(rows.next()); + assertThat(serializer.fromRow(binaryEntry.fullRow())) + .isEqualTo(entries.get(position++)); + } + } + } + + assertThat(position).isEqualTo(entries.size()); + } + @Test void testScanProjectedManifestKeepsEntryValidWhenAdvancing() throws Exception { List entries = Arrays.asList(gen.next(), gen.next()); diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java new file mode 100644 index 000000000000..8e8d9b164195 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlock.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.file; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** Reusable compressed Avro block. */ +public final class RawBlock { + + private DataFileStream.DataBlock block; + private Codec codec; + private Schema schema; + private boolean decompressed; + private ByteBuffer decompressedBuffer; + + RawBlock(DataFileStream.DataBlock block, Codec codec, Schema schema) { + replace(block, codec, schema); + } + + RawBlock replace(DataFileStream.DataBlock block, Codec codec, Schema schema) { + this.block = block; + this.codec = codec; + this.schema = schema; + this.decompressed = false; + this.decompressedBuffer = null; + return this; + } + + DataFileStream.DataBlock dataBlock() { + return block; + } + + public long recordCount() { + return block.getNumEntries(); + } + + public int compressedSize() { + if (decompressed) { + throw new IllegalStateException("The Avro block has already been decompressed."); + } + return block.getBlockSize(); + } + + public ByteBuffer decompress(ByteBuffer reuse) throws IOException { + if (!decompressed) { + if (codec instanceof ZstandardCodec) { + ByteBuffer source = block.getAsByteBuffer(); + ByteBuffer target = + reuse != null && reuse.hasArray() ? reuse : ByteBuffer.allocate(256 * 1024); + int size = 0; + try (InputStream compressed = + new ByteArrayInputStream( + source.array(), + source.arrayOffset() + source.position(), + source.remaining()); + InputStream input = ZstandardLoader.input(compressed, true)) { + while (true) { + if (size == target.capacity()) { + int grownCapacity = + target.capacity() == 0 + ? 256 * 1024 + : Math.multiplyExact(target.capacity(), 2); + ByteBuffer grown = ByteBuffer.allocate(grownCapacity); + System.arraycopy( + target.array(), + target.arrayOffset(), + grown.array(), + grown.arrayOffset(), + size); + target = grown; + } + int read = + input.read( + target.array(), + target.arrayOffset() + size, + target.capacity() - size); + if (read < 0) { + break; + } + size += read; + } + } + target.position(0); + target.limit(size); + decompressedBuffer = target.duplicate(); + } else { + block.decompressUsing(codec); + decompressedBuffer = block.getAsByteBuffer().duplicate(); + } + decompressed = true; + } + return decompressedBuffer.duplicate(); + } + + /** Returns a single-block stream for appending this compressed block to an Avro writer. */ + public DataFileStream asStream() throws IOException { + if (decompressed) { + throw new IllegalStateException("A decompressed Avro block cannot be copied raw."); + } + return new SingleBlockStream(schema, codec, block); + } + + private static final class SingleBlockStream extends DataFileStream { + + private final Schema schema; + private final Codec codec; + private DataBlock block; + + private SingleBlockStream(Schema schema, Codec codec, DataBlock block) throws IOException { + super(new NoOpDatumReader()); + this.schema = schema; + this.codec = codec; + this.block = block; + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + Codec resolveCodec() { + return codec; + } + + @Override + boolean hasNextBlock() { + return block != null; + } + + @Override + DataBlock nextRawBlock(DataBlock reuse) { + DataBlock result = block; + block = null; + return result; + } + + @Override + public void close() {} + } + + private static final class NoOpDatumReader implements DatumReader { + + @Override + public void setSchema(Schema schema) {} + + @Override + public D read(D reuse, Decoder decoder) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java new file mode 100644 index 000000000000..43a68c54a44d --- /dev/null +++ b/paimon-format/src/main/java/org/apache/avro/file/RawBlockReader.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.file; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; + +import java.io.IOException; +import java.io.InputStream; + +/** Package bridge exposing Avro's compressed blocks without reflection. */ +public final class RawBlockReader extends DataFileStream { + + public RawBlockReader(InputStream input) throws IOException { + super(input, new NoOpDatumReader()); + } + + public boolean hasNextRawBlock() { + return super.hasNextBlock(); + } + + public RawBlock nextRawBlock(RawBlock reuse) throws IOException { + DataBlock raw = super.nextRawBlock(reuse == null ? null : reuse.dataBlock()); + return reuse == null + ? new RawBlock(raw, resolveCodec(), getSchema()) + : reuse.replace(raw, resolveCodec(), getSchema()); + } + + private static final class NoOpDatumReader implements DatumReader { + + @Override + public void setSchema(Schema schema) {} + + @Override + public D read(D reuse, Decoder decoder) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java index ef98b15fa035..d6b2af6a7257 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockReader.java @@ -22,8 +22,10 @@ import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; -import org.apache.avro.file.DataFileStream; -import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.file.RawBlock; +import org.apache.avro.file.RawBlockReader; + +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; @@ -31,27 +33,30 @@ import java.nio.ByteBuffer; /** - * Reader which exposes decompressed blocks from an Avro object container file. + * Reader which exposes compressed and decompressed blocks from an Avro object container file. * *

      This reader owns the input stream and closes it when construction fails or {@link #close()} is * called. */ public final class AvroBlockReader implements Closeable { - private final DataFileStream reader; + private final RawBlockReader reader; + private @Nullable AvroRawBlock borrowedRawBlock; + private @Nullable ByteBuffer decompressionBuffer; private long currentBlockRecordCount = -1; public AvroBlockReader(InputStream input) throws IOException { try { - this.reader = new DataFileStream<>(input, new GenericDatumReader<>()); + this.reader = new RawBlockReader(input); } catch (IOException | RuntimeException | Error e) { IOUtils.closeQuietly(input); throw e; } } - Schema schema() { + /** Returns the writer schema stored in the Avro file header. */ + public Schema schema() { return reader.getSchema(); } @@ -62,7 +67,7 @@ public AvroRecordDecoder createRecordDecoder() { /** Returns whether another block is available. */ public boolean hasNextBlock() throws IOException { - return replaceAvroRuntimeException(reader::hasNext); + return replaceAvroRuntimeException(reader::hasNextRawBlock); } /** @@ -85,8 +90,8 @@ public byte[] nextBlock() throws IOException { * {@link #hasNextBlock()}, {@link #nextBlock()}, or this method, or when this reader is closed. */ public BorrowedBlock nextBorrowedBlock() throws IOException { - ByteBuffer block = replaceAvroRuntimeException(reader::nextBlock); - currentBlockRecordCount = reader.getBlockCount(); + ByteBuffer block = nextBorrowedRawBlock().decompress(decompressionBuffer); + decompressionBuffer = block; return new BorrowedBlock( block.array(), block.arrayOffset() + block.position(), @@ -94,6 +99,33 @@ public BorrowedBlock nextBorrowedBlock() throws IOException { currentBlockRecordCount); } + /** + * Returns a borrowed view of the next compressed block. + * + *

      The returned holder and its storage are owned by this reader and reused by the next call + * to this method. Consume the block before advancing this reader. + */ + public AvroRawBlock nextBorrowedRawBlock() throws IOException { + borrowedRawBlock = nextRawBlock(borrowedRawBlock); + return borrowedRawBlock; + } + + /** + * Returns the next compressed block, optionally reusing the supplied holder and its storage. + * + *

      A block returned with a non-null reuse argument remains valid only until that holder is + * reused again. The block can be skipped without decompression, decompressed lazily, or copied + * directly to a compatible Avro writer. + */ + public AvroRawBlock nextRawBlock(@Nullable AvroRawBlock reuse) throws IOException { + RawBlock block = + replaceAvroRuntimeException( + () -> reader.nextRawBlock(reuse == null ? null : reuse.rawBlock())); + AvroRawBlock result = reuse == null ? new AvroRawBlock(block) : reuse.replace(block); + currentBlockRecordCount = result.recordCount(); + return result; + } + /** Returns the record count of the last block returned by a block-reading method. */ public long currentBlockRecordCount() { if (currentBlockRecordCount < 0) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java new file mode 100644 index 000000000000..da55bb7c599b --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBlockWriter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.avro; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.PositionOutputStream; + +import org.apache.avro.file.DataFileWriter; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** Avro writer which accepts normal rows, encoded records and compressed blocks. */ +public final class AvroBlockWriter implements AutoCloseable { + + private final DataFileWriter writer; + private final PositionOutputStream out; + + public AvroBlockWriter(DataFileWriter writer, PositionOutputStream out) { + this.writer = writer; + this.out = out; + } + + public void addElement(InternalRow element) throws IOException { + writer.append(element); + } + + public void addEncoded(ByteBuffer record) throws IOException { + writer.appendEncoded(record); + } + + public void addEncodedBlock(AvroRawBlock block) throws IOException { + writer.appendAllFrom(block.asStream(), false); + } + + public boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws IOException { + return suggestedCheck && out.getPos() >= targetSize; + } + + @Override + public void close() throws IOException { + writer.close(); + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 43105fd1cd51..c459e6cee329 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -25,6 +25,7 @@ import org.apache.paimon.format.FormatWriter; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.fs.CloseShieldOutputStream; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; @@ -85,6 +86,26 @@ public FormatWriterFactory createWriterFactory(RowType type) { return new RowAvroWriterFactory(type); } + public Schema createAvroSchema(RowType rowType) { + return AvroSchemaConverter.convertToSchema(rowType, options.get(AVRO_ROW_NAME_MAPPING)); + } + + public boolean supportsRawBlockCopy(RowType rowType, Schema encodedSchema) { + return AvroSchemaConverter.convertToSchema(rowType, options.get(AVRO_ROW_NAME_MAPPING)) + .equals(encodedSchema); + } + + public AvroBlockWriter createBlockWriter( + PositionOutputStream out, RowType rowType, String compression) throws IOException { + Schema schema = createAvroSchema(rowType); + AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType); + DataFileWriter writer = new DataFileWriter<>(datumWriter); + writer.setCodec(createCodecFactory(compression)); + writer.setFlushOnEveryBlock(false); + writer.create(schema, new CloseShieldOutputStream(out)); + return new AvroBlockWriter(writer, out); + } + @Override public Optional createStatsExtractor( RowType type, SimpleColStatsCollector.Factory[] statsCollectors) { @@ -119,9 +140,7 @@ private RowAvroWriterFactory(RowType rowType) { this.factory = new AvroWriterFactory<>( (out, compression) -> { - Schema schema = - AvroSchemaConverter.convertToSchema( - rowType, options.get(AVRO_ROW_NAME_MAPPING)); + Schema schema = createAvroSchema(rowType); AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType); DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java new file mode 100644 index 000000000000..a607423ce9b6 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRawBlock.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.avro; + +import org.apache.avro.file.DataFileStream; +import org.apache.avro.file.RawBlock; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** Reusable compressed block from an Avro object container file. */ +public final class AvroRawBlock { + + private RawBlock block; + + AvroRawBlock(RawBlock block) { + this.block = block; + } + + AvroRawBlock replace(RawBlock block) { + this.block = block; + return this; + } + + RawBlock rawBlock() { + return block; + } + + public long recordCount() { + return block.recordCount(); + } + + /** Returns the compressed payload size. This block must not have been decompressed. */ + public int compressedSize() { + return block.compressedSize(); + } + + /** + * Lazily decompresses this block, reusing the supplied heap buffer when possible. + * + *

      The returned view remains owned by this block and is invalidated when this holder is + * reused for another block. + */ + public ByteBuffer decompress(@Nullable ByteBuffer reuse) throws IOException { + return block.decompress(reuse); + } + + DataFileStream asStream() throws IOException { + return block.asStream(); + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRecordDecoder.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRecordDecoder.java index 78b494189792..fe0c728bf931 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRecordDecoder.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRecordDecoder.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; /** * Decoder for sequentially reading records from Avro blocks without exposing Avro classes to @@ -38,6 +39,9 @@ public final class AvroRecordDecoder { private final int recordBranch; private @Nullable BinaryDecoder decoder; + private @Nullable ByteBuffer borrowedView; + private int blockOffset; + private int blockLength; AvroRecordDecoder(Schema writerSchema) { if (writerSchema.getType() == Schema.Type.UNION) { @@ -84,15 +88,37 @@ public FieldType fieldType(int position) { /** Creates a decoder for one writer field. */ public FieldDecoder createFieldDecoder(int position, @Nullable DataType readType) { - FieldReader reader = - new FieldReaderFactory() - .visit(recordSchema.getFields().get(position).schema(), readType); - return new FieldDecoder(reader); + return createFieldDecoder(recordSchema.getFields().get(position).schema(), readType); + } + + /** Creates a decoder for the supplied writer field schema. */ + public FieldDecoder createFieldDecoder(Schema fieldSchema, @Nullable DataType readType) { + return new FieldDecoder(new FieldReaderFactory().visit(fieldSchema, readType)); } /** Reuses this decoder for another block. */ public void reset(byte[] bytes, int offset, int length) { decoder = DecoderFactory.get().binaryDecoder(bytes, offset, length, decoder); + blockOffset = offset; + blockLength = length; + if (borrowedView == null || borrowedView.array() != bytes) { + borrowedView = ByteBuffer.wrap(bytes); + } + } + + /** Reuses this decoder for another decompressed block. */ + public void reset(ByteBuffer block) { + int position = block.position(); + int length = block.remaining(); + if (block.hasArray()) { + reset(block.array(), block.arrayOffset() + position, length); + return; + } + + byte[] bytes = new byte[length]; + block.get(bytes); + block.position(position); + reset(bytes, 0, length); } /** Returns whether a block has been supplied through {@link #reset(byte[], int, int)}. */ @@ -114,6 +140,76 @@ public int readInt() throws IOException { return decoder().readInt(); } + public boolean readBoolean() throws IOException { + return decoder().readBoolean(); + } + + public long readLong() throws IOException { + return decoder().readLong(); + } + + public int readIndex() throws IOException { + return decoder().readIndex(); + } + + public void skipFixed(int length) throws IOException { + decoder().skipFixed(length); + } + + /** Returns the byte position relative to the beginning of the current block. */ + public int position() throws IOException { + return blockLength - decoder().inputStream().available(); + } + + /** Returns the absolute byte position in the current block's backing array. */ + public int absolutePosition() throws IOException { + return blockOffset + position(); + } + + /** + * Reads an Avro byte sequence and returns a borrowed view of its payload. + * + *

      The returned object is reused by this decoder and remains valid only until the next method + * which returns a borrowed view is called. Callers which need to retain the range should copy + * its array, offset and length rather than retaining the {@link ByteBuffer} object. + */ + public ByteBuffer readBytesView() throws IOException { + long length = readLong(); + if (length < 0 || length > Integer.MAX_VALUE) { + throw new IOException("Invalid Avro byte sequence length " + length); + } + int start = absolutePosition(); + if (length > blockOffset + blockLength - start) { + throw new IOException("Avro byte sequence exceeds the current block."); + } + int end = start + (int) length; + ByteBuffer result = borrowedView(start, end); + skipFixed((int) length); + return result; + } + + /** + * Returns a borrowed view of an absolute range in the current block's backing array. + * + *

      The returned object follows the same reuse contract as {@link #readBytesView()}. + */ + public ByteBuffer borrowedView(int start, int end) { + if (borrowedView == null) { + throw new IllegalStateException("No Avro block has been supplied."); + } + int blockEnd = blockOffset + blockLength; + if (start < blockOffset || end < start || end > blockEnd) { + throw new IllegalArgumentException( + String.format( + "Borrowed Avro byte range [%s, %s) is outside block range [%s, %s).", + start, end, blockOffset, blockEnd)); + } + borrowedView.clear(); + borrowedView.position(start); + borrowedView.limit(end); + return borrowedView; + } + public byte[] readBytes() throws IOException { return decoder().readBytes(null).array(); } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index b8cb87f1c42c..b20e43cba9a2 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -50,6 +50,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.NoSuchElementException; @@ -251,6 +252,69 @@ void testReadBorrowedBlocks() throws IOException { assertThat(nextValue).isEqualTo(numRecords); } + @Test + void testReadBorrowedRawBlocks() throws IOException { + RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull(); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + int numRecords = 100_000; + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = fileFormat.createWriterFactory(rowType).create(out, "zstd"); + for (int i = 0; i < numRecords; i++) { + writer.addElement(GenericRow.of(i)); + } + writer.close(); + } + + long records = 0; + int blocks = 0; + AvroRawBlock previous = null; + try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) { + while (reader.hasNextBlock()) { + AvroRawBlock block = reader.nextBorrowedRawBlock(); + if (previous != null) { + assertThat(block).isSameAs(previous); + } + previous = block; + records += block.recordCount(); + blocks++; + } + assertThatThrownBy(reader::nextBorrowedRawBlock) + .isInstanceOf(NoSuchElementException.class); + } + + assertThat(blocks).isGreaterThan(1); + assertThat(records).isEqualTo(numRecords); + } + + @Test + void testRecordDecoderReturnsReusedBorrowedByteViews() throws IOException { + RowType rowType = + DataTypes.ROW(DataTypes.BYTES().notNull(), DataTypes.BYTES().notNull()).notNull(); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = fileFormat.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(new byte[] {1, 2}, new byte[] {3, 4, 5})); + writer.close(); + } + + try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) { + AvroRecordDecoder decoder = reader.createRecordDecoder(); + decoder.reset(reader.nextRawBlock(null).decompress(null)); + assertThat(decoder.readRecordStart()).isTrue(); + + ByteBuffer first = decoder.readBytesView(); + assertThat(bytes(first)).containsExactly(1, 2); + ByteBuffer second = decoder.readBytesView(); + assertThat(second).isSameAs(first); + assertThat(bytes(second)).containsExactly(3, 4, 5); + assertThat(decoder.isEnd()).isTrue(); + } + } + @Test void testRowReaderProjectsIntoReusedRow() throws IOException { Schema writerSchema = @@ -283,6 +347,36 @@ void testRowReaderProjectsIntoReusedRow() throws IOException { assertThat(decoder.isEnd()).isTrue(); } + @Test + void testReadsLargeZstdBlock() throws IOException { + RowType rowType = + RowType.builder() + .field("payload", DataTypes.VARBINARY(500_000).notNull()) + .field("id", DataTypes.INT().notNull()) + .build(); + AvroFileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO fileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + byte[] payload = new byte[400_000]; + Arrays.fill(payload, (byte) 7); + + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(payload, 42)); + writer.close(); + } + + try (AvroBlockReader blockReader = new AvroBlockReader(fileIO.newInputStream(file))) { + AvroRawBlock block = blockReader.nextRawBlock(null); + assertThat(block.recordCount()).isEqualTo(1); + assertThat(block.compressedSize()).isPositive(); + ByteBuffer decoded = block.decompress(null); + assertThat(decoded.remaining()).isGreaterThan(payload.length); + assertThat(decoded.get(10)).isEqualTo((byte) 7); + assertThat(block.decompress(ByteBuffer.allocate(1))).isEqualTo(decoded); + } + } + @Test void testGetRealIOException() throws IOException { RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()); @@ -359,4 +453,10 @@ void testCompression() throws IOException { .hasMessageContaining("Unrecognized codec: unsupported"); } } + + private static byte[] bytes(ByteBuffer buffer) { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + return bytes; + } }