From 165d88c8d31acfb7fc016549e6c0e6025b182dcd Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sat, 26 Sep 2026 20:17:21 +0800 Subject: [PATCH 1/3] [core] Merge stored bitmap32 deletion vectors after enabling bitmap64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flipping deletion-vectors.bitmap64 on a table with existing deletion vectors leaves bitmap32 vectors in the index files — a state the Iceberg callback already acknowledges. The append maintainer merged the stored vector into the fresh bitmap64 one unconditionally, which threw for the mixed types and crashed every subsequent delete on those files. Convert a stored bitmap32 vector to bitmap64 before merging. Assisted-by: GLM-5.3 --- .../append/AppendDeleteFileMaintainer.java | 13 +++++- .../AppendDeletionFileMaintainerTest.java | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java index 93ea8a7676a8..96799573b6fe 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java @@ -20,6 +20,8 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; import org.apache.paimon.fs.Path; @@ -131,7 +133,16 @@ public DeletionFile notifyRemovedDeletionVector(String dataFile) { public void notifyNewDeletionVector(String dataFile, DeletionVector deletionVector) { DeletionFile previous = notifyRemovedDeletionVector(dataFile); if (previous != null) { - deletionVector.merge(dvIndexFile.readDeletionVector(previous)); + DeletionVector stored = dvIndexFile.readDeletionVector(previous); + // a stored vector may predate flipping deletion-vectors.bitmap64: normalize it + // to the new vector's type so the merge does not crash on mixed state + if (stored instanceof BitmapDeletionVector + && deletionVector instanceof Bitmap64DeletionVector) { + stored = + Bitmap64DeletionVector.fromBitmapDeletionVector( + (BitmapDeletionVector) stored); + } + deletionVector.merge(stored); } deletionVectors.put(dataFile, deletionVector); } diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java index 3a03985c885d..4f6da362e1d1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.TestAppendFileStore; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.local.LocalFileIO; @@ -32,6 +33,7 @@ import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.table.source.DeletionFile; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -48,6 +50,48 @@ class AppendDeletionFileMaintainerTest { @TempDir java.nio.file.Path tempDir; + @Test + public void testMergeStoredBitmap32IntoBitmap64() throws Exception { + // write DVs as bitmap32, then flip deletion-vectors.bitmap64 on: the next + // notification must merge the stored vector instead of crashing on the type + Map options = new HashMap<>(); + options.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), "false"); + TestAppendFileStore store = TestAppendFileStore.createAppendStore(tempDir, options); + + CommitMessageImpl commitMessage = + store.writeDVIndexFiles( + BinaryRow.EMPTY_ROW, + 0, + Collections.singletonMap("f1", Arrays.asList(1, 3, 5))); + store.commit(commitMessage); + + IndexPathFactory indexPathFactory = + store.pathFactory().indexFileFactory(BinaryRow.EMPTY_ROW, 0); + Map dataFileToDeletionFiles = + createDeletionFileMapFromIndexFileMetas( + indexPathFactory, commitMessage.newFilesIncrement().newIndexFiles()); + + Map flipped = new HashMap<>(); + flipped.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), "true"); + TestAppendFileStore flippedStore = TestAppendFileStore.createAppendStore(tempDir, flipped); + AppendDeleteFileMaintainer dvIFMaintainer = + flippedStore.createDVIFMaintainer(BinaryRow.EMPTY_ROW, dataFileToDeletionFiles); + + Bitmap64DeletionVector fresh = new Bitmap64DeletionVector(); + fresh.delete(7); + dvIFMaintainer.notifyNewDeletionVector("f1", fresh); + + List res = dvIFMaintainer.persist(); + assertThat(res).hasSize(2); + // the old index file is replaced by one holding the merged vector: stored 3 + // deletions plus the fresh one + assertThat(res).anyMatch(entry -> entry.kind() == FileKind.DELETE); + IndexManifestEntry added = + res.stream().filter(entry -> entry.kind() == FileKind.ADD).findAny().get(); + assertThat(added.indexFile().dvRanges()).containsKey("f1"); + assertThat(added.indexFile().dvRanges().get("f1").cardinality()).isEqualTo(4); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void test(boolean bitmap64) throws Exception { From 6b2cc0a1c41de8f1d02a4a5558331afa81b957c4 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 28 Sep 2026 03:13:33 +0800 Subject: [PATCH 2/3] fix: merge append deletion vectors symmetrically across a bitmap64 flip Extract DeletionVector.mergeVectors, promoting the bitmap32 side to bitmap64 whenever either side is bitmap64, so notifyNewDeletionVector no longer throws when a fresh bitmap32 vector meets a stored bitmap64 one after the option is flipped off (the reverse of the case the original fix covered). --- .../deletionvectors/DeletionVector.java | 21 ++++++++++ .../append/AppendDeleteFileMaintainer.java | 12 +----- .../AppendDeletionFileMaintainerTest.java | 41 +++++++++++++++++++ 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVector.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVector.java index d3f7eed1cede..e8c9cba9d42a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVector.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVector.java @@ -197,6 +197,27 @@ static DeletionVector deserializeFromBytes(byte[] bytes) { } } + /** + * Merges {@code stored} into {@code fresh} and returns the vector holding the union. + * + *

{@code deletion-vectors.bitmap64} can be flipped on an existing table, so a freshly + * created vector and a previously stored one may not share the same bitmap width. When they + * differ, the {@link BitmapDeletionVector} side is promoted to {@link Bitmap64DeletionVector} + * so the merge runs at the wider format instead of {@link #merge} throwing on the type + * mismatch. A bitmap64 result stays readable under a bitmap32 table because vectors are + * dispatched by magic number on read. + */ + static DeletionVector mergeVectors(DeletionVector fresh, DeletionVector stored) { + if (fresh instanceof Bitmap64DeletionVector && stored instanceof BitmapDeletionVector) { + stored = Bitmap64DeletionVector.fromBitmapDeletionVector((BitmapDeletionVector) stored); + } else if (fresh instanceof BitmapDeletionVector + && stored instanceof Bitmap64DeletionVector) { + fresh = Bitmap64DeletionVector.fromBitmapDeletionVector((BitmapDeletionVector) fresh); + } + fresh.merge(stored); + return fresh; + } + /** Interface to create {@link DeletionVector}. */ interface Factory { Optional create(String fileName) throws IOException; diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java index 96799573b6fe..496bbc846d0e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/AppendDeleteFileMaintainer.java @@ -20,8 +20,6 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; -import org.apache.paimon.deletionvectors.BitmapDeletionVector; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; import org.apache.paimon.fs.Path; @@ -134,15 +132,7 @@ public void notifyNewDeletionVector(String dataFile, DeletionVector deletionVect DeletionFile previous = notifyRemovedDeletionVector(dataFile); if (previous != null) { DeletionVector stored = dvIndexFile.readDeletionVector(previous); - // a stored vector may predate flipping deletion-vectors.bitmap64: normalize it - // to the new vector's type so the merge does not crash on mixed state - if (stored instanceof BitmapDeletionVector - && deletionVector instanceof Bitmap64DeletionVector) { - stored = - Bitmap64DeletionVector.fromBitmapDeletionVector( - (BitmapDeletionVector) stored); - } - deletionVector.merge(stored); + deletionVector = DeletionVector.mergeVectors(deletionVector, stored); } deletionVectors.put(dataFile, deletionVector); } diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java index 4f6da362e1d1..78fe7a2098b8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.TestAppendFileStore; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.local.LocalFileIO; @@ -92,6 +93,46 @@ public void testMergeStoredBitmap32IntoBitmap64() throws Exception { assertThat(added.indexFile().dvRanges().get("f1").cardinality()).isEqualTo(4); } + @Test + public void testMergeStoredBitmap64IntoBitmap32() throws Exception { + // reverse of the case above: write DVs as bitmap64, then flip deletion-vectors.bitmap64 + // off. A fresh bitmap32 vector must still merge the stored bitmap64 one instead of crashing + Map options = new HashMap<>(); + options.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), "true"); + TestAppendFileStore store = TestAppendFileStore.createAppendStore(tempDir, options); + + CommitMessageImpl commitMessage = + store.writeDVIndexFiles( + BinaryRow.EMPTY_ROW, + 0, + Collections.singletonMap("f1", Arrays.asList(1, 3, 5))); + store.commit(commitMessage); + + IndexPathFactory indexPathFactory = + store.pathFactory().indexFileFactory(BinaryRow.EMPTY_ROW, 0); + Map dataFileToDeletionFiles = + createDeletionFileMapFromIndexFileMetas( + indexPathFactory, commitMessage.newFilesIncrement().newIndexFiles()); + + Map flipped = new HashMap<>(); + flipped.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), "false"); + TestAppendFileStore flippedStore = TestAppendFileStore.createAppendStore(tempDir, flipped); + AppendDeleteFileMaintainer dvIFMaintainer = + flippedStore.createDVIFMaintainer(BinaryRow.EMPTY_ROW, dataFileToDeletionFiles); + + BitmapDeletionVector fresh = new BitmapDeletionVector(); + fresh.delete(7); + dvIFMaintainer.notifyNewDeletionVector("f1", fresh); + + List res = dvIFMaintainer.persist(); + assertThat(res).hasSize(2); + assertThat(res).anyMatch(entry -> entry.kind() == FileKind.DELETE); + IndexManifestEntry added = + res.stream().filter(entry -> entry.kind() == FileKind.ADD).findAny().get(); + assertThat(added.indexFile().dvRanges()).containsKey("f1"); + assertThat(added.indexFile().dvRanges().get("f1").cardinality()).isEqualTo(4); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void test(boolean bitmap64) throws Exception { From 6b22efa842889dc7939a48af89e25291139829d5 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 28 Sep 2026 03:13:40 +0800 Subject: [PATCH 3/3] fix: merge bucketed deletion vectors across a bitmap64 flip BucketedDvMaintainer.mergeNewDeletion (the bucketed-append path via BucketedAppendDeleteFileMaintainer) had the same mixed-type merge crash as the unaware-append path: a fresh option-driven vector was merged into a stored vector of the other type after the option was flipped. Route it through DeletionVector.mergeVectors so the bitmap32 side is promoted first. --- .../deletionvectors/BucketedDvMaintainer.java | 2 +- .../BucketedDvMaintainerTest.java | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BucketedDvMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BucketedDvMaintainer.java index 788ea1cce85d..14a6ec8475de 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BucketedDvMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BucketedDvMaintainer.java @@ -87,7 +87,7 @@ public void notifyNewDeletion(String fileName, DeletionVector deletionVector) { public void mergeNewDeletion(String fileName, DeletionVector deletionVector) { DeletionVector old = deletionVectors.get(fileName); if (old != null) { - deletionVector.merge(old); + deletionVector = DeletionVector.mergeVectors(deletionVector, old); } deletionVectors.put(fileName, deletionVector); modified = true; diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/BucketedDvMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/BucketedDvMaintainerTest.java index 018d55e93304..2dd3c44199a9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/BucketedDvMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/BucketedDvMaintainerTest.java @@ -283,6 +283,52 @@ public void testReadAndWriteMixedDv(boolean bitmap64) { assertThat(dvs.get("f3").getCardinality()).isEqualTo(2); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testMergeNewDeletionAcrossBitmap64Flip(boolean initialBitmap64) { + // write a stored dv under one bitmap64 setting, then flip the option and merge a fresh + // vector of the opposite type into it: the merge must not crash on the type mismatch + initIndexHandler(initialBitmap64); + BucketedDvMaintainer.Factory factory1 = BucketedDvMaintainer.factory(fileHandler); + BucketedDvMaintainer dvMaintainer1 = factory1.create(partition, 0, new HashMap<>()); + dvMaintainer1.notifyNewDeletion("f1", 1); + dvMaintainer1.notifyNewDeletion("f1", 3); + commitIndexFile(dvMaintainer1.writeDeletionVectorsIndex().get()); + + initIndexHandler(!initialBitmap64); + BucketedDvMaintainer.Factory factory2 = BucketedDvMaintainer.factory(fileHandler); + List indexFiles = + fileHandler.scan( + table.latestSnapshot().get(), DELETION_VECTORS_INDEX, partition, 0); + BucketedDvMaintainer dvMaintainer2 = factory2.create(partition, 0, indexFiles); + + DeletionVector fresh = createDeletionVector(!initialBitmap64); + fresh.delete(5); + dvMaintainer2.mergeNewDeletion("f1", fresh); + + DeletionVector merged = dvMaintainer2.deletionVectorOf("f1").get(); + assertThat(merged.isDeleted(1)).isTrue(); + assertThat(merged.isDeleted(3)).isTrue(); + assertThat(merged.isDeleted(5)).isTrue(); + assertThat(merged.getCardinality()).isEqualTo(3); + } + + private void commitIndexFile(IndexFileMeta file) { + CommitMessage commitMessage = + new CommitMessageImpl( + partition, + 0, + 1, + DataIncrement.emptyIncrement(), + new CompactIncrement( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(file), + Collections.emptyList())); + table.newBatchWriteBuilder().newCommit().commit(Collections.singletonList(commitMessage)); + } + private DeletionVector createDeletionVector(boolean bitmap64) { return bitmap64 ? new Bitmap64DeletionVector() : new BitmapDeletionVector(); }