From bf667bb4109eb51b7bff81377ab2001476de203c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:37:10 +0800 Subject: [PATCH] [Fix] Address aligned memory accounting Sonar issues --- .../dataregion/memtable/TsFileProcessor.java | 37 ++++++++++-------- .../db/utils/datastructure/AlignedTVList.java | 39 ++++++++++--------- ...BitmapMemoryAccountingPerformanceTest.java | 4 +- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index ee9b39d2ffe4..ad88f7e8ee09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -313,9 +313,9 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) ensureMemTable(infoForMetrics); workMemTable.checkDataType(insertRowNode); - AlignedTVListRamCostSnapshot alignedRamCostSnapshot = + AlignedTvListRamCostSnapshot alignedRamCostSnapshot = insertRowNode.isAligned() - ? new AlignedTVListRamCostSnapshot(workMemTable, insertRowNode.getDeviceID()) + ? new AlignedTvListRamCostSnapshot(workMemTable, insertRowNode.getDeviceID()) : null; long[] memIncrements; @@ -391,7 +391,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) pointInserted = workMemTable.insert(insertRowNode); } } finally { - reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]); + reconcileAlignedTvListRamCost(alignedRamCostSnapshot, memIncrements[0]); } // Update start time of this memtable @@ -423,10 +423,10 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics) alignedDeviceIds.add(insertRowNode.getDeviceID()); } } - AlignedTVListRamCostSnapshot alignedRamCostSnapshot = + AlignedTvListRamCostSnapshot alignedRamCostSnapshot = alignedDeviceIds.isEmpty() ? null - : new AlignedTVListRamCostSnapshot(workMemTable, alignedDeviceIds); + : new AlignedTvListRamCostSnapshot(workMemTable, alignedDeviceIds); long memControlStartTime = System.nanoTime(); if (insertRowsNode.isMixingAlignment()) { @@ -523,7 +523,7 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics) } } } finally { - reconcileAlignedTVListRamCost(alignedRamCostSnapshot, alignedMemTableIncrement); + reconcileAlignedTvListRamCost(alignedRamCostSnapshot, alignedMemTableIncrement); } tsFileResource.updateProgressIndex(insertRowsNode.getProgressIndex()); @@ -638,6 +638,7 @@ private long[] checkAlignedMemCost( * @param rangeList start and end index list of rows to be inserted in insertTabletPlan * @param results result array */ + @SuppressWarnings("java:S6541") // Keep ordered write-path state updates together. public void insertTablet( InsertTabletNode insertTabletNode, List rangeList, @@ -657,10 +658,10 @@ public void insertTablet( } } } - AlignedTVListRamCostSnapshot alignedRamCostSnapshot = + AlignedTvListRamCostSnapshot alignedRamCostSnapshot = alignedDeviceIds.isEmpty() ? null - : new AlignedTVListRamCostSnapshot(workMemTable, alignedDeviceIds); + : new AlignedTvListRamCostSnapshot(workMemTable, alignedDeviceIds); long[] memIncrements = scheduleMemoryBlock(insertTabletNode, rangeList, results, infoForMetrics); @@ -764,7 +765,7 @@ public void insertTablet( } } } finally { - reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]); + reconcileAlignedTvListRamCost(alignedRamCostSnapshot, memIncrements[0]); } tsFileResource.updateProgressIndex(insertTabletNode.getProgressIndex()); @@ -933,7 +934,8 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( return new long[] {memTableIncrement, textDataIncrement, chunkMetadataIncrement}; } - @SuppressWarnings("squid:S3776") // high Cognitive Complexity + // This estimator is one stateful pass over the incoming aligned rows. + @SuppressWarnings({"squid:S3776", "java:S6541"}) private long[] checkAlignedMemCostAndAddToTspInfoForRows(List insertRowNodeList) throws WriteProcessException { // Fixed-size TVList structures and materialized value primitive arrays. @@ -1260,6 +1262,7 @@ private void updateAlignedMemCost( * when the tablet has at least one successful non-null value in that block and the working TVList * has not already allocated its value array. */ + @SuppressWarnings("java:S107") // Parameters mirror the tablet write representation. private static long calculateTabletValueArrayMemCost( AlignedWritableMemChunk alignedMemChunk, String[] measurementIds, @@ -1376,8 +1379,8 @@ private static boolean isFieldMeasurement( && columnCategories[index] == TsTableColumnCategory.FIELD); } - private void reconcileAlignedTVListRamCost( - AlignedTVListRamCostSnapshot snapshot, long estimatedMemTableIncrement) { + private void reconcileAlignedTvListRamCost( + AlignedTvListRamCostSnapshot snapshot, long estimatedMemTableIncrement) { if (snapshot == null) { return; } @@ -1394,21 +1397,21 @@ private void reconcileAlignedTVListRamCost( } } - static final class AlignedTVListRamCostSnapshot { + static final class AlignedTvListRamCostSnapshot { private final IMemTable memTable; private final IDeviceID deviceId; private final Set deviceIds; private final long ramCostBeforeWrite; - AlignedTVListRamCostSnapshot(IMemTable memTable, IDeviceID deviceId) { + AlignedTvListRamCostSnapshot(IMemTable memTable, IDeviceID deviceId) { this.memTable = memTable; this.deviceId = deviceId; this.deviceIds = null; this.ramCostBeforeWrite = getRamCost(memTable, deviceId); } - AlignedTVListRamCostSnapshot(IMemTable memTable, Set deviceIds) { + AlignedTvListRamCostSnapshot(IMemTable memTable, Set deviceIds) { this.memTable = memTable; this.deviceId = null; this.deviceIds = deviceIds; @@ -1438,8 +1441,8 @@ private static long getRamCost(IMemTable memTable, IDeviceID deviceId) { AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; long ramCost = alignedMemChunk.getWorkingTVList().getRamSize(); - for (AlignedTVList sortedTVList : alignedMemChunk.getSortedList()) { - ramCost += sortedTVList.getRamSize(); + for (AlignedTVList sortedTvList : alignedMemChunk.getSortedList()) { + ramCost += sortedTvList.getRamSize(); } return ramCost; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index a589e1ea97a6..5eac3fa492a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -184,7 +184,7 @@ public TVList getTvListByColumnIndex( int materializedArrayCount = materializedValueArrayCounts[columnIndex]; alignedTvList.materializedValueArrayCounts[i] = materializedArrayCount; alignedTvList.materializedValueArrayMemCost += - (long) materializedArrayCount * valueListArrayMemCost(dataTypeList.get(i)); + materializedArrayCount * valueListArrayMemCost(dataTypeList.get(i)); } } @@ -435,7 +435,7 @@ public void extendColumn(TSDataType dataType) { columnBitMaps.add(bitMap); } materializedBitmapMemoryCost += - (long) timestamps.size() * (bitmapReferenceRamCost() + bitmapRamCost()); + timestamps.size() * (bitmapReferenceRamCost() + bitmapRamCost()); this.bitMaps.add(columnBitMaps); this.values.add(columnValue); this.dataTypes.add(dataType); @@ -736,7 +736,7 @@ public void deleteColumn(int columnIndex) { } bitMaps.set(columnIndex, columnBitMaps); materializedBitmapMemoryCost += - (long) columnBitMaps.size() * (bitmapReferenceRamCost() + bitmapRamCost()); + columnBitMaps.size() * (bitmapReferenceRamCost() + bitmapRamCost()); } for (int i = 0; i < bitMaps.get(columnIndex).size(); i++) { if (bitMaps.get(columnIndex).get(i) == null) { @@ -1125,7 +1125,7 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) { columnBitMaps.add(null); } bitMaps.set(columnIndex, columnBitMaps); - materializedBitmapMemoryCost += (long) columnBitMaps.size() * bitmapReferenceRamCost(); + materializedBitmapMemoryCost += columnBitMaps.size() * bitmapReferenceRamCost(); } // if the bitmap in arrayIndex is null, init the bitmap @@ -1164,7 +1164,7 @@ public synchronized RamInfo calculateRamSize() { } public synchronized long getRamSize() { - return (long) timestamps.size() * alignedTvListArrayMemCostWithoutPrimitiveArrays() + return timestamps.size() * alignedTvListArrayMemCostWithoutPrimitiveArrays() + materializedValueArrayMemCost + materializedBitmapMemoryCost; } @@ -1178,7 +1178,7 @@ private static long calculateBitmapRamCost(List> bitMaps) { if (columnBitMaps == null) { continue; } - size += (long) columnBitMaps.size() * bitmapReferenceRamCost(); + size += columnBitMaps.size() * bitmapReferenceRamCost(); for (BitMap bitMap : columnBitMaps) { if (bitMap != null) { size += bitMap.ramBytesUsed(); @@ -1255,19 +1255,6 @@ public long alignedTvListArrayMemCostWithoutPrimitiveArrays() { + (indices != null ? (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES : 0); } - private void refreshArrayMemCostWithoutPrimitiveArrays() { - long size = alignedTvListArrayMemCost(); - if (indices != null) { - size -= (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES; - } - for (TSDataType dataType : dataTypes) { - if (dataType != null) { - size -= valueListArrayMemCost(dataType); - } - } - arrayMemCostWithoutPrimitiveArraysAndIndex = size; - } - public static long alignedTvListArrayMemCostWithoutPrimitiveArrays( TSDataType[] types, TsTableColumnCategory[] columnCategories) { long size = alignedTvListArrayMemCost(types, columnCategories); @@ -1281,6 +1268,19 @@ public static long alignedTvListArrayMemCostWithoutPrimitiveArrays( return size; } + private void refreshArrayMemCostWithoutPrimitiveArrays() { + long size = alignedTvListArrayMemCost(); + if (indices != null) { + size -= (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES; + } + for (TSDataType dataType : dataTypes) { + if (dataType != null) { + size -= valueListArrayMemCost(dataType); + } + } + arrayMemCostWithoutPrimitiveArraysAndIndex = size; + } + /** * Get the single column array mem cost by give type. * @@ -2222,6 +2222,7 @@ private LazyBitMap markRowAsInvalid( } @Override + @SuppressWarnings("java:S6541") // The scan passes share hot-path cursor state. public TsBlock nextBatch() { int maxRowCountOfCurrentBatch = Math.min(rows - index, maxNumberOfPointsInPage); TsBlockBuilder builder = new TsBlockBuilder(maxRowCountOfCurrentBatch, dataTypeList); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java index 8a872f6af61e..6915b939ade5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java @@ -132,8 +132,8 @@ private static Measurement measureWrite(Scenario scenario, int iterations) { private static void runReconciliation(AccountingTarget target, int iterations) { long correction = 0; for (int i = 0; i < iterations; i++) { - TsFileProcessor.AlignedTVListRamCostSnapshot snapshot = - new TsFileProcessor.AlignedTVListRamCostSnapshot(target.memTable, target.deviceId); + TsFileProcessor.AlignedTvListRamCostSnapshot snapshot = + new TsFileProcessor.AlignedTvListRamCostSnapshot(target.memTable, target.deviceId); correction += snapshot.getMemoryCorrection(0); } benchmarkBlackhole = correction + iterations;