Skip to content

[Fix] Address aligned memory accounting Sonar issues #18372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 1 commit into
base: master
Choose a base branch
from
+42 −38
Open

[Fix] Address aligned memory accounting Sonar issues #18372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 20 additions & 17 deletions .../src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,9 +313,9 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics)

ensureMemTable(infoForMetrics);
workMemTable.checkDataType(insertRowNode);
AlignedTVListRamCostSnapshot alignedRamCostSnapshot =
AlignedTvListRamCostSnapshot alignedRamCostSnapshot =
insertRowNode.isAligned()
? new AlignedTVListRamCostSnapshot(workMemTable, insertRowNode.getDeviceID())
? new AlignedTvListRamCostSnapshot(workMemTable, insertRowNode.getDeviceID())
: null;

long[] memIncrements;
Expand Down Expand Up @@ -391,7 +391,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics)
pointInserted = workMemTable.insert(insertRowNode);
}
} finally {
reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]);
reconcileAlignedTvListRamCost(alignedRamCostSnapshot, memIncrements[0]);
}

// Update start time of this memtable
Expand Down Expand Up @@ -423,10 +423,10 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics)
alignedDeviceIds.add(insertRowNode.getDeviceID());
}
}
AlignedTVListRamCostSnapshot alignedRamCostSnapshot =
AlignedTvListRamCostSnapshot alignedRamCostSnapshot =
alignedDeviceIds.isEmpty()
? null
: new AlignedTVListRamCostSnapshot(workMemTable, alignedDeviceIds);
: new AlignedTvListRamCostSnapshot(workMemTable, alignedDeviceIds);

long memControlStartTime = System.nanoTime();
if (insertRowsNode.isMixingAlignment()) {
Expand Down Expand Up @@ -523,7 +523,7 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics)
}
}
} finally {
reconcileAlignedTVListRamCost(alignedRamCostSnapshot, alignedMemTableIncrement);
reconcileAlignedTvListRamCost(alignedRamCostSnapshot, alignedMemTableIncrement);
}

tsFileResource.updateProgressIndex(insertRowsNode.getProgressIndex());
Expand Down Expand Up @@ -638,6 +638,7 @@ private long[] checkAlignedMemCost(
* @param rangeList start and end index list of rows to be inserted in insertTabletPlan
* @param results result array
*/
@SuppressWarnings("java:S6541") // Keep ordered write-path state updates together.
public void insertTablet(
InsertTabletNode insertTabletNode,
List<int[]> rangeList,
Expand All @@ -657,10 +658,10 @@ public void insertTablet(
}
}
}
AlignedTVListRamCostSnapshot alignedRamCostSnapshot =
AlignedTvListRamCostSnapshot alignedRamCostSnapshot =
alignedDeviceIds.isEmpty()
? null
: new AlignedTVListRamCostSnapshot(workMemTable, alignedDeviceIds);
: new AlignedTvListRamCostSnapshot(workMemTable, alignedDeviceIds);

long[] memIncrements =
scheduleMemoryBlock(insertTabletNode, rangeList, results, infoForMetrics);
Expand Down Expand Up @@ -764,7 +765,7 @@ public void insertTablet(
}
}
} finally {
reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]);
reconcileAlignedTvListRamCost(alignedRamCostSnapshot, memIncrements[0]);
}
tsFileResource.updateProgressIndex(insertTabletNode.getProgressIndex());

Expand Down Expand Up @@ -933,7 +934,8 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow(
return new long[] {memTableIncrement, textDataIncrement, chunkMetadataIncrement};
}

@SuppressWarnings("squid:S3776") // high Cognitive Complexity
// This estimator is one stateful pass over the incoming aligned rows.
@SuppressWarnings({"squid:S3776", "java:S6541"})
private long[] checkAlignedMemCostAndAddToTspInfoForRows(List<InsertRowNode> insertRowNodeList)
throws WriteProcessException {
// Fixed-size TVList structures and materialized value primitive arrays.
Expand Down Expand Up @@ -1260,6 +1262,7 @@ private void updateAlignedMemCost(
* when the tablet has at least one successful non-null value in that block and the working TVList
* has not already allocated its value array.
*/
@SuppressWarnings("java:S107") // Parameters mirror the tablet write representation.
private static long calculateTabletValueArrayMemCost(
AlignedWritableMemChunk alignedMemChunk,
String[] measurementIds,
Expand Down Expand Up @@ -1376,8 +1379,8 @@ private static boolean isFieldMeasurement(
&& columnCategories[index] == TsTableColumnCategory.FIELD);
}

private void reconcileAlignedTVListRamCost(
AlignedTVListRamCostSnapshot snapshot, long estimatedMemTableIncrement) {
private void reconcileAlignedTvListRamCost(
AlignedTvListRamCostSnapshot snapshot, long estimatedMemTableIncrement) {
if (snapshot == null) {
return;
}
Expand All @@ -1394,21 +1397,21 @@ private void reconcileAlignedTVListRamCost(
}
}

static final class AlignedTVListRamCostSnapshot {
static final class AlignedTvListRamCostSnapshot {

private final IMemTable memTable;
private final IDeviceID deviceId;
private final Set<IDeviceID> deviceIds;
private final long ramCostBeforeWrite;

AlignedTVListRamCostSnapshot(IMemTable memTable, IDeviceID deviceId) {
AlignedTvListRamCostSnapshot(IMemTable memTable, IDeviceID deviceId) {
this.memTable = memTable;
this.deviceId = deviceId;
this.deviceIds = null;
this.ramCostBeforeWrite = getRamCost(memTable, deviceId);
}

AlignedTVListRamCostSnapshot(IMemTable memTable, Set<IDeviceID> deviceIds) {
AlignedTvListRamCostSnapshot(IMemTable memTable, Set<IDeviceID> deviceIds) {
this.memTable = memTable;
this.deviceId = null;
this.deviceIds = deviceIds;
Expand Down Expand Up @@ -1438,8 +1441,8 @@ private static long getRamCost(IMemTable memTable, IDeviceID deviceId) {

AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk;
long ramCost = alignedMemChunk.getWorkingTVList().getRamSize();
for (AlignedTVList sortedTVList : alignedMemChunk.getSortedList()) {
ramCost += sortedTVList.getRamSize();
for (AlignedTVList sortedTvList : alignedMemChunk.getSortedList()) {
ramCost += sortedTvList.getRamSize();
}
return ramCost;
}
Expand Down
39 changes: 20 additions & 19 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ public TVList getTvListByColumnIndex(
int materializedArrayCount = materializedValueArrayCounts[columnIndex];
alignedTvList.materializedValueArrayCounts[i] = materializedArrayCount;
alignedTvList.materializedValueArrayMemCost +=
(long) materializedArrayCount * valueListArrayMemCost(dataTypeList.get(i));
materializedArrayCount * valueListArrayMemCost(dataTypeList.get(i));
}
}

Expand Down Expand Up @@ -435,7 +435,7 @@ public void extendColumn(TSDataType dataType) {
columnBitMaps.add(bitMap);
}
materializedBitmapMemoryCost +=
(long) timestamps.size() * (bitmapReferenceRamCost() + bitmapRamCost());
timestamps.size() * (bitmapReferenceRamCost() + bitmapRamCost());
this.bitMaps.add(columnBitMaps);
this.values.add(columnValue);
this.dataTypes.add(dataType);
Expand Down Expand Up @@ -736,7 +736,7 @@ public void deleteColumn(int columnIndex) {
}
bitMaps.set(columnIndex, columnBitMaps);
materializedBitmapMemoryCost +=
(long) columnBitMaps.size() * (bitmapReferenceRamCost() + bitmapRamCost());
columnBitMaps.size() * (bitmapReferenceRamCost() + bitmapRamCost());
}
for (int i = 0; i < bitMaps.get(columnIndex).size(); i++) {
if (bitMaps.get(columnIndex).get(i) == null) {
Expand Down Expand Up @@ -1125,7 +1125,7 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) {
columnBitMaps.add(null);
}
bitMaps.set(columnIndex, columnBitMaps);
materializedBitmapMemoryCost += (long) columnBitMaps.size() * bitmapReferenceRamCost();
materializedBitmapMemoryCost += columnBitMaps.size() * bitmapReferenceRamCost();
}

// if the bitmap in arrayIndex is null, init the bitmap
Expand Down Expand Up @@ -1164,7 +1164,7 @@ public synchronized RamInfo calculateRamSize() {
}

public synchronized long getRamSize() {
return (long) timestamps.size() * alignedTvListArrayMemCostWithoutPrimitiveArrays()
return timestamps.size() * alignedTvListArrayMemCostWithoutPrimitiveArrays()
+ materializedValueArrayMemCost
+ materializedBitmapMemoryCost;
}
Expand All @@ -1178,7 +1178,7 @@ private static long calculateBitmapRamCost(List<List<BitMap>> bitMaps) {
if (columnBitMaps == null) {
continue;
}
size += (long) columnBitMaps.size() * bitmapReferenceRamCost();
size += columnBitMaps.size() * bitmapReferenceRamCost();
for (BitMap bitMap : columnBitMaps) {
if (bitMap != null) {
size += bitMap.ramBytesUsed();
Expand Down Expand Up @@ -1255,19 +1255,6 @@ public long alignedTvListArrayMemCostWithoutPrimitiveArrays() {
+ (indices != null ? (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES : 0);
}

private void refreshArrayMemCostWithoutPrimitiveArrays() {
long size = alignedTvListArrayMemCost();
if (indices != null) {
size -= (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES;
}
for (TSDataType dataType : dataTypes) {
if (dataType != null) {
size -= valueListArrayMemCost(dataType);
}
}
arrayMemCostWithoutPrimitiveArraysAndIndex = size;
}

public static long alignedTvListArrayMemCostWithoutPrimitiveArrays(
TSDataType[] types, TsTableColumnCategory[] columnCategories) {
long size = alignedTvListArrayMemCost(types, columnCategories);
Expand All @@ -1281,6 +1268,19 @@ public static long alignedTvListArrayMemCostWithoutPrimitiveArrays(
return size;
}

private void refreshArrayMemCostWithoutPrimitiveArrays() {
long size = alignedTvListArrayMemCost();
if (indices != null) {
size -= (long) PrimitiveArrayManager.ARRAY_SIZE * Integer.BYTES;
}
for (TSDataType dataType : dataTypes) {
if (dataType != null) {
size -= valueListArrayMemCost(dataType);
}
}
arrayMemCostWithoutPrimitiveArraysAndIndex = size;
}

/**
* Get the single column array mem cost by give type.
*
Expand Down Expand Up @@ -2222,6 +2222,7 @@ private LazyBitMap markRowAsInvalid(
}

@Override
@SuppressWarnings("java:S6541") // The scan passes share hot-path cursor state.
public TsBlock nextBatch() {
int maxRowCountOfCurrentBatch = Math.min(rows - index, maxNumberOfPointsInPage);
TsBlockBuilder builder = new TsBlockBuilder(maxRowCountOfCurrentBatch, dataTypeList);
Expand Down
4 changes: 2 additions & 2 deletions ...db/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,8 @@ private static Measurement measureWrite(Scenario scenario, int iterations) {
private static void runReconciliation(AccountingTarget target, int iterations) {
long correction = 0;
for (int i = 0; i < iterations; i++) {
TsFileProcessor.AlignedTVListRamCostSnapshot snapshot =
new TsFileProcessor.AlignedTVListRamCostSnapshot(target.memTable, target.deviceId);
TsFileProcessor.AlignedTvListRamCostSnapshot snapshot =
new TsFileProcessor.AlignedTvListRamCostSnapshot(target.memTable, target.deviceId);
correction += snapshot.getMemoryCorrection(0);
}
benchmarkBlackhole = correction + iterations;
Expand Down
Loading