Skip to content

HBASE-30447 Table-level metrics can stop updating until the RegionServer restarts #8730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
virajjasani merged 2 commits into from
Oct 6, 2026
Merged

HBASE-30447 Table-level metrics can stop updating until the RegionServer restarts #8730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
415a83f
HBASE-30447 Table-level metrics can stop updating until the RegionSer…
d-c-manning Oct 1, 2026
ce99f6a
HBASE-30447 Apply spotless formatting
d-c-manning Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 115 additions & 105 deletions ...e-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,117 +275,127 @@ public class HRegionMetricsWrapperRunnable implements Runnable {

@Override
public void run() {
long tempNumStoreFiles = 0;
int tempStoreRefCount = 0;
int tempMaxCompactedStoreFileRefCount = 0;
long tempMemstoreSize = 0;
long tempMemstoreHeapSize = 0;
long tempMemstoreOffHeapSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
long tempNumReferenceFiles = 0;
long tempMaxCompactionQueueSize = 0;
long tempMaxFlushQueueSize = 0;
long avgAgeNumerator = 0;
long numHFiles = 0;
if (region.stores != null) {
for (HStore store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
tempStoreRefCount += currentStoreRefCount;
int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
tempMaxCompactedStoreFileRefCount =
Math.max(tempMaxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount);
final MemStoreSize memStore = store.getMemStoreSize();
tempMemstoreSize += memStore.getDataSize();
tempMemstoreHeapSize += memStore.getHeapSize();
tempMemstoreOffHeapSize += memStore.getOffHeapSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
if (
storeMaxStoreFileAge.isPresent()
&& storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge
) {
tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong();
// The executor only reschedules this task (scheduleWithFixedDelay) if run() returns
// normally. Any Throwable that escapes here (e.g. an NPE from a region closing mid-scan)
// is swallowed by the underlying ScheduledFutureTask with no log output anywhere, and this
// task permanently stops running. Catch everything so a failure is visible AND the next
// scheduled run still happens.
try {
long tempNumStoreFiles = 0;
int tempStoreRefCount = 0;
int tempMaxCompactedStoreFileRefCount = 0;
long tempMemstoreSize = 0;
long tempMemstoreHeapSize = 0;
long tempMemstoreOffHeapSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
long tempNumReferenceFiles = 0;
long tempMaxCompactionQueueSize = 0;
long tempMaxFlushQueueSize = 0;
long avgAgeNumerator = 0;
long numHFiles = 0;
if (region.stores != null) {
for (HStore store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
tempStoreRefCount += currentStoreRefCount;
int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
tempMaxCompactedStoreFileRefCount =
Math.max(tempMaxCompactedStoreFileRefCount, currentMaxCompactedStoreFileRefCount);
final MemStoreSize memStore = store.getMemStoreSize();
tempMemstoreSize += memStore.getDataSize();
tempMemstoreHeapSize += memStore.getHeapSize();
tempMemstoreOffHeapSize += memStore.getOffHeapSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
if (
storeMaxStoreFileAge.isPresent()
&& storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge
) {
tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong();
}

OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge();
if (
storeMinStoreFileAge.isPresent()
&& storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge
) {
tempMinStoreFileAge = storeMinStoreFileAge.getAsLong();
}

long storeHFiles = store.getNumHFiles();
numHFiles += storeHFiles;
tempNumReferenceFiles += store.getNumReferenceFiles();

OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
if (storeAvgStoreFileAge.isPresent()) {
avgAgeNumerator += (long) storeAvgStoreFileAge.getAsDouble() * storeHFiles;
}
if (mixedReadsOnStore == null) {
mixedReadsOnStore = new HashMap<String, Long>();
}
Long tempVal = mixedReadsOnStore.get(store.getColumnFamilyName());
if (tempVal == null) {
tempVal = 0L;
} else {
tempVal += store.getMixedRowReadsCount();
}
mixedReadsOnStore.put(store.getColumnFamilyName(), tempVal);
if (readsOnlyFromMemstore == null) {
readsOnlyFromMemstore = new HashMap<String, Long>();
}
tempVal = readsOnlyFromMemstore.get(store.getColumnFamilyName());
if (tempVal == null) {
tempVal = 0L;
} else {
tempVal += store.getMemstoreOnlyRowReadsCount();
}
readsOnlyFromMemstore.put(store.getColumnFamilyName(), tempVal);
}

OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge();
if (
storeMinStoreFileAge.isPresent()
&& storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge
) {
tempMinStoreFileAge = storeMinStoreFileAge.getAsLong();
}

long storeHFiles = store.getNumHFiles();
numHFiles += storeHFiles;
tempNumReferenceFiles += store.getNumReferenceFiles();

OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
if (storeAvgStoreFileAge.isPresent()) {
avgAgeNumerator += (long) storeAvgStoreFileAge.getAsDouble() * storeHFiles;
}
if (mixedReadsOnStore == null) {
mixedReadsOnStore = new HashMap<String, Long>();
}
Long tempVal = mixedReadsOnStore.get(store.getColumnFamilyName());
if (tempVal == null) {
tempVal = 0L;
} else {
tempVal += store.getMixedRowReadsCount();
}
mixedReadsOnStore.put(store.getColumnFamilyName(), tempVal);
if (readsOnlyFromMemstore == null) {
readsOnlyFromMemstore = new HashMap<String, Long>();
}
tempVal = readsOnlyFromMemstore.get(store.getColumnFamilyName());
if (tempVal == null) {
tempVal = 0L;
} else {
tempVal += store.getMemstoreOnlyRowReadsCount();
}
MutableLong regionCachedAmount = new MutableLong(0);
region.getBlockCache().getRegionCachedInfo()
.ifPresent(regionCacheRatio -> regionCachedAmount
.addAndGet(regionCacheRatio.getOrDefault(region.getRegionInfo().getEncodedName(), 0L)));
if (tempStoreFileSize > 0) {
LOG.debug("Region {}, had cached {} bytes from a total of {}",
region.getRegionInfo().getEncodedName(), regionCachedAmount.getValue(),
tempStoreFileSize);
currentRegionCacheRatio = regionCachedAmount.floatValue() / tempStoreFileSize;
if (DataTieringManager.getInstance() != null) {
currentRegionColdDataRatio = DataTieringManager.getInstance().getRegionColdDataSize()
.getOrDefault(region.getRegionInfo().getEncodedName(), new Pair<>(null, 0L))
.getSecond() / (float) tempStoreFileSize;
}
readsOnlyFromMemstore.put(store.getColumnFamilyName(), tempVal);
}
}
MutableLong regionCachedAmount = new MutableLong(0);
region.getBlockCache().getRegionCachedInfo().ifPresent(regionCacheRatio -> regionCachedAmount
.addAndGet(regionCacheRatio.getOrDefault(region.getRegionInfo().getEncodedName(), 0L)));
if (tempStoreFileSize > 0) {
LOG.debug("Region {}, had cached {} bytes from a total of {}",
region.getRegionInfo().getEncodedName(), regionCachedAmount.getValue(),
tempStoreFileSize);
currentRegionCacheRatio = regionCachedAmount.floatValue() / tempStoreFileSize;
if (DataTieringManager.getInstance() != null) {
currentRegionColdDataRatio = DataTieringManager.getInstance().getRegionColdDataSize()
.getOrDefault(region.getRegionInfo().getEncodedName(), new Pair<>(null, 0L)).getSecond()
/ (float) tempStoreFileSize;
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
memstoreSize = tempMemstoreSize;
memstoreHeapSize = tempMemstoreHeapSize;
memstoreOffHeapSize = tempMemstoreOffHeapSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (tempMinStoreFileAge != Long.MAX_VALUE) {
minStoreFileAge = tempMinStoreFileAge;
}
}
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
memstoreSize = tempMemstoreSize;
memstoreHeapSize = tempMemstoreHeapSize;
memstoreOffHeapSize = tempMemstoreOffHeapSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (tempMinStoreFileAge != Long.MAX_VALUE) {
minStoreFileAge = tempMinStoreFileAge;
}

if (numHFiles != 0) {
avgStoreFileAge = avgAgeNumerator / numHFiles;
}
if (numHFiles != 0) {
avgStoreFileAge = avgAgeNumerator / numHFiles;
}

numReferenceFiles = tempNumReferenceFiles;
tempMaxCompactionQueueSize = getNumCompactionsQueued();
tempMaxFlushQueueSize = getNumFlushesQueued();
if (tempMaxCompactionQueueSize > maxCompactionQueueSize) {
maxCompactionQueueSize = tempMaxCompactionQueueSize;
}
if (tempMaxFlushQueueSize > maxFlushQueueSize) {
maxFlushQueueSize = tempMaxFlushQueueSize;
numReferenceFiles = tempNumReferenceFiles;
tempMaxCompactionQueueSize = getNumCompactionsQueued();
tempMaxFlushQueueSize = getNumFlushesQueued();
if (tempMaxCompactionQueueSize > maxCompactionQueueSize) {
maxCompactionQueueSize = tempMaxCompactionQueueSize;
}
if (tempMaxFlushQueueSize > maxFlushQueueSize) {
maxFlushQueueSize = tempMaxFlushQueueSize;
}
} catch (Throwable t) {
LOG.error("Failed to compute per-region metrics; will retry on next scheduled run", t);
}
}
}
Expand Down
Loading
Loading