Skip to content

HBASE-30242 Add Region server level HFile Compression Metrics #8389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Umeshkumar9414 wants to merge 1 commit into
base: master
Choose a base branch
from
+49 −0
Open

HBASE-30242 Add Region server level HFile Compression Metrics #8389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String MEMSTORE_OFFHEAP_SIZE_DESC = "Off-heap Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
String STOREFILE_SIZE_GROWTH_RATE = "storeFileSizeGrowthRate";
String STOREFILE_UNCOMPRESSED_SIZE = "storeFileUncompressedSize";
String STOREFILE_COMPRESSION_RATIO = "storeFileCompressionRatio";
String MAX_STORE_FILE_AGE = "maxStoreFileAge";
String MIN_STORE_FILE_AGE = "minStoreFileAge";
String AVG_STORE_FILE_AGE = "avgStoreFileAge";
Expand All @@ -244,6 +246,10 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
String STOREFILE_SIZE_GROWTH_RATE_DESC =
"Bytes per second by which the size of storefiles being served grows.";
String STOREFILE_UNCOMPRESSED_SIZE_DESC = "Total uncompressed size of storefiles being served.";
String STOREFILE_COMPRESSION_RATIO_DESC =
"Compression ratio of storefiles (compressed/uncompressed). Lower values indicate better"
+ " compression. Returns 1.0 when there is no data.";
Comment on lines +249 to +252
String TOTAL_REQUEST_COUNT = "totalRequestCount";
String TOTAL_REQUEST_COUNT_DESC =
"Total number of requests this RegionServer has answered; increments the count once for "
Expand Down
4 changes: 4 additions & 0 deletions ...pat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,10 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde
.addGauge(Interns.info(MEMSTORE_OFFHEAP_SIZE, MEMSTORE_OFFHEAP_SIZE_DESC),
rsWrap.getOffHeapMemStoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
.addGauge(Interns.info(STOREFILE_UNCOMPRESSED_SIZE, STOREFILE_UNCOMPRESSED_SIZE_DESC),
rsWrap.getStoreFileUncompressedSize())
.addGauge(Interns.info(STOREFILE_COMPRESSION_RATIO, STOREFILE_COMPRESSION_RATIO_DESC),
rsWrap.getStoreFileCompressionRatio())
.addGauge(Interns.info(STOREFILE_SIZE_GROWTH_RATE, STOREFILE_SIZE_GROWTH_RATE_DESC),
rsWrap.getStoreFileSizeGrowthRate())
.addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
Expand Down
11 changes: 11 additions & 0 deletions ...compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,17 @@ public interface MetricsRegionServerWrapper {
*/
long getStoreFileSize();

/**
* Get the total uncompressed size of the store files this region server is serving from.
*/
long getStoreFileUncompressedSize();

/**
* Get the compression ratio of store files on this region server. This is the ratio of compressed
* on-disk size to uncompressed data size. Returns 1.0 when there is no data.
*/
double getStoreFileCompressionRatio();
Comment on lines +124 to +128

/**
* Get the growth rate of the store files this region server is serving from.
*/
Expand Down
16 changes: 16 additions & 0 deletions ...er/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,20 @@ public long getStoreFileSize() {
return aggregate.storeFileSize;
}

@Override
public long getStoreFileUncompressedSize() {
return aggregate.storeFileUncompressedSize;
}

@Override
public double getStoreFileCompressionRatio() {

@apurtell apurtell Jun 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally for compression factor, higher is better. This is what command line compression tools like xz or zstd or etc. typically report. And for compression savings, higher is also better, e.g 1 − compressed / uncompressed, which is easy to read as a percentage. This method provides the inverse of that so it makes the desired direction "down", which is awkward for Prometheus or Grafana dashboards. Operators reading a dashboard tile labeled storeFileCompressionRatio with a value e.g. of 0.33 will routinely interpret it the opposite of what is intended.

Recommendation: either rename to storeFileCompressedToUncompressedRatio so the direction is clearly established by the name of the metric, or reimplement as uncompressed / compressed, or storeFileCompressionSavingsRatio as (1 − compressed/uncompressed).

long uncompressed = aggregate.storeFileUncompressedSize;
if (uncompressed == 0) {

@apurtell apurtell Jun 22, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A value of "1.0" will generally look identical to an operator like somehow compression has stopped working, and will generate false positive alerts.

(Claude calls this a "footgun for dashboards", lol. He means well...)

Recommendation: Return 0.0 instead, and update the description to something like "Returns 0.0 when there is no data."

return 1.0;
}
return (double) aggregate.storeFileSize / uncompressed;
}

@Override
public double getStoreFileSizeGrowthRate() {
return aggregate.storeFileSizeGrowthRate;
Expand Down Expand Up @@ -783,6 +797,7 @@ private static final class RegionMetricAggregate {
private long onHeapMemstoreSize = 0;
private long offHeapMemstoreSize = 0;
private long storeFileSize = 0;
private long storeFileUncompressedSize = 0;
private double storeFileSizeGrowthRate = 0;
private long maxStoreFileCount = 0;
private long maxStoreFileAge = 0;
Expand Down Expand Up @@ -979,6 +994,7 @@ private StoreFileStats aggregateStores(List<HStore> stores) {
onHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
offHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize();
storeFileSize += store.getStorefilesSize();
storeFileUncompressedSize += store.getStoreSizeUncompressed();
maxStoreFileCount = Math.max(maxStoreFileCount, store.getStorefilesCount());

maxStoreFileAge =
Expand Down
10 changes: 10 additions & 0 deletions ...er/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ public long getStoreFileSize() {
return 1900;
}

@Override
public long getStoreFileUncompressedSize() {
return 3800;
}

@Override
public double getStoreFileCompressionRatio() {
return 0.5;
}

@Override
public double getStoreFileSizeGrowthRate() {
return 50.0;
Expand Down
2 changes: 2 additions & 0 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ public void testWrapperSource() {
HELPER.assertGauge("memstoreHeapSize", 500, serverSource);
HELPER.assertGauge("memstoreOffHeapSize", 600, serverSource);
HELPER.assertGauge("storeFileSize", 1900, serverSource);
HELPER.assertGauge("storeFileUncompressedSize", 3800, serverSource);
HELPER.assertGauge("storeFileCompressionRatio", 0.5, serverSource);
HELPER.assertGauge("storeFileSizeGrowthRate", 50.0, serverSource);
HELPER.assertCounter("totalRequestCount", 899, serverSource);
HELPER.assertCounter("totalRowActionRequestCount",
Expand Down