Skip to content

Add RocksDB memory metrics by component type #2392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
platinumhamburg wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add RocksDB memory metrics by component type #2392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,27 @@ public class MetricNames {
/** Total memory usage across all RocksDB instances in this server (Sum aggregation). */
public static final String ROCKSDB_MEMORY_USAGE_TOTAL = "rocksdbMemoryUsageTotal";

// Table-level RocksDB memory metrics (Sum aggregation)
/** Total memtable memory usage across all buckets of this table. */
public static final String ROCKSDB_MEMTABLE_MEMORY_USAGE_TOTAL =
"rocksdbMemTableMemoryUsageTotal";

/** Total unflushed memtable memory usage across all buckets of this table. */
public static final String ROCKSDB_MEMTABLE_UNFLUSHED_MEMORY_USAGE_TOTAL =
"rocksdbMemTableUnFlushedMemoryUsageTotal";

/** Total table readers (indexes and filters) memory usage across all buckets of this table. */
public static final String ROCKSDB_TABLE_READERS_MEMORY_USAGE_TOTAL =
"rocksdbTableReadersMemoryUsageTotal";

/** Total block cache memory usage across all buckets of this table. */
public static final String ROCKSDB_BLOCK_CACHE_MEMORY_USAGE_TOTAL =
"rocksdbBlockCacheMemoryUsageTotal";

/** Total pinned memory in block cache across all buckets of this table. */
public static final String ROCKSDB_BLOCK_CACHE_PINNED_USAGE_TOTAL =
"rocksdbBlockCachePinnedUsageTotal";

// --------------------------------------------------------------------------------------------
// metrics for table bucket
// --------------------------------------------------------------------------------------------
Expand Down
82 changes: 77 additions & 5 deletions fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,6 @@ public long getTotalMemoryUsage() {
return 0L;
}

// Create cache set for memory usage calculation.
// If blockCache is null, pass null to MemoryUtil (will only count memtables, etc.)
Set<Cache> caches = null;
if (blockCache != null) {
caches = new HashSet<>();
Expand All @@ -239,9 +237,83 @@ public long getTotalMemoryUsage() {
Collections.singletonList(db), caches);
return memoryUsage.values().stream().mapToLong(Long::longValue).sum();
} catch (Exception e) {
LOG.debug(
"Failed to get total memory usage from RocksDB (possibly closed or unavailable)",
e);
LOG.debug("Failed to get total memory usage from RocksDB", e);
return 0L;
}
}

// ==================== Memory Metrics by Component ====================

/**
* Get memory usage for all memtables (active and immutable).
*
* @return memtable memory usage in bytes, or 0 if not available
*/
public long getMemTableMemoryUsage() {
return getMemoryUsageByType(MemoryUsageType.kMemTableTotal);
}

/**
* Get memory usage for unflushed memtables.
*
* @return unflushed memtable memory usage in bytes, or 0 if not available
*/
public long getMemTableUnFlushedMemoryUsage() {
return getMemoryUsageByType(MemoryUsageType.kMemTableUnFlushed);
}

/**
* Get memory usage for table readers (indexes and bloom filters).
*
* @return table readers memory usage in bytes, or 0 if not available
*/
public long getTableReadersMemoryUsage() {
return getMemoryUsageByType(MemoryUsageType.kTableReadersTotal);
}

/**
* Get memory usage for block cache via MemoryUtil API.
*
* @return block cache memory usage in bytes, or 0 if not available
*/
public long getBlockCacheMemoryUsage() {
return getMemoryUsageByType(MemoryUsageType.kCacheTotal);
}

/**
* Get pinned memory usage in block cache.
*
* @return pinned memory usage in bytes, or 0 if not available
*/
public long getBlockCachePinnedUsage() {
try (ResourceGuard.Lease lease = resourceGuard.acquireResource()) {
if (blockCache != null) {
return blockCache.getPinnedUsage();
}
} catch (Exception e) {
LOG.debug("Failed to get pinned usage from RocksDB", e);
}
return 0L;
}

private long getMemoryUsageByType(MemoryUsageType type) {
try (ResourceGuard.Lease lease = resourceGuard.acquireResource()) {
if (db == null) {
return 0L;
}

Set<Cache> caches = null;
if (blockCache != null) {
caches = new HashSet<>();
caches.add(blockCache);
}

Map<MemoryUsageType, Long> memoryUsage =
MemoryUtil.getApproximateMemoryUsageByType(
Collections.singletonList(db), caches);
return memoryUsage.getOrDefault(type, 0L);
} catch (Exception e) {
LOG.debug("Failed to get memory usage for type {} from RocksDB", type, e);
return 0L;
}
}
Expand Down
32 changes: 32 additions & 0 deletions fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,38 @@ private void registerRocksDBMetrics() {
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getCompactionBytesWritten)
.sum());

// Fine-grained memory metrics - track memory usage by component type
gauge(
MetricNames.ROCKSDB_MEMTABLE_MEMORY_USAGE_TOTAL,
() ->
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getMemTableMemoryUsage)
.sum());
gauge(
MetricNames.ROCKSDB_MEMTABLE_UNFLUSHED_MEMORY_USAGE_TOTAL,
() ->
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getMemTableUnFlushedMemoryUsage)
.sum());
gauge(
MetricNames.ROCKSDB_TABLE_READERS_MEMORY_USAGE_TOTAL,
() ->
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getTableReadersMemoryUsage)
.sum());
gauge(
MetricNames.ROCKSDB_BLOCK_CACHE_MEMORY_USAGE_TOTAL,
() ->
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getBlockCacheMemoryUsage)
.sum());
gauge(
MetricNames.ROCKSDB_BLOCK_CACHE_PINNED_USAGE_TOTAL,
() ->
allRocksDBStatistics()
.mapToLong(RocksDBStatistics::getBlockCachePinnedUsage)
.sum());
}

/** Metric group for specific kind of tablet of a table. */
Expand Down
46 changes: 46 additions & 0 deletions fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1510,6 +1510,45 @@ void testRocksDBMetrics() throws Exception {
.as("Total memory usage must be positive")
.isGreaterThan(0);

// ========== Phase 5.1: Verify Fine-Grained Memory Metrics ==========
// All fine-grained memory metrics should be non-negative
long memTableUsage = statistics.getMemTableMemoryUsage();
assertThat(memTableUsage)
.as("MemTable memory usage should be non-negative")
.isGreaterThanOrEqualTo(0);

long memTableUnFlushedUsage = statistics.getMemTableUnFlushedMemoryUsage();
assertThat(memTableUnFlushedUsage)
.as("Unflushed memtable memory usage should be non-negative")
.isGreaterThanOrEqualTo(0);

long tableReadersUsage = statistics.getTableReadersMemoryUsage();
assertThat(tableReadersUsage)
.as("Table readers memory usage should be non-negative")
.isGreaterThanOrEqualTo(0);

long blockCacheMemoryUsage = statistics.getBlockCacheMemoryUsage();
assertThat(blockCacheMemoryUsage)
.as("Block cache memory usage should be non-negative")
.isGreaterThanOrEqualTo(0);

long blockCachePinnedUsage = statistics.getBlockCachePinnedUsage();
assertThat(blockCachePinnedUsage)
.as("Block cache pinned usage should be non-negative")
.isGreaterThanOrEqualTo(0);

// Total memory usage should be at least as large as any individual component
long totalMemoryUsage = statistics.getTotalMemoryUsage();
assertThat(totalMemoryUsage)
.as("Total memory should be at least as large as memtable usage")
.isGreaterThanOrEqualTo(memTableUsage);
assertThat(totalMemoryUsage)
.as("Total memory should be at least as large as table readers usage")
.isGreaterThanOrEqualTo(tableReadersUsage);
assertThat(totalMemoryUsage)
.as("Total memory should be at least as large as block cache usage")
.isGreaterThanOrEqualTo(blockCacheMemoryUsage);

// ========== Phase 6: Verify Metrics After Close ==========
kvTablet.close();

Expand All @@ -1526,5 +1565,12 @@ void testRocksDBMetrics() throws Exception {
assertThat(statistics.getCompactionBytesWritten()).isEqualTo(0);
assertThat(statistics.getCompactionTimeMicros()).isEqualTo(0);
assertThat(statistics.getTotalMemoryUsage()).isEqualTo(0);

// Fine-grained memory metrics should also return 0 after close
assertThat(statistics.getMemTableMemoryUsage()).isEqualTo(0);
assertThat(statistics.getMemTableUnFlushedMemoryUsage()).isEqualTo(0);
assertThat(statistics.getTableReadersMemoryUsage()).isEqualTo(0);
assertThat(statistics.getBlockCacheMemoryUsage()).isEqualTo(0);
assertThat(statistics.getBlockCachePinnedUsage()).isEqualTo(0);
}
}
45 changes: 44 additions & 1 deletion website/docs/maintenance/observability/monitor-metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -948,7 +948,50 @@ These metrics use Sum aggregation to show the total value across all tables in a
<th rowspan="1"><strong>tabletserver</strong></th>
<td style={{textAlign: 'center', verticalAlign: 'middle' }} rowspan="1">-</td>
<td>rocksdbMemoryUsageTotal</td>
<td>Total memory usage across all RocksDB instances in this server (in bytes). This includes memory used by memtables, block cache, and other RocksDB internal structures.</td>
<td>Total memory usage across all RocksDB instances in this server (in bytes).</td>
<td>Gauge</td>
</tr>
</tbody>
</table>

#### Table-level RocksDB Memory Metrics (Sum Aggregation)

<table class="table table-bordered">
<thead>
<tr>
<th class="text-left" style={{width: '30pt'}}>Scope</th>
<th class="text-left" style={{width: '150pt'}}>Infix</th>
<th class="text-left" style={{width: '80pt'}}>Metrics</th>
<th class="text-left" style={{width: '300pt'}}>Description</th>
<th class="text-left" style={{width: '40pt'}}>Type</th>
</tr>
</thead>
<tbody>
<tr>
<th rowspan="6"><strong>tabletserver</strong></th>
<td rowspan="6">table</td>
<td>rocksdbMemTableMemoryUsageTotal</td>
<td>Total memtable memory usage across all buckets of this table (in bytes).</td>
<td>Gauge</td>
</tr>
<tr>
<td>rocksdbMemTableUnFlushedMemoryUsageTotal</td>
<td>Total unflushed memtable memory usage across all buckets of this table (in bytes).</td>
<td>Gauge</td>
</tr>
<tr>
<td>rocksdbTableReadersMemoryUsageTotal</td>
<td>Total table readers (indexes and filters) memory usage across all buckets of this table (in bytes).</td>
<td>Gauge</td>
</tr>
<tr>
<td>rocksdbBlockCacheMemoryUsageTotal</td>
<td>Total block cache memory usage across all buckets of this table (in bytes).</td>
<td>Gauge</td>
</tr>
<tr>
<td>rocksdbBlockCachePinnedUsageTotal</td>
<td>Total pinned memory in block cache across all buckets of this table (in bytes).</td>
<td>Gauge</td>
</tr>
</tbody>
Expand Down