diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 60eb942e8b..454b75bdc2 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -179,6 +179,27 @@ public class MetricNames { /** Total memory usage across all RocksDB instances in this server (Sum aggregation). */ public static final String ROCKSDB_MEMORY_USAGE_TOTAL = "rocksdbMemoryUsageTotal"; + // Table-level RocksDB memory metrics (Sum aggregation) + /** Total memtable memory usage across all buckets of this table. */ + public static final String ROCKSDB_MEMTABLE_MEMORY_USAGE_TOTAL = + "rocksdbMemTableMemoryUsageTotal"; + + /** Total unflushed memtable memory usage across all buckets of this table. */ + public static final String ROCKSDB_MEMTABLE_UNFLUSHED_MEMORY_USAGE_TOTAL = + "rocksdbMemTableUnFlushedMemoryUsageTotal"; + + /** Total table readers (indexes and filters) memory usage across all buckets of this table. */ + public static final String ROCKSDB_TABLE_READERS_MEMORY_USAGE_TOTAL = + "rocksdbTableReadersMemoryUsageTotal"; + + /** Total block cache memory usage across all buckets of this table. */ + public static final String ROCKSDB_BLOCK_CACHE_MEMORY_USAGE_TOTAL = + "rocksdbBlockCacheMemoryUsageTotal"; + + /** Total pinned memory in block cache across all buckets of this table. */ + public static final String ROCKSDB_BLOCK_CACHE_PINNED_USAGE_TOTAL = + "rocksdbBlockCachePinnedUsageTotal"; + // -------------------------------------------------------------------------------------------- // metrics for table bucket // -------------------------------------------------------------------------------------------- diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBStatistics.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBStatistics.java index fd7a20cd7e..3a4e4e5bf8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBStatistics.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rocksdb/RocksDBStatistics.java @@ -226,8 +226,6 @@ public long getTotalMemoryUsage() { return 0L; } - // Create cache set for memory usage calculation. - // If blockCache is null, pass null to MemoryUtil (will only count memtables, etc.) Set caches = null; if (blockCache != null) { caches = new HashSet<>(); @@ -239,9 +237,83 @@ public long getTotalMemoryUsage() { Collections.singletonList(db), caches); return memoryUsage.values().stream().mapToLong(Long::longValue).sum(); } catch (Exception e) { - LOG.debug( - "Failed to get total memory usage from RocksDB (possibly closed or unavailable)", - e); + LOG.debug("Failed to get total memory usage from RocksDB", e); + return 0L; + } + } + + // ==================== Memory Metrics by Component ==================== + + /** + * Get memory usage for all memtables (active and immutable). + * + * @return memtable memory usage in bytes, or 0 if not available + */ + public long getMemTableMemoryUsage() { + return getMemoryUsageByType(MemoryUsageType.kMemTableTotal); + } + + /** + * Get memory usage for unflushed memtables. + * + * @return unflushed memtable memory usage in bytes, or 0 if not available + */ + public long getMemTableUnFlushedMemoryUsage() { + return getMemoryUsageByType(MemoryUsageType.kMemTableUnFlushed); + } + + /** + * Get memory usage for table readers (indexes and bloom filters). + * + * @return table readers memory usage in bytes, or 0 if not available + */ + public long getTableReadersMemoryUsage() { + return getMemoryUsageByType(MemoryUsageType.kTableReadersTotal); + } + + /** + * Get memory usage for block cache via MemoryUtil API. + * + * @return block cache memory usage in bytes, or 0 if not available + */ + public long getBlockCacheMemoryUsage() { + return getMemoryUsageByType(MemoryUsageType.kCacheTotal); + } + + /** + * Get pinned memory usage in block cache. + * + * @return pinned memory usage in bytes, or 0 if not available + */ + public long getBlockCachePinnedUsage() { + try (ResourceGuard.Lease lease = resourceGuard.acquireResource()) { + if (blockCache != null) { + return blockCache.getPinnedUsage(); + } + } catch (Exception e) { + LOG.debug("Failed to get pinned usage from RocksDB", e); + } + return 0L; + } + + private long getMemoryUsageByType(MemoryUsageType type) { + try (ResourceGuard.Lease lease = resourceGuard.acquireResource()) { + if (db == null) { + return 0L; + } + + Set caches = null; + if (blockCache != null) { + caches = new HashSet<>(); + caches.add(blockCache); + } + + Map memoryUsage = + MemoryUtil.getApproximateMemoryUsageByType( + Collections.singletonList(db), caches); + return memoryUsage.getOrDefault(type, 0L); + } catch (Exception e) { + LOG.debug("Failed to get memory usage for type {} from RocksDB", type, e); return 0L; } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java index fc40463f2d..5f280c8e49 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TableMetricGroup.java @@ -356,6 +356,38 @@ private void registerRocksDBMetrics() { allRocksDBStatistics() .mapToLong(RocksDBStatistics::getCompactionBytesWritten) .sum()); + + // Fine-grained memory metrics - track memory usage by component type + gauge( + MetricNames.ROCKSDB_MEMTABLE_MEMORY_USAGE_TOTAL, + () -> + allRocksDBStatistics() + .mapToLong(RocksDBStatistics::getMemTableMemoryUsage) + .sum()); + gauge( + MetricNames.ROCKSDB_MEMTABLE_UNFLUSHED_MEMORY_USAGE_TOTAL, + () -> + allRocksDBStatistics() + .mapToLong(RocksDBStatistics::getMemTableUnFlushedMemoryUsage) + .sum()); + gauge( + MetricNames.ROCKSDB_TABLE_READERS_MEMORY_USAGE_TOTAL, + () -> + allRocksDBStatistics() + .mapToLong(RocksDBStatistics::getTableReadersMemoryUsage) + .sum()); + gauge( + MetricNames.ROCKSDB_BLOCK_CACHE_MEMORY_USAGE_TOTAL, + () -> + allRocksDBStatistics() + .mapToLong(RocksDBStatistics::getBlockCacheMemoryUsage) + .sum()); + gauge( + MetricNames.ROCKSDB_BLOCK_CACHE_PINNED_USAGE_TOTAL, + () -> + allRocksDBStatistics() + .mapToLong(RocksDBStatistics::getBlockCachePinnedUsage) + .sum()); } /** Metric group for specific kind of tablet of a table. */ diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java index 52b1080ed1..0a5bcf8a40 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java @@ -1510,6 +1510,45 @@ void testRocksDBMetrics() throws Exception { .as("Total memory usage must be positive") .isGreaterThan(0); + // ========== Phase 5.1: Verify Fine-Grained Memory Metrics ========== + // All fine-grained memory metrics should be non-negative + long memTableUsage = statistics.getMemTableMemoryUsage(); + assertThat(memTableUsage) + .as("MemTable memory usage should be non-negative") + .isGreaterThanOrEqualTo(0); + + long memTableUnFlushedUsage = statistics.getMemTableUnFlushedMemoryUsage(); + assertThat(memTableUnFlushedUsage) + .as("Unflushed memtable memory usage should be non-negative") + .isGreaterThanOrEqualTo(0); + + long tableReadersUsage = statistics.getTableReadersMemoryUsage(); + assertThat(tableReadersUsage) + .as("Table readers memory usage should be non-negative") + .isGreaterThanOrEqualTo(0); + + long blockCacheMemoryUsage = statistics.getBlockCacheMemoryUsage(); + assertThat(blockCacheMemoryUsage) + .as("Block cache memory usage should be non-negative") + .isGreaterThanOrEqualTo(0); + + long blockCachePinnedUsage = statistics.getBlockCachePinnedUsage(); + assertThat(blockCachePinnedUsage) + .as("Block cache pinned usage should be non-negative") + .isGreaterThanOrEqualTo(0); + + // Total memory usage should be at least as large as any individual component + long totalMemoryUsage = statistics.getTotalMemoryUsage(); + assertThat(totalMemoryUsage) + .as("Total memory should be at least as large as memtable usage") + .isGreaterThanOrEqualTo(memTableUsage); + assertThat(totalMemoryUsage) + .as("Total memory should be at least as large as table readers usage") + .isGreaterThanOrEqualTo(tableReadersUsage); + assertThat(totalMemoryUsage) + .as("Total memory should be at least as large as block cache usage") + .isGreaterThanOrEqualTo(blockCacheMemoryUsage); + // ========== Phase 6: Verify Metrics After Close ========== kvTablet.close(); @@ -1526,5 +1565,12 @@ void testRocksDBMetrics() throws Exception { assertThat(statistics.getCompactionBytesWritten()).isEqualTo(0); assertThat(statistics.getCompactionTimeMicros()).isEqualTo(0); assertThat(statistics.getTotalMemoryUsage()).isEqualTo(0); + + // Fine-grained memory metrics should also return 0 after close + assertThat(statistics.getMemTableMemoryUsage()).isEqualTo(0); + assertThat(statistics.getMemTableUnFlushedMemoryUsage()).isEqualTo(0); + assertThat(statistics.getTableReadersMemoryUsage()).isEqualTo(0); + assertThat(statistics.getBlockCacheMemoryUsage()).isEqualTo(0); + assertThat(statistics.getBlockCachePinnedUsage()).isEqualTo(0); } } diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md index 65109551c2..4b082b2f2e 100644 --- a/website/docs/maintenance/observability/monitor-metrics.md +++ b/website/docs/maintenance/observability/monitor-metrics.md @@ -948,7 +948,50 @@ These metrics use Sum aggregation to show the total value across all tables in a tabletserver - rocksdbMemoryUsageTotal - Total memory usage across all RocksDB instances in this server (in bytes). This includes memory used by memtables, block cache, and other RocksDB internal structures. + Total memory usage across all RocksDB instances in this server (in bytes). + Gauge + + + + +#### Table-level RocksDB Memory Metrics (Sum Aggregation) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ScopeInfixMetricsDescriptionType
tabletservertablerocksdbMemTableMemoryUsageTotalTotal memtable memory usage across all buckets of this table (in bytes).Gauge
rocksdbMemTableUnFlushedMemoryUsageTotalTotal unflushed memtable memory usage across all buckets of this table (in bytes).Gauge
rocksdbTableReadersMemoryUsageTotalTotal table readers (indexes and filters) memory usage across all buckets of this table (in bytes).Gauge
rocksdbBlockCacheMemoryUsageTotalTotal block cache memory usage across all buckets of this table (in bytes).Gauge
rocksdbBlockCachePinnedUsageTotalTotal pinned memory in block cache across all buckets of this table (in bytes). Gauge