Merge branch 'feat/db-size' into 'main'

feat(reporter): add database size to H001, H002, H004, F004, F005 reports

See merge request postgres-ai/postgres_ai!91

Author: bogdan-ts

reporter/postgres_reports.py

@@ -274,7 +274,7 @@ def generate_a004_cluster_report(self, cluster: str = "local", node_name: str =
             'active_connections': f'sum(last_over_time(pgwatch_pg_stat_activity_count{{cluster="{cluster}", node_name="{node_name}", state="active"}}[3h]))',
             'idle_connections': f'sum(last_over_time(pgwatch_pg_stat_activity_count{{cluster="{cluster}", node_name="{node_name}", state="idle"}}[3h]))',
             'total_connections': f'sum(last_over_time(pgwatch_pg_stat_activity_count{{cluster="{cluster}", node_name="{node_name}"}}[3h]))',
-            'database_sizes': f'sum(last_over_time(pgwatch_pg_database_size_bytes{{cluster="{cluster}", node_name="{node_name}"}}[3h]))',
+            'database_sizes': f'sum(last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h]))',
             'cache_hit_ratio': f'sum(last_over_time(pgwatch_db_stats_blks_hit{{cluster="{cluster}", node_name="{node_name}"}}[3h])) / clamp_min(sum(last_over_time(pgwatch_db_stats_blks_hit{{cluster="{cluster}", node_name="{node_name}"}}[3h])) + sum(last_over_time(pgwatch_db_stats_blks_read{{cluster="{cluster}", node_name="{node_name}"}}[3h])), 1) * 100',
             'transactions_per_sec': f'sum(rate(pgwatch_db_stats_xact_commit{{cluster="{cluster}", node_name="{node_name}"}}[5m])) + sum(rate(pgwatch_db_stats_xact_rollback{{cluster="{cluster}", node_name="{node_name}"}}[5m]))',
             'checkpoints_per_sec': f'sum(rate(pgwatch_pg_stat_bgwriter_checkpoints_timed{{cluster="{cluster}", node_name="{node_name}"}}[5m])) + sum(rate(pgwatch_pg_stat_bgwriter_checkpoints_req{{cluster="{cluster}", node_name="{node_name}"}}[5m]))',
@@ -297,7 +297,7 @@ def generate_a004_cluster_report(self, cluster: str = "local", node_name: str =
                     }
 
         # Get database sizes
-        db_sizes_query = f'last_over_time(pgwatch_pg_database_size_bytes{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
         db_sizes_result = self.query_instant(db_sizes_query)
         database_sizes = {}
 
@@ -374,6 +374,17 @@ def generate_h001_invalid_indexes_report(self, cluster: str = "local", node_name
         # Get all databases
         databases = self.get_all_databases(cluster, node_name)
 
+        # Get database sizes
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_result = self.query_instant(db_sizes_query)
+        database_sizes = {}
+
+        if db_sizes_result.get('status') == 'success' and db_sizes_result.get('data', {}).get('result'):
+            for result in db_sizes_result['data']['result']:
+                db_name = result['metric'].get('datname', 'unknown')
+                size_bytes = float(result['value'][1])
+                database_sizes[db_name] = size_bytes
+
         invalid_indexes_by_db = {}
         for db_name in databases:
             # Query invalid indexes for each database
@@ -408,11 +419,14 @@ def generate_h001_invalid_indexes_report(self, cluster: str = "local", node_name
                     invalid_indexes.append(invalid_index)
                     total_size += index_size_bytes
 
+            db_size_bytes = database_sizes.get(db_name, 0)
             invalid_indexes_by_db[db_name] = {
                 "invalid_indexes": invalid_indexes,
                 "total_count": len(invalid_indexes),
                 "total_size_bytes": total_size,
-                "total_size_pretty": self.format_bytes(total_size)
+                "total_size_pretty": self.format_bytes(total_size),
+                "database_size_bytes": db_size_bytes,
+                "database_size_pretty": self.format_bytes(db_size_bytes)
             }
 
         return self.format_report_data("H001", invalid_indexes_by_db, node_name)
@@ -433,6 +447,17 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name:
         # Get all databases
         databases = self.get_all_databases(cluster, node_name)
 
+        # Get database sizes
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_result = self.query_instant(db_sizes_query)
+        database_sizes = {}
+
+        if db_sizes_result.get('status') == 'success' and db_sizes_result.get('data', {}).get('result'):
+            for result in db_sizes_result['data']['result']:
+                db_name = result['metric'].get('datname', 'unknown')
+                size_bytes = float(result['value'][1])
+                database_sizes[db_name] = size_bytes
+
         # Query postmaster uptime to get startup time
         postmaster_uptime_query = f'last_over_time(pgwatch_db_stats_postmaster_uptime_s{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
         postmaster_uptime_result = self.query_instant(postmaster_uptime_query)
@@ -509,11 +534,14 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name:
 
             total_unused_size = sum(idx['index_size_bytes'] for idx in unused_indexes)
 
+            db_size_bytes = database_sizes.get(db_name, 0)
             unused_indexes_by_db[db_name] = {
                 "unused_indexes": unused_indexes,
                 "total_count": len(unused_indexes),
                 "total_size_bytes": total_unused_size,
                 "total_size_pretty": self.format_bytes(total_unused_size),
+                "database_size_bytes": db_size_bytes,
+                "database_size_pretty": self.format_bytes(db_size_bytes),
                 "stats_reset": {
                     "stats_reset_epoch": stats_reset_epoch,
                     "stats_reset_time": stats_reset_time,
@@ -542,6 +570,17 @@ def generate_h004_redundant_indexes_report(self, cluster: str = "local", node_na
         # Get all databases
         databases = self.get_all_databases(cluster, node_name)
 
+        # Get database sizes
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_result = self.query_instant(db_sizes_query)
+        database_sizes = {}
+
+        if db_sizes_result.get('status') == 'success' and db_sizes_result.get('data', {}).get('result'):
+            for result in db_sizes_result['data']['result']:
+                db_name = result['metric'].get('datname', 'unknown')
+                size_bytes = float(result['value'][1])
+                database_sizes[db_name] = size_bytes
+
         redundant_indexes_by_db = {}
         for db_name in databases:
             # Fetch index definitions from the sink for this database (used to aid remediation)
@@ -606,11 +645,14 @@ def generate_h004_redundant_indexes_report(self, cluster: str = "local", node_na
             # Sort by index size descending
             redundant_indexes.sort(key=lambda x: x['index_size_bytes'], reverse=True)
 
+            db_size_bytes = database_sizes.get(db_name, 0)
             redundant_indexes_by_db[db_name] = {
                 "redundant_indexes": redundant_indexes,
                 "total_count": len(redundant_indexes),
                 "total_size_bytes": total_size,
-                "total_size_pretty": self.format_bytes(total_size)
+                "total_size_pretty": self.format_bytes(total_size),
+                "database_size_bytes": db_size_bytes,
+                "database_size_pretty": self.format_bytes(db_size_bytes)
             }
 
         return self.format_report_data("H004", redundant_indexes_by_db, node_name)
@@ -872,6 +914,17 @@ def generate_f005_btree_bloat_report(self, cluster: str = "local", node_name: st
         # Get all databases
         databases = self.get_all_databases(cluster, node_name)
 
+        # Get database sizes
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_result = self.query_instant(db_sizes_query)
+        database_sizes = {}
+
+        if db_sizes_result.get('status') == 'success' and db_sizes_result.get('data', {}).get('result'):
+            for result in db_sizes_result['data']['result']:
+                db_name = result['metric'].get('datname', 'unknown')
+                size_bytes = float(result['value'][1])
+                database_sizes[db_name] = size_bytes
+
         bloated_indexes_by_db = {}
         for db_name in databases:
             # Query btree bloat using multiple metrics for each database with last_over_time [1d]
@@ -922,11 +975,14 @@ def generate_f005_btree_bloat_report(self, cluster: str = "local", node_name: st
             # Sort by bloat percentage descending
             bloated_indexes_list.sort(key=lambda x: x['bloat_pct'], reverse=True)
 
+            db_size_bytes = database_sizes.get(db_name, 0)
             bloated_indexes_by_db[db_name] = {
                 "bloated_indexes": bloated_indexes_list,
                 "total_count": len(bloated_indexes_list),
                 "total_bloat_size_bytes": total_bloat_size,
-                "total_bloat_size_pretty": self.format_bytes(total_bloat_size)
+                "total_bloat_size_pretty": self.format_bytes(total_bloat_size),
+                "database_size_bytes": db_size_bytes,
+                "database_size_pretty": self.format_bytes(db_size_bytes)
             }
 
         return self.format_report_data("F005", bloated_indexes_by_db, node_name)
@@ -1120,6 +1176,17 @@ def generate_f004_heap_bloat_report(self, cluster: str = "local", node_name: str
         if not databases:
             print("Warning: F004 - No databases found")
 
+        # Get database sizes
+        db_sizes_query = f'last_over_time(pgwatch_db_size_size_b{{cluster="{cluster}", node_name="{node_name}"}}[3h])'
+        db_sizes_result = self.query_instant(db_sizes_query)
+        database_sizes = {}
+
+        if db_sizes_result.get('status') == 'success' and db_sizes_result.get('data', {}).get('result'):
+            for result in db_sizes_result['data']['result']:
+                db_name = result['metric'].get('datname', 'unknown')
+                size_bytes = float(result['value'][1])
+                database_sizes[db_name] = size_bytes
+
         bloated_tables_by_db = {}
         for db_name in databases:
             # Query table bloat using multiple metrics for each database
@@ -1174,11 +1241,14 @@ def generate_f004_heap_bloat_report(self, cluster: str = "local", node_name: str
             # Sort by bloat percentage descending
             bloated_tables_list.sort(key=lambda x: x['bloat_pct'], reverse=True)
 
+            db_size_bytes = database_sizes.get(db_name, 0)
             bloated_tables_by_db[db_name] = {
                 "bloated_tables": bloated_tables_list,
                 "total_count": len(bloated_tables_list),
                 "total_bloat_size_bytes": total_bloat_size,
-                "total_bloat_size_pretty": self.format_bytes(total_bloat_size)
+                "total_bloat_size_pretty": self.format_bytes(total_bloat_size),
+                "database_size_bytes": db_size_bytes,
+                "database_size_pretty": self.format_bytes(db_size_bytes)
             }
 
         return self.format_report_data("F004", bloated_tables_by_db, node_name)

tests/reporter/test_generators_unit.py

@@ -165,7 +165,7 @@ def test_generate_a004_cluster_report(
     generator: PostgresReportGenerator,
 ) -> None:
     def fake_query(query: str) -> dict[str, Any]:
-        if "pgwatch_pg_database_size_bytes" in query and "sum(" not in query:
+        if "pgwatch_db_size_size_b" in query and "sum(" not in query:
             return {
                 "status": "success",
                 "data": {