feat(monitoring): add lock_waits metric and dashboard

Author: Dementii Priadko

config/grafana/dashboards/Dashboard_13_Lock_waits.json

@@ -12,7 +12,6 @@ metrics:
           queryid is not null
           and dbid = (select oid from pg_database where datname = current_database())
         order by total_exec_time desc
-        limit 1000;
     gauges:
       - '*' 
 
@@ -198,5 +197,5 @@ presets:
   full:
     description: "Full metrics for PostgreSQL storage"
     metrics:
-      pgss_queryid_queries: 300
+      pgss_queryid_queries: 30
       index_definitions: 3600

config/pgwatch-postgres/metrics.yml

@@ -819,6 +819,71 @@ metrics:
     gauges:
       - queries
     statement_timeout_seconds: 15
+  lock_waits:
+    description: >
+      Retrieves detailed information about lock waits, including blocked and blocking processes with their queries, users, and application names.
+      It returns blocked and blocker process IDs, lock modes and types, affected tables, queries, and wait/transaction durations.
+      This metric helps administrators identify and diagnose lock contention issues in detail.
+    sqls:
+      14: |-
+        with sa_snapshot as ( /* pgwatch_generated */
+          select *
+          from pg_stat_activity
+          where
+            datname = current_database()
+            and pid <> pg_backend_pid()
+            and state in ('active', 'idle in transaction', 'idle in transaction (aborted)')
+        ),
+        pid_tables as (
+          select distinct on (pid) pid, relation::regclass::text as table_name
+          from pg_catalog.pg_locks
+          where relation is not null
+            and locktype in ('tuple', 'relation')
+            and relation::regclass::text not like '%_pkey'
+            and relation::regclass::text not like '%_idx'
+          order by pid, locktype
+        )
+        select
+          blocked.pid as blocked_pid,
+          current_database() as tag_datname,
+          blocked_stm.usename::text as tag_blocked_user,
+          blocked_stm.application_name::text as tag_blocked_appname,
+          blocked.mode as blocked_mode,
+          blocked.locktype as blocked_locktype,
+          coalesce(blocked.relation::regclass::text, blocked_tbl.table_name, '') as tag_blocked_table,
+          blocked_stm.query_id::text as tag_blocked_query_id,
+          (extract(epoch from (clock_timestamp() - blocked_stm.state_change)) * 1000)::bigint as blocked_ms,
+          blocker.pid as blocker_pid,
+          blocker_stm.usename::text as tag_blocker_user,
+          blocker_stm.application_name::text as tag_blocker_appname,
+          blocker.mode as blocker_mode,
+          blocker.locktype as blocker_locktype,
+          coalesce(blocker.relation::regclass::text, blocker_tbl.table_name, '') as tag_blocker_table,
+          blocker_stm.query_id::text as tag_blocker_query_id,
+          (extract(epoch from (clock_timestamp() - blocker_stm.xact_start)) * 1000)::bigint as blocker_tx_ms
+        from pg_catalog.pg_locks as blocked
+        join sa_snapshot as blocked_stm on blocked_stm.pid = blocked.pid
+        join pg_catalog.pg_locks as blocker on
+          blocked.pid <> blocker.pid
+          and blocker.granted
+          and (
+            (blocked.database = blocker.database)
+            or (blocked.database is null and blocker.database is null)
+          )
+          and (
+            blocked.relation = blocker.relation
+            or blocked.transactionid = blocker.transactionid
+          )
+        join sa_snapshot as blocker_stm on blocker_stm.pid = blocker.pid
+        left join pid_tables as blocked_tbl on blocked_tbl.pid = blocked.pid
+        left join pid_tables as blocker_tbl on blocker_tbl.pid = blocker.pid
+        where not blocked.granted
+        order by blocked_ms desc
+        limit 10000
+    gauges:
+      - blocked_ms
+      - blocker_tx_ms
+    statement_timeout_seconds: 15
   pg_database_wraparound:
     sqls:
       11: |
@@ -2335,6 +2400,7 @@ presets:
       pg_statio_all_tables: 30
       pg_statio_all_indexes: 30
       pg_total_relation_size: 30
+      lock_waits: 30
       pg_blocked: 30
       pg_long_running_transactions: 30
       pg_stuck_idle_in_transaction: 30

config/pgwatch-prometheus/metrics.yml

@@ -0,0 +1,226 @@
+# Lock Waits Metric Testing
+
+This directory contains tests and scripts to verify that the `lock_waits` metric is working correctly.
+
+## Overview
+
+The `lock_waits` metric collects detailed information about lock waits in PostgreSQL, including:
+- Waiting and blocking process IDs
+- User names and application names
+- Lock modes and types
+- Affected tables
+- Query IDs (PostgreSQL 14+)
+- Wait durations and blocker transaction durations
+
+## Test Components
+
+### 1. Python Test Script (`test_lock_waits_metric.py`)
+
+Automated test that:
+- Creates lock contention scenarios in the target database
+- Waits for pgwatch to collect metrics
+- Verifies the metric is collected in Prometheus/VictoriaMetrics
+- Validates the metric structure and labels
+
+### 2. SQL Script (`create_lock_contention.sql`)
+
+Manual SQL script to create lock contention for testing. Can be run in multiple psql sessions.
+
+## Prerequisites
+
+1. Docker Compose stack running:
+   ```bash
+   docker-compose up -d
+   ```
+
+2. Python dependencies:
+   ```bash
+   pip install psycopg requests
+   ```
+
+3. Ensure `lock_waits` metric is enabled in pgwatch configuration:
+   - Check `config/pgwatch-prometheus/metrics.yml` includes `lock_waits`
+   - Verify pgwatch is collecting metrics from the target database
+
+## Running the Automated Test
+
+### Basic Usage
+
+```bash
+# From the project root
+python tests/lock_waits/test_lock_waits_metric.py
+```
+
+### With Custom Configuration
+
+```bash
+python tests/lock_waits/test_lock_waits_metric.py \
+  --target-db-url "postgresql://postgres:postgres@localhost:55432/target_database" \
+  --prometheus-url "http://localhost:59090" \
+  --test-dbname "target_database" \
+  --collection-wait 90
+```
+
+### Environment Variables
+
+You can also set these via environment variables:
+
+```bash
+export TARGET_DB_URL="postgresql://postgres:postgres@localhost:55432/target_database"
+export PROMETHEUS_URL="http://localhost:59090"
+export TEST_DBNAME="target_database"
+export COLLECTION_WAIT_SECONDS=90
+
+python tests/lock_waits/test_lock_waits_metric.py
+```
+
+## Manual Testing
+
+### Step 1: Create Lock Contention
+
+Open two psql sessions to the target database:
+
+**Session 1 (Blocker):**
+```sql
+BEGIN;
+SELECT * FROM lock_test_table WHERE id = 1 FOR UPDATE;
+-- Keep this transaction open
+```
+
+**Session 2 (Waiter):**
+```sql
+BEGIN;
+SELECT * FROM lock_test_table WHERE id = 1 FOR UPDATE;
+-- This will wait for Session 1 to release the lock
+```
+
+### Step 2: Verify Metric Collection
+
+Wait for pgwatch to collect metrics (check collection interval in pgwatch config, typically 15-30 seconds), then query Prometheus:
+
+```bash
+# Query Prometheus API for lock_waits metrics
+curl "http://localhost:59090/api/v1/query?query=pgwatch_lock_waits_waiting_ms{datname=\"target_database\"}"
+
+# Or use PromQL in Grafana Explore
+pgwatch_lock_waits_waiting_ms{datname="target_database"}
+pgwatch_lock_waits_blocker_tx_ms{datname="target_database"}
+```
+
+### Step 3: Check Grafana Dashboard
+
+1. Open Grafana: http://localhost:3000
+2. Navigate to "Lock waits details" dashboard
+3. Select the database from the dropdown
+4. Verify that lock wait events appear in the panels
+
+## Expected Results
+
+### Successful Test Output
+
+```
+Setting up test environment...
+✓ Test table created
+
+Creating lock contention for 30 seconds...
+✓ Blocker transaction started (holding lock on row id=1)
+✓ Waiter transaction started (waiting for lock on row id=1)
+  Holding locks for 30 seconds...
+✓ Lock contention ended
+
+Verifying metric collection...
+  Waiting 60 seconds for pgwatch to collect metrics...
+  ✓ Found 5 lock_waits records
+
+Validating metric structure...
+
+  Record 1:
+    ✓ All required data fields present
+    ✓ waiting_ms is numeric: 25000 ms
+    ✓ blocker_tx_ms is numeric: 30000 ms
+
+✅ Test PASSED: lock_waits metric is working correctly
+```
+
+## Troubleshooting
+
+### No Records Found
+
+- **Check pgwatch is running**: `docker ps | grep pgwatch-prometheus`
+- **Check pgwatch logs**: `docker logs pgwatch-prometheus`
+- **Verify metric is enabled**: Check `config/pgwatch-prometheus/metrics.yml`
+- **Check Prometheus is accessible**: `curl http://localhost:59090/api/v1/status/config`
+- **Increase wait time**: Use `--collection-wait 120` to wait longer
+- **Check database name**: Ensure `--test-dbname` matches the monitored database
+- **Verify metrics exist**: `curl "http://localhost:59090/api/v1/label/__name__/values" | grep lock_waits`
+
+### Invalid Data Structure
+
+- **Check PostgreSQL version**: Metric requires PostgreSQL 14+ for query_id support
+- **Verify metric SQL**: Check the SQL query in `metrics.yml` is correct
+- **Check pgwatch version**: Ensure pgwatch version supports the metric format
+- **Check Prometheus labels**: Verify metrics have expected labels (datname, waiting_pid, blocker_pid, etc.)
+
+### Connection Errors
+
+- **Verify Docker containers**: `docker-compose ps`
+- **Check connection strings**: Verify URLs match your docker-compose configuration
+- **Check Prometheus URL**: Ensure Prometheus/VictoriaMetrics is accessible at the specified URL
+- **Check network**: Ensure containers can communicate (same Docker network)
+
+## Integration with CI/CD
+
+The test can be integrated into CI/CD pipelines:
+
+```yaml
+# Example GitLab CI
+test_lock_waits:
+  stage: test
+  script:
+    - docker-compose up -d
+    - sleep 30  # Wait for services to start
+    - pip install psycopg
+    - python tests/lock_waits/test_lock_waits_metric.py
+      --target-db-url "$TARGET_DB_URL"
+      --sink-db-url "$SINK_DB_URL"
+      --collection-wait 90
+  only:
+    - merge_requests
+    - main
+```
+
+## Additional Test Scenarios
+
+### Test Different Lock Types
+
+Modify the test to create different types of locks:
+
+```sql
+-- Table-level lock
+LOCK TABLE lock_test_table IN EXCLUSIVE MODE;
+
+-- Advisory lock
+SELECT pg_advisory_lock(12345);
+```
+
+### Test Multiple Concurrent Waits
+
+Create multiple waiting transactions to test the LIMIT clause:
+
+```sql
+-- Session 1: Blocker
+BEGIN;
+SELECT * FROM lock_test_table WHERE id = 1 FOR UPDATE;
+
+-- Sessions 2-10: Multiple waiters
+-- Each in separate psql session
+BEGIN;
+SELECT * FROM lock_test_table WHERE id = 1 FOR UPDATE;
+```
+
+## Related Files
+
+- `config/pgwatch-prometheus/metrics.yml` - Metric definition
+- `config/grafana/dashboards/Dashboard_13_Lock_waits.json` - Grafana dashboard
+- `workload_examples/lock_wait_test.sql` - Basic lock test SQL
+

tests/lock_waits/README.md

@@ -0,0 +1,2 @@
+# Lock waits metric testing package
+

tests/lock_waits/__init__.py

@@ -0,0 +1,73 @@
+-- SQL script to manually create lock contention for testing lock_waits metric
+-- 
+-- Usage:
+--   1. Run this script in Session 1 (blocker)
+--   2. Run the same script in Session 2 (waiter) - it will wait
+--   3. Check the sink database for lock_waits records
+--   4. Commit or rollback Session 1 to release the lock
+
+-- Create test table if it doesn't exist
+drop table if exists lock_test_table cascade;
+create table lock_test_table (
+    id int8 generated always as identity primary key,
+    name text not null,
+    value numeric(10, 2),
+    created_at timestamptz default now()
+);
+
+insert into lock_test_table (name, value)
+values
+    ('Item 1', 100.50),
+    ('Item 2', 200.75),
+    ('Item 3', 300.25);
+
+-- ============================================
+-- SESSION 1 (BLOCKER) - Run this first
+-- ============================================
+begin;
+
+-- Acquire exclusive lock on row id=1
+-- Keep this transaction open to hold the lock
+select * from lock_test_table where id = 1 for update;
+
+-- Transaction is now holding the lock
+-- DO NOT COMMIT YET - keep this session open
+
+-- ============================================
+-- SESSION 2 (WAITER) - Run this in another psql session
+-- ============================================
+begin;
+
+-- This will wait for Session 1 to release the lock
+select * from lock_test_table where id = 1 for update;
+
+-- This query will block until Session 1 commits or rolls back
+-- You should see it waiting in pg_stat_activity
+
+-- ============================================
+-- To release the lock, commit or rollback Session 1:
+-- ============================================
+-- commit;  -- or rollback;
+
+-- ============================================
+-- Alternative: Test with different lock types
+-- ============================================
+
+-- Test with table-level lock
+-- SESSION 1:
+-- begin;
+-- lock table lock_test_table in exclusive mode;
+
+-- SESSION 2:
+-- begin;
+-- select * from lock_test_table;  -- Will wait
+
+-- Test with advisory lock
+-- SESSION 1:
+-- begin;
+-- select pg_advisory_lock(12345);
+
+-- SESSION 2:
+-- begin;
+-- select pg_advisory_lock(12345);  -- Will wait
+