diff --git a/backend/.env.example b/backend/.env.example index 14cb690..f9bde07 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -37,3 +37,13 @@ SOROBAN_RPC_URL=https://soroban-testnet.stellar.org:443 # Additional Configuration # Add other configuration variables as needed ENVIRONMENT=development + +# Monitoring Configuration +# Enable/disable metrics collection (true/false) +METRICS_ENABLED=true +# Prometheus metrics endpoint path +METRICS_PATH=/metrics +# HTTP metrics endpoint path (auto-enabled by actix-web-prom) +HTTP_METRICS_PATH=/metrics/http +# Grafana admin password (required for docker-compose) +GRAFANA_ADMIN_PASSWORD=your_secure_password_here diff --git a/backend/README.md b/backend/README.md index 63bd292..eaa7d47 100644 --- a/backend/README.md +++ b/backend/README.md @@ -303,6 +303,98 @@ For questions or issues: --- +## 📊 Monitoring & Observability + +XLMate backend includes comprehensive Prometheus metrics and Grafana dashboards for monitoring system health and performance. + +### Metrics Endpoints + +The backend exposes metrics at two endpoints: + +- **`/metrics`** - Custom application metrics (active games, WebSocket connections, etc.) +- **`/metrics/http`** - HTTP request metrics (request rate, duration, status codes) + +### Available Metrics + +#### Application Metrics +- `xlmate_active_games` - Current number of active games (Gauge) +- `xlmate_ws_connections` - Active WebSocket connections (Gauge) +- `xlmate_db_query_duration_seconds` - Database query latency (Histogram) +- `xlmate_matchmaking_queue_size` - Players waiting in matchmaking queue (Gauge) +- `xlmate_ai_requests_total` - Total AI analysis requests (Counter) +- `xlmate_auth_events_total` - Authentication events (Counter) +- `xlmate_game_events_total` - Game lifecycle events (Counter) + +#### HTTP Metrics (via actix-web-prom) +- `xlmate_http_requests_total` - HTTP request count by method and status +- `xlmate_http_request_duration_seconds` - HTTP request duration (Histogram) + +### Running with Monitoring + +Start the full monitoring stack with Docker Compose: + +```bash +# Set Grafana admin password (required) +export GRAFANA_ADMIN_PASSWORD=your_secure_password + +# Start all services +docker-compose up -d +``` + +This will start: +- **PostgreSQL** on port 5432 +- **Redis** on port 6379 +- **Prometheus** on port 9090 +- **Grafana** on port 3000 + +### Accessing Dashboards + +- **Prometheus**: http://localhost:9090 +- **Grafana**: http://localhost:3000 + - Username: `admin` + - Password: Value of `GRAFANA_ADMIN_PASSWORD` environment variable +- **Metrics Endpoint**: http://localhost:8080/metrics + +### Grafana Dashboards + +The Grafana instance comes pre-configured with: +- Prometheus datasource +- XLMate Backend Monitoring dashboard with panels for: + - Active Games (stat panel) + - WebSocket Connections (stat panel) + - Database Query Latency P50/P95 (time series) + - HTTP Request Rate (time series) + - Game Events Over Time (time series) + - AI Request Rate (time series) + +### Custom Queries + +You can query metrics directly in Prometheus or Grafana: + +```promql +# Current active games +xlmate_active_games + +# WebSocket connection rate +rate(xlmate_ws_connections[5m]) + +# 95th percentile database query latency +histogram_quantile(0.95, rate(xlmate_db_query_duration_seconds_bucket[5m])) + +# HTTP error rate +rate(xlmate_http_requests_total{status=~"5.."}[5m]) +``` + +### Testing Metrics + +Run the metrics unit tests: + +```bash +cargo test metrics_tests +``` + +--- + ## 📋 Quick Reference | Document | Purpose | Read Time | diff --git a/backend/modules/api/Cargo.toml b/backend/modules/api/Cargo.toml index 9e7c46e..fba9e77 100644 --- a/backend/modules/api/Cargo.toml +++ b/backend/modules/api/Cargo.toml @@ -40,6 +40,11 @@ st_core = { path = "../st_core", features = ["api"] } # For Redis Pub/Sub in WebSocket redis = { version = "0.24", features = ["tokio-comp", "json"] } +# For Prometheus metrics +prometheus = "0.13" +actix-web-prom = "0.7" +once_cell = "1.19" + [dev-dependencies] tokio = { version = "1", features = ["full"] } actix-rt = "2.9" diff --git a/backend/modules/api/src/ai.rs b/backend/modules/api/src/ai.rs index 8a87cf5..3a7fb3c 100644 --- a/backend/modules/api/src/ai.rs +++ b/backend/modules/api/src/ai.rs @@ -12,6 +12,7 @@ use validator::Validate; use service::engine_service::EngineService; use std::env; +use crate::metrics::increment_ai_requests; #[utoipa::path( post, @@ -28,6 +29,9 @@ use std::env; )] #[post("/suggest")] pub async fn get_ai_suggestion(payload: Json) -> HttpResponse { + // Track AI request + increment_ai_requests("suggestion"); + match payload.0.validate() { Ok(_) => { let engine_path = env::var("ENGINE_PATH").unwrap_or_else(|_| "stockfish".to_string()); @@ -90,6 +94,9 @@ pub async fn get_ai_suggestion(payload: Json) -> HttpRespon )] #[post("/analyze")] pub async fn analyze_position(payload: Json) -> HttpResponse { + // Track AI request + increment_ai_requests("analysis"); + match payload.0.validate() { Ok(_) => { let engine_path = env::var("ENGINE_PATH").unwrap_or_else(|_| "stockfish".to_string()); diff --git a/backend/modules/api/src/auth.rs b/backend/modules/api/src/auth.rs index b6cb4b2..a61059b 100644 --- a/backend/modules/api/src/auth.rs +++ b/backend/modules/api/src/auth.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use dto::auth::{RegisterRequest, LoginRequest, AuthResponse, ErrorResponse, RefreshTokenRequest, RefreshResponse, LogoutResponse}; use security::{JwtService, TokenService, TokenServiceError}; use sea_orm::DatabaseConnection; +use crate::metrics::increment_auth_events; /// Register a new user #[utoipa::path( @@ -32,6 +33,8 @@ pub async fn register( } // For now, return a mock response + increment_auth_events("register", true); + HttpResponse::Created().json(AuthResponse { access_token: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...".to_string(), refresh_token: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...".to_string(), @@ -77,6 +80,7 @@ pub async fn login( let access_token = match jwt_service.generate_token(user_id, &username) { Ok(t) => t, Err(_) => { + increment_auth_events("login", false); return HttpResponse::InternalServerError().json(ErrorResponse { message: "Failed to generate access token".to_string(), code: "TOKEN_ERROR".to_string(), @@ -123,6 +127,10 @@ pub async fn login( .finish(); response.add_cookie(&cookie).ok(); + + // Track successful login + increment_auth_events("login", true); + response } diff --git a/backend/modules/api/src/games.rs b/backend/modules/api/src/games.rs index 83b4a16..3068465 100644 --- a/backend/modules/api/src/games.rs +++ b/backend/modules/api/src/games.rs @@ -16,6 +16,7 @@ use validator::Validate; use uuid::Uuid; use sea_orm::DatabaseConnection; use service::games::GameService; +use crate::metrics::{increment_active_games, decrement_active_games, increment_game_events}; // --------------------------------------------------------------------------- // Helper: extract authenticated player UUID inserted by the JWT middleware. @@ -62,10 +63,16 @@ pub async fn create_game( }; match GameService::create_game(db.get_ref(), creator_id, payload.0).await { - Ok(game_dto) => HttpResponse::Created().json(json!({ - "message": "Game created successfully", - "data": { "game": game_dto } - })), + Ok(game_dto) => { + // Track game creation metric + increment_active_games(); + increment_game_events("created"); + + HttpResponse::Created().json(json!({ + "message": "Game created successfully", + "data": { "game": game_dto } + })) + } Err(e) => { eprintln!("create_game error: {e}"); HttpResponse::InternalServerError().json(json!({ @@ -340,10 +347,16 @@ pub async fn abandon_game( let game_id = id.into_inner(); match GameService::abandon_game(db.get_ref(), game_id, player_id).await { - Ok(_) => HttpResponse::Ok().json(json!({ - "message": "Game abandoned successfully", - "data": {} - })), + Ok(_) => { + // Track game abandonment metric + decrement_active_games(); + increment_game_events("abandoned"); + + HttpResponse::Ok().json(json!({ + "message": "Game abandoned successfully", + "data": {} + })) + } Err(ApiError::NotFound(_)) => HttpResponse::NotFound().json(json!({ "message": "Game not found" })), @@ -540,6 +553,10 @@ pub async fn complete_game( // Complete the game and update ratings match GameService::complete_game(db.get_ref(), game_id, result_enum.clone(), Some(rating_config)).await { Ok((white_new_rating, black_new_rating)) => { + // Track game completion metric + decrement_active_games(); + increment_game_events("completed"); + let white_change = white_new_rating - white_old_rating; let black_change = black_new_rating - black_old_rating; diff --git a/backend/modules/api/src/lib.rs b/backend/modules/api/src/lib.rs index d898451..b6dfc5c 100644 --- a/backend/modules/api/src/lib.rs +++ b/backend/modules/api/src/lib.rs @@ -7,6 +7,10 @@ pub mod config; pub mod server; pub mod players; pub mod games; +pub mod metrics; + +#[cfg(test)] +mod metrics_tests; // External modules extern crate challenge; diff --git a/backend/modules/api/src/metrics.rs b/backend/modules/api/src/metrics.rs new file mode 100644 index 0000000..64809f5 --- /dev/null +++ b/backend/modules/api/src/metrics.rs @@ -0,0 +1,237 @@ +use prometheus::{Registry, CounterVec, Gauge, Histogram, HistogramOpts, Opts, Encoder, TextEncoder}; +use std::sync::Arc; +use once_cell::sync::{Lazy, OnceCell}; +use actix_web::{HttpResponse, web}; + +/// Global metrics registry +static REGISTRY: Lazy = Lazy::new(|| Registry::new()); + +/// Global metrics instance +static GLOBAL_METRICS: OnceCell> = OnceCell::new(); + +/// Track if metrics have been registered to prevent duplicate registrations +static METRICS_REGISTERED: OnceCell = OnceCell::new(); + +/// Custom metrics for XLMate backend +pub struct Metrics { + /// Number of currently active games + pub active_games: Gauge, + + /// Number of active WebSocket connections + pub ws_connections: Gauge, + + /// Database query duration in seconds + pub db_query_duration: Histogram, + + /// Number of players in matchmaking queue + pub matchmaking_queue_size: Gauge, + + /// Total AI requests (labeled by type: suggestion, analysis) + pub ai_requests_total: CounterVec, + + /// Total authentication events (labeled by type and success status) + pub auth_events_total: CounterVec, + + /// Total game events (labeled by type: created, completed, abandoned) + pub game_events_total: CounterVec, +} + +impl Metrics { + /// Create a new Metrics instance with all metrics registered + pub fn new() -> Self { + let active_games = Gauge::new( + "xlmate_active_games", + "Current number of active games" + ).expect("Failed to create active_games gauge"); + + let ws_connections = Gauge::new( + "xlmate_ws_connections", + "Current number of active WebSocket connections" + ).expect("Failed to create ws_connections gauge"); + + let db_query_duration = Histogram::with_opts( + HistogramOpts::new( + "xlmate_db_query_duration_seconds", + "Database query duration in seconds" + ) + .buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]) + ).expect("Failed to create db_query_duration histogram"); + + let matchmaking_queue_size = Gauge::new( + "xlmate_matchmaking_queue_size", + "Number of players waiting in matchmaking queue" + ).expect("Failed to create matchmaking_queue_size gauge"); + + let ai_requests_total = CounterVec::new( + Opts::new( + "xlmate_ai_requests_total", + "Total number of AI requests" + ), + &["request_type"] + ).expect("Failed to create ai_requests_total counter"); + + let auth_events_total = CounterVec::new( + Opts::new( + "xlmate_auth_events_total", + "Total number of authentication events" + ), + &["event_type", "success"] + ).expect("Failed to create auth_events_total counter"); + + let game_events_total = CounterVec::new( + Opts::new( + "xlmate_game_events_total", + "Total number of game events" + ), + &["event_type"] + ).expect("Failed to create game_events_total counter"); + + // Register all metrics (only once) + METRICS_REGISTERED.get_or_init(|| { + REGISTRY.register(Box::new(active_games.clone())).expect("Failed to register active_games"); + REGISTRY.register(Box::new(ws_connections.clone())).expect("Failed to register ws_connections"); + REGISTRY.register(Box::new(db_query_duration.clone())).expect("Failed to register db_query_duration"); + REGISTRY.register(Box::new(matchmaking_queue_size.clone())).expect("Failed to register matchmaking_queue_size"); + REGISTRY.register(Box::new(ai_requests_total.clone())).expect("Failed to register ai_requests_total"); + REGISTRY.register(Box::new(auth_events_total.clone())).expect("Failed to register auth_events_total"); + REGISTRY.register(Box::new(game_events_total.clone())).expect("Failed to register game_events_total"); + true + }); + + Metrics { + active_games, + ws_connections, + db_query_duration, + matchmaking_queue_size, + ai_requests_total, + auth_events_total, + game_events_total, + } + } + + /// Get the global registry + pub fn registry() -> &'static Registry { + ®ISTRY + } +} + +/// Initialize metrics and return shared instance (idempotent) +pub fn init_metrics() -> Arc { + GLOBAL_METRICS.get_or_init(|| { + Arc::new(Metrics::new()) + }).clone() +} + +/// Get global metrics instance +fn get_global_metrics() -> Option<&'static Arc> { + GLOBAL_METRICS.get() +} + +/// Increment active games counter +pub fn increment_active_games() { + if let Some(metrics) = get_global_metrics() { + metrics.active_games.inc(); + } +} + +/// Decrement active games counter +pub fn decrement_active_games() { + if let Some(metrics) = get_global_metrics() { + metrics.active_games.dec(); + } +} + +/// Increment WebSocket connections counter +pub fn increment_ws_connections() { + if let Some(metrics) = get_global_metrics() { + metrics.ws_connections.inc(); + } +} + +/// Decrement WebSocket connections counter +pub fn decrement_ws_connections() { + if let Some(metrics) = get_global_metrics() { + metrics.ws_connections.dec(); + } +} + +/// Set WebSocket connections to specific value +pub fn set_ws_connections(count: i64) { + if let Some(metrics) = get_global_metrics() { + metrics.ws_connections.set(count as f64); + } +} + +/// Observe database query duration +pub fn observe_db_query_duration(duration: f64) { + if let Some(metrics) = get_global_metrics() { + metrics.db_query_duration.observe(duration); + } +} + +/// Set matchmaking queue size +pub fn set_matchmaking_queue_size(size: i64) { + if let Some(metrics) = get_global_metrics() { + metrics.matchmaking_queue_size.set(size as f64); + } +} + +/// Increment matchmaking queue size +pub fn increment_matchmaking_queue() { + if let Some(metrics) = get_global_metrics() { + metrics.matchmaking_queue_size.inc(); + } +} + +/// Decrement matchmaking queue size +pub fn decrement_matchmaking_queue() { + if let Some(metrics) = get_global_metrics() { + metrics.matchmaking_queue_size.dec(); + } +} + +/// Increment AI requests counter +pub fn increment_ai_requests(request_type: &str) { + if let Some(metrics) = get_global_metrics() { + metrics.ai_requests_total + .with_label_values(&[request_type]) + .inc(); + } +} + +/// Increment authentication events counter +pub fn increment_auth_events(event_type: &str, success: bool) { + if let Some(metrics) = get_global_metrics() { + metrics.auth_events_total + .with_label_values(&[event_type, if success { "true" } else { "false" }]) + .inc(); + } +} + +/// Increment game events counter +pub fn increment_game_events(event_type: &str) { + if let Some(metrics) = get_global_metrics() { + metrics.game_events_total + .with_label_values(&[event_type]) + .inc(); + } +} + +/// Metrics endpoint handler - returns Prometheus-formatted metrics +pub async fn metrics_handler() -> HttpResponse { + let encoder = TextEncoder::new(); + let metric_families = REGISTRY.gather(); + let mut buffer = Vec::new(); + + match encoder.encode(&metric_families, &mut buffer) { + Ok(_) => { + HttpResponse::Ok() + .content_type("text/plain; version=0.0.4; charset=utf-8") + .body(buffer) + } + Err(e) => { + HttpResponse::InternalServerError() + .body(format!("Failed to encode metrics: {}", e)) + } + } +} diff --git a/backend/modules/api/src/metrics_tests.rs b/backend/modules/api/src/metrics_tests.rs new file mode 100644 index 0000000..82c7e86 --- /dev/null +++ b/backend/modules/api/src/metrics_tests.rs @@ -0,0 +1,194 @@ +#[cfg(test)] +mod tests { + use crate::metrics::{Metrics, init_metrics}; + use prometheus::Encoder; + use std::sync::Arc; + + // Initialize metrics once for all tests + fn get_metrics() -> Arc { + init_metrics() + } + + #[test] + fn test_metrics_initialization() { + // Initialize metrics + let metrics = get_metrics(); + + // Verify metrics instance is created + assert!(metrics.active_games.get() >= 0.0); + assert!(metrics.ws_connections.get() >= 0.0); + } + + #[test] + fn test_active_games_increment_decrement() { + let metrics = get_metrics(); + + let initial_value = metrics.active_games.get(); + + // Increment + metrics.active_games.inc(); + assert_eq!(metrics.active_games.get(), initial_value + 1.0); + + // Decrement + metrics.active_games.dec(); + assert_eq!(metrics.active_games.get(), initial_value); + } + + #[test] + fn test_ws_connections_increment_decrement() { + let metrics = get_metrics(); + + let initial_value = metrics.ws_connections.get(); + + // Increment + metrics.ws_connections.inc(); + assert_eq!(metrics.ws_connections.get(), initial_value + 1.0); + + // Decrement + metrics.ws_connections.dec(); + assert_eq!(metrics.ws_connections.get(), initial_value); + } + + #[test] + fn test_ws_connections_set() { + let metrics = get_metrics(); + + // Set to specific value + metrics.ws_connections.set(42.0); + assert_eq!(metrics.ws_connections.get(), 42.0); + + // Set to another value + metrics.ws_connections.set(10.0); + assert_eq!(metrics.ws_connections.get(), 10.0); + } + + #[test] + fn test_db_query_duration_histogram() { + let metrics = get_metrics(); + + // Observe some durations + metrics.db_query_duration.observe(0.05); + metrics.db_query_duration.observe(0.1); + metrics.db_query_duration.observe(0.15); + + // Verify the histogram has samples + let metric_protos = metrics.db_query_duration.collect(); + assert!(!metric_protos.is_empty()); + } + + #[test] + fn test_counters_increment() { + let metrics = get_metrics(); + + // Increment counters with labels + metrics.ai_requests_total.with_label_values(&["suggestion"]).inc(); + metrics.auth_events_total.with_label_values(&["login", "true"]).inc(); + metrics.game_events_total.with_label_values(&["created"]).inc(); + + // Verify counters incremented + assert_eq!(metrics.ai_requests_total.with_label_values(&["suggestion"]).get(), 1.0); + assert_eq!(metrics.auth_events_total.with_label_values(&["login", "true"]).get(), 1.0); + assert_eq!(metrics.game_events_total.with_label_values(&["created"]).get(), 1.0); + } + + #[test] + fn test_matchmaking_queue_operations() { + let metrics = get_metrics(); + + let initial_value = metrics.matchmaking_queue_size.get(); + + // Increment queue + metrics.matchmaking_queue_size.inc(); + assert_eq!(metrics.matchmaking_queue_size.get(), initial_value + 1.0); + + // Set to specific value + metrics.matchmaking_queue_size.set(15.0); + assert_eq!(metrics.matchmaking_queue_size.get(), 15.0); + + // Decrement queue + metrics.matchmaking_queue_size.dec(); + assert_eq!(metrics.matchmaking_queue_size.get(), 14.0); + } + + #[test] + fn test_metrics_registry_gather() { + let metrics = get_metrics(); + + // Gather metrics from registry + let registry = Metrics::registry(); + let metric_families = registry.gather(); + + // Verify we have metrics registered + assert!(!metric_families.is_empty()); + + // Should have at least our 7 custom metrics + assert!(metric_families.len() >= 7); + } + + #[test] + fn test_metrics_encoding() { + let metrics = get_metrics(); + + // Perform some operations + metrics.active_games.inc(); + metrics.ws_connections.set(5.0); + metrics.ai_requests_total.with_label_values(&["suggestion"]).inc(); + + // Encode metrics + let encoder = prometheus::TextEncoder::new(); + let registry = Metrics::registry(); + let metric_families = registry.gather(); + let mut buffer = Vec::new(); + + let result = encoder.encode(&metric_families, &mut buffer); + + // Encoding should succeed + assert!(result.is_ok()); + + // Buffer should contain metric data + assert!(!buffer.is_empty()); + + // Convert to string and verify format + let output = String::from_utf8(buffer).unwrap(); + assert!(output.contains("xlmate_active_games")); + assert!(output.contains("xlmate_ws_connections")); + assert!(output.contains("xlmate_ai_requests_total")); + } + + #[test] + fn test_concurrent_metric_updates() { + use std::thread; + + let metrics = get_metrics(); + + // Snapshot counter values before spawning threads + let initial_active_games = metrics.active_games.get(); + let initial_ai_requests = metrics.ai_requests_total.with_label_values(&["suggestion"]).get(); + + let mut handles = vec![]; + + // Spawn multiple threads to increment metrics + for _ in 0..10 { + let metrics_clone = metrics.clone(); + let handle = thread::spawn(move || { + for _ in 0..100 { + metrics_clone.active_games.inc(); + metrics_clone.ai_requests_total.with_label_values(&["suggestion"]).inc(); + } + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } + + // Verify delta is correct (10 threads * 100 increments) + let delta_active_games = metrics.active_games.get() - initial_active_games; + let delta_ai_requests = metrics.ai_requests_total.with_label_values(&["suggestion"]).get() - initial_ai_requests; + + assert_eq!(delta_active_games, 1000.0); + assert_eq!(delta_ai_requests, 1000.0); + } +} diff --git a/backend/modules/api/src/server.rs b/backend/modules/api/src/server.rs index 3dab72e..deb3076 100644 --- a/backend/modules/api/src/server.rs +++ b/backend/modules/api/src/server.rs @@ -18,12 +18,14 @@ use crate::auth::{login, register, refresh, logout}; use crate::ai::{get_ai_suggestion, analyze_position}; use crate::ws::{LobbyState, ws_route}; use crate::config::AppConfig; +use crate::metrics::{init_metrics, metrics_handler}; use actix_governor::{Governor, GovernorConfigBuilder}; use matchmaking::service::MatchmakingService; use matchmaking::redis::{create_redis_pool, test_redis_connection}; use challenge::puzzle_validation::PuzzleValidationService; use challenge::api::configure_puzzle_routes; use st_core::endpoint::configure as configure_nft_routes; +use actix_web_prom::PrometheusMetrics; use crate::openapi::ApiDoc; @@ -101,6 +103,15 @@ pub async fn main() -> std::io::Result<()> { // Initialize Puzzle Validation Service let puzzle_service = Arc::new(PuzzleValidationService::new(jwt_secret.clone())); + // Initialize Metrics + let metrics = init_metrics(); + + // Configure Prometheus middleware for HTTP metrics (shared across all workers) + let prometheus: PrometheusMetrics = actix_web_prom::PrometheusMetricsBuilder::new("xlmate_http") + .endpoint("/metrics/http") + .build() + .expect("Failed to create Prometheus metrics middleware"); + eprintln!("Starting HTTP server on {}", server_addr); // Define the app factory closure @@ -110,6 +121,8 @@ pub async fn main() -> std::io::Result<()> { let jwt_secret = jwt_secret.clone(); let matchmaking_service = matchmaking_service.clone(); let puzzle_service = puzzle_service.clone(); + let metrics = metrics.clone(); + let prometheus = prometheus.clone(); // Configure CORS middleware with environment variables for flexibility let cors = { @@ -153,15 +166,18 @@ pub async fn main() -> std::io::Result<()> { App::new() // Global middleware .wrap(cors) + .wrap(prometheus) // App data .app_data(web::Data::from(db.clone())) .app_data(web::Data::new(jwt_service.clone())) .app_data(web::Data::new(lobby.clone())) .app_data(web::Data::new(matchmaking_service.clone())) .app_data(web::Data::new(puzzle_service.clone())) + .app_data(web::Data::new(metrics)) // Register your routes .route("/health", web::get().to(health)) .route("/", web::get().to(greet)) + .route("/metrics", web::get().to(metrics_handler)) // Puzzle routes .configure(configure_puzzle_routes) // Player routes diff --git a/backend/modules/api/src/ws.rs b/backend/modules/api/src/ws.rs index 2fbea83..28c4553 100644 --- a/backend/modules/api/src/ws.rs +++ b/backend/modules/api/src/ws.rs @@ -9,12 +9,13 @@ use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm}; use actix_web::error::ErrorUnauthorized; use serde_json::{Value, json}; use uuid::Uuid; +use std::sync::Arc; +use crate::metrics::{increment_ws_connections, decrement_ws_connections, Metrics}; // For Redis Pub/Sub use redis::AsyncCommands; use redis::aio::ConnectionManager; use tokio::task::JoinHandle; -use std::sync::Arc; /// Core WebSocket message types #[derive(Message, Serialize, Clone, Debug, PartialEq)] @@ -150,6 +151,9 @@ impl Actor for WsSession { self.hb(ctx); let addr = ctx.address().recipient(); self.lobby.do_send(Connect { game_id: self.game_id.clone(), addr }); + + // Track WebSocket connection + increment_ws_connections(); // If Redis connection is available, subscribe to match channel if let Some(redis_conn) = self.redis_conn.clone() { @@ -178,6 +182,9 @@ impl Actor for WsSession { fn stopped(&mut self, ctx: &mut Self::Context) { log::info!("WebSocket disconnected for game: {}", self.game_id); + // Track WebSocket disconnection + decrement_ws_connections(); + // Send reconnection token to client for seamless reconnection if let Ok(reconnect_token) = self.generate_reconnect_token() { let reconnect_msg = WsMessage::ReconnectToken { diff --git a/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml b/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..aaf1d26 --- /dev/null +++ b/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'XLMate Dashboards' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/backend/monitoring/grafana/provisioning/dashboards/xlmate-dashboard.json b/backend/monitoring/grafana/provisioning/dashboards/xlmate-dashboard.json new file mode 100644 index 0000000..c4faa2a --- /dev/null +++ b/backend/monitoring/grafana/provisioning/dashboards/xlmate-dashboard.json @@ -0,0 +1,501 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "xlmate_active_games", + "refId": "A" + } + ], + "title": "Active Games", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "xlmate_ws_connections", + "refId": "A" + } + ], + "title": "WebSocket Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "histogram_quantile(0.95, rate(xlmate_db_query_duration_seconds_bucket[5m]))", + "legendFormat": "P95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "histogram_quantile(0.50, rate(xlmate_db_query_duration_seconds_bucket[5m]))", + "legendFormat": "P50", + "refId": "B" + } + ], + "title": "Database Query Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "rate(xlmate_http_requests_total[5m])", + "legendFormat": "{{method}} {{status}}", + "refId": "A" + } + ], + "title": "HTTP Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "rate(xlmate_game_events_total[5m])", + "legendFormat": "{{event_type}}", + "refId": "A" + } + ], + "title": "Game Events Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "default" + }, + "expr": "rate(xlmate_ai_requests_total[5m])", + "legendFormat": "AI Requests", + "refId": "A" + } + ], + "title": "AI Request Rate", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["xlmate", "chess"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "XLMate Backend Monitoring", + "uid": "xlmate-backend", + "version": 1, + "weekStart": "" +} diff --git a/backend/monitoring/grafana/provisioning/datasources/prometheus.yml b/backend/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..d4a4903 --- /dev/null +++ b/backend/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + uid: default diff --git a/backend/monitoring/prometheus.yml b/backend/monitoring/prometheus.yml new file mode 100644 index 0000000..c5698c9 --- /dev/null +++ b/backend/monitoring/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'xlmate-backend' + static_configs: + - targets: ['host.docker.internal:8080'] + metrics_path: '/metrics' + scrape_interval: 10s + + - job_name: 'xlmate-http-metrics' + static_configs: + - targets: ['host.docker.internal:8080'] + metrics_path: '/metrics/http' + scrape_interval: 10s diff --git a/docker-compose.yml b/docker-compose.yml index 1d4e51c..a582135 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,3 +39,39 @@ volumes: driver: local postgres_data: driver: local + prometheus_data: + driver: local + grafana_data: + driver: local + + prometheus: + image: prom/prometheus:latest + container_name: xlmate-prometheus + ports: + - "9090:9090" + volumes: + - ./backend/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + extra_hosts: + - "host.docker.internal:host-gateway" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + container_name: xlmate-grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?must be set} + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana_data:/var/lib/grafana + - ./backend/monitoring/grafana/provisioning:/etc/grafana/provisioning + restart: unless-stopped + depends_on: + - prometheus