Refactor: Use deferred function to eliminate code duplication in provider metrics recording

ginaxu1 · ginaxu1 · commit ce390495409d · 2026-01-06T12:55:36.000+05:30
diff --git a/exchange/consent-engine/main.go b/exchange/consent-engine/main.go
@@ -40,6 +40,17 @@ func main() {
 	// Setup logging
 	utils.SetupLogging(cfg.Logging.Format, cfg.Logging.Level)
 
+	// Initialize monitoring/observability (optional - can be disabled via ENABLE_OBSERVABILITY=false)
+	// Services will continue to function normally even if observability is disabled
+	if monitoring.IsObservabilityEnabled() {
+		monitoringConfig := monitoring.DefaultConfig("consent-engine")
+		if err := monitoring.Initialize(monitoringConfig); err != nil {
+			slog.Warn("Failed to initialize monitoring (service will continue)", "error", err)
+		}
+	} else {
+		slog.Info("Observability disabled via environment variable")
+	}
+
 	slog.Info("Starting consent engine",
 		"environment", cfg.Environment,
 		"port", cfg.Service.Port,
diff --git a/exchange/docker-compose.yml b/exchange/docker-compose.yml
@@ -7,9 +7,9 @@ services:
     image: postgres:16-alpine
     container_name: pdp-db-${ENVIRONMENT:-local}
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=password
-      - POSTGRES_DB=testdb
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - POSTGRES_DB=${POSTGRES_DB:-testdb}
     ports:
       - "${PDP_DB_PORT:-5433}:5432"
     healthcheck:
@@ -20,6 +20,23 @@ services:
     networks:
       - opendif-network
 
+  ce-db:
+    image: postgres:16-alpine
+    container_name: ce-db-${ENVIRONMENT:-local}
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - POSTGRES_DB=${CE_DB_NAME:-consent_db}
+    ports:
+      - "${CE_DB_PORT:-5434}:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - opendif-network
+
   policy-decision-point:
     build:
       context: .
@@ -38,12 +55,14 @@ services:
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_FORMAT=${LOG_FORMAT:-text}
       - SERVICE_NAME=policy-decision-point
+      - ENABLE_OBSERVABILITY=${ENABLE_OBSERVABILITY:-true}
+      - OTEL_METRICS_ENABLED=${OTEL_METRICS_ENABLED:-true}
       - OTEL_METRICS_EXPORTER=${OTEL_METRICS_EXPORTER:-prometheus}
       - CHOREO_OPENDIF_DATABASE_HOSTNAME=pdp-db
       - CHOREO_OPENDIF_DATABASE_PORT=5432
-      - CHOREO_OPENDIF_DATABASE_USERNAME=postgres
-      - CHOREO_OPENDIF_DATABASE_PASSWORD=password
-      - CHOREO_OPENDIF_DATABASE_DATABASENAME=testdb
+      - CHOREO_OPENDIF_DATABASE_USERNAME=${POSTGRES_USER:-postgres}
+      - CHOREO_OPENDIF_DATABASE_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - CHOREO_OPENDIF_DATABASE_DATABASENAME=${POSTGRES_DB:-testdb}
       - DB_SSLMODE=disable
       - RUN_MIGRATION=true
     depends_on:
@@ -76,12 +95,30 @@ services:
       - PORT=8081
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_FORMAT=${LOG_FORMAT:-text}
+      - SERVICE_NAME=consent-engine
+      - ENABLE_OBSERVABILITY=${ENABLE_OBSERVABILITY:-true}
+      - OTEL_METRICS_ENABLED=${OTEL_METRICS_ENABLED:-true}
+      - OTEL_METRICS_EXPORTER=${OTEL_METRICS_EXPORTER:-prometheus}
+      - CHOREO_OPENDIF_DATABASE_HOSTNAME=ce-db
+      - CHOREO_OPENDIF_DATABASE_PORT=5432
+      - CHOREO_OPENDIF_DATABASE_USERNAME=${POSTGRES_USER:-postgres}
+      - CHOREO_OPENDIF_DATABASE_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - CHOREO_OPENDIF_DATABASE_DATABASENAME=${CE_DB_NAME:-consent_db}
+      - DB_SSLMODE=disable
+      - RUN_MIGRATION=true
+      - ASGARDEO_JWKS_URL=${ASGARDEO_JWKS_URL:-https://www.googleapis.com/oauth2/v3/certs}
+      - ASGARDEO_ISSUER=${ASGARDEO_ISSUER:-https://accounts.google.com}
+      - ASGARDEO_AUDIENCE=${ASGARDEO_AUDIENCE:-test-audience}
+      - ASGARDEO_ORG_NAME=${ASGARDEO_ORG_NAME:-test-org}
+    depends_on:
+      ce-db:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/health"]
       interval: 30s
       timeout: 10s
       retries: 3
-      start_period: 10s
+      start_period: 30s
     restart: unless-stopped
     networks:
       - opendif-network
@@ -104,8 +141,16 @@ services:
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_FORMAT=${LOG_FORMAT:-text}
       - SERVICE_NAME=orchestration-engine
+      - ENABLE_OBSERVABILITY=${ENABLE_OBSERVABILITY:-true}
+      - OTEL_METRICS_ENABLED=${OTEL_METRICS_ENABLED:-true}
       - OTEL_METRICS_EXPORTER=${OTEL_METRICS_EXPORTER:-prometheus}
       - CONFIG_PATH=/app/config/config.json
+      - DB_HOST=${DB_HOST?DB_HOST is required}
+      - DB_PORT=${DB_PORT:-5432}
+      - DB_USER=${DB_USER:-postgres}
+      - DB_PASSWORD=${DB_PASSWORD?DB_PASSWORD is required}
+      - DB_NAME=${DB_NAME:-orchestration_engine}
+      - DB_SSLMODE=${DB_SSLMODE:-disable}
     volumes:
       - ./minimal-config.json:/app/config/config.json:ro
     healthcheck:
diff --git a/exchange/orchestration-engine/main.go b/exchange/orchestration-engine/main.go
@@ -16,11 +16,15 @@ import (
 func main() {
 	logger.Init()
 
-	// Initialize monitoring/observability
-	// This ensures metrics are properly set up before the server starts
-	monitoringConfig := monitoring.DefaultConfig("orchestration-engine")
-	if err := monitoring.Initialize(monitoringConfig); err != nil {
-		logger.Log.Warn("Failed to initialize monitoring (service will continue)", "error", err)
+	// Initialize monitoring/observability (optional - can be disabled via ENABLE_OBSERVABILITY=false)
+	// Services will continue to function normally even if observability is disabled
+	if monitoring.IsObservabilityEnabled() {
+		monitoringConfig := monitoring.DefaultConfig("orchestration-engine")
+		if err := monitoring.Initialize(monitoringConfig); err != nil {
+			logger.Log.Warn("Failed to initialize monitoring (service will continue)", "error", err)
+		}
+	} else {
+		logger.Log.Info("Observability disabled via environment variable")
 	}
 
 	// Load configuration with proper error handling
diff --git a/exchange/orchestration-engine/provider/provider.go b/exchange/orchestration-engine/provider/provider.go
@@ -59,27 +59,28 @@ func (p *Provider) PerformRequest(ctx context.Context, reqBody []byte) (resp *ht
 	req.Header.Set("Content-Type", "application/json")
 
 	start := time.Now()
+	defer func() {
+		monitoring.RecordExternalCall(p.ServiceKey, "provider_request", time.Since(start), err)
+	}()
 
 	if p.Auth != nil {
 		switch p.Auth.Type {
 		case auth.AuthTypeOAuth2:
 			if p.OAuth2Config == nil {
 				err = fmt.Errorf("OAuth2Config is nil")
 				logger.Log.Error(err.Error(), "providerKey", p.ServiceKey)
-				return nil, err
+				return
 			}
 
 			client := p.OAuth2Config.Client(ctx)
 			resp, err = client.Do(req)
-			monitoring.RecordExternalCall(p.ServiceKey, "provider_request", time.Since(start), err)
-			return resp, err
+			return
 		case auth.AuthTypeAPIKey:
 			req.Header.Set(p.Auth.APIKeyName, p.Auth.APIKeyValue)
 		}
 	}
 
 	// Default client execution (for API Key or no auth)
 	resp, err = p.Client.Do(req)
-	monitoring.RecordExternalCall(p.ServiceKey, "provider_request", time.Since(start), err)
-	return resp, err
+	return
 }
diff --git a/exchange/policy-decision-point/main.go b/exchange/policy-decision-point/main.go
@@ -27,11 +27,15 @@ func main() {
 	// Setup logging
 	utils.SetupLogging("json", getEnvOrDefault("LOG_LEVEL", "info"))
 
-	// Initialize monitoring/observability
-	monitoringConfig := monitoring.DefaultConfig("policy-decision-point")
-	if err := monitoring.Initialize(monitoringConfig); err != nil {
-		slog.Error("Failed to initialize monitoring", "error", err)
-		// Don't exit - service can continue without metrics
+	// Initialize monitoring/observability (optional - can be disabled via ENABLE_OBSERVABILITY=false)
+	// Services will continue to function normally even if observability is disabled
+	if monitoring.IsObservabilityEnabled() {
+		monitoringConfig := monitoring.DefaultConfig("policy-decision-point")
+		if err := monitoring.Initialize(monitoringConfig); err != nil {
+			slog.Warn("Failed to initialize monitoring (service will continue)", "error", err)
+		}
+	} else {
+		slog.Info("Observability disabled via environment variable")
 	}
 
 	slog.Info("Starting policy decision point (V1)",
diff --git a/exchange/shared/monitoring/metrics.go b/exchange/shared/monitoring/metrics.go
@@ -26,8 +26,22 @@ var (
 
 // ensureInitialized ensures OpenTelemetry is initialized with default config
 // This is called automatically when metrics functions are used
+// Observability can be disabled via ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false
 func ensureInitialized() {
 	initOnce.Do(func() {
+		// Check if observability is explicitly disabled
+		enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true)
+		otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true)
+
+		if !enableObservability || !otelMetricsEnabled {
+			slog.Info("Observability disabled via environment variable, skipping initialization",
+				"ENABLE_OBSERVABILITY", enableObservability,
+				"OTEL_METRICS_ENABLED", otelMetricsEnabled)
+			// Set initErr to nil so IsInitialized() returns false but no error is reported
+			initErr = nil
+			return
+		}
+
 		// Try to get service name from environment or use default
 		serviceName := os.Getenv("SERVICE_NAME")
 		if serviceName == "" {
@@ -61,6 +75,15 @@ func IsInitialized() bool {
 	return initErr == nil
 }
 
+// IsObservabilityEnabled checks if observability is enabled via environment variables
+// Returns false if ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false
+// Returns true otherwise (default behavior)
+func IsObservabilityEnabled() bool {
+	enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true)
+	otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true)
+	return enableObservability && otelMetricsEnabled
+}
+
 // RegisterRoutes registers routes for normalization. Supports static routes and templates with :id or {id} placeholders.
 // Templates match incoming paths and normalize dynamic segments. Call during service initialization.
 //
diff --git a/exchange/shared/monitoring/metrics_test.go b/exchange/shared/monitoring/metrics_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -427,6 +428,85 @@ func TestMultipleInitializations(t *testing.T) {
 	}
 }
 
+// TestIsObservabilityEnabled tests the IsObservabilityEnabled function
+func TestIsObservabilityEnabled(t *testing.T) {
+	tests := []struct {
+		name                string
+		enableObservability string
+		otelMetricsEnabled  string
+		expected            bool
+	}{
+		{
+			name:                "Both enabled (default)",
+			enableObservability: "",
+			otelMetricsEnabled:  "",
+			expected:            true,
+		},
+		{
+			name:                "ENABLE_OBSERVABILITY=false disables",
+			enableObservability: "false",
+			otelMetricsEnabled:  "",
+			expected:            false,
+		},
+		{
+			name:                "OTEL_METRICS_ENABLED=false disables",
+			enableObservability: "",
+			otelMetricsEnabled:  "false",
+			expected:            false,
+		},
+		{
+			name:                "Both false disables",
+			enableObservability: "false",
+			otelMetricsEnabled:  "false",
+			expected:            false,
+		},
+		{
+			name:                "Both true enables",
+			enableObservability: "true",
+			otelMetricsEnabled:  "true",
+			expected:            true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Save original values
+			origEnable := os.Getenv("ENABLE_OBSERVABILITY")
+			origOtel := os.Getenv("OTEL_METRICS_ENABLED")
+
+			// Set test values
+			if tt.enableObservability != "" {
+				os.Setenv("ENABLE_OBSERVABILITY", tt.enableObservability)
+			} else {
+				os.Unsetenv("ENABLE_OBSERVABILITY")
+			}
+			if tt.otelMetricsEnabled != "" {
+				os.Setenv("OTEL_METRICS_ENABLED", tt.otelMetricsEnabled)
+			} else {
+				os.Unsetenv("OTEL_METRICS_ENABLED")
+			}
+
+			// Test (function doesn't depend on initOnce, so no reset needed)
+			result := IsObservabilityEnabled()
+			if result != tt.expected {
+				t.Errorf("IsObservabilityEnabled() = %v, want %v", result, tt.expected)
+			}
+
+			// Restore original values
+			if origEnable != "" {
+				os.Setenv("ENABLE_OBSERVABILITY", origEnable)
+			} else {
+				os.Unsetenv("ENABLE_OBSERVABILITY")
+			}
+			if origOtel != "" {
+				os.Setenv("OTEL_METRICS_ENABLED", origOtel)
+			} else {
+				os.Unsetenv("OTEL_METRICS_ENABLED")
+			}
+		})
+	}
+}
+
 // TestHTTPMetricsMiddlewareWithDifferentStatusCodes tests that different HTTP status codes are recorded
 func TestHTTPMetricsMiddlewareWithDifferentStatusCodes(t *testing.T) {
 	testCases := []struct {
diff --git a/exchange/shared/monitoring/otel_metrics.go b/exchange/shared/monitoring/otel_metrics.go
@@ -87,9 +87,28 @@ type Config struct {
 }
 
 // DefaultConfig returns a default configuration
+// Observability can be disabled by setting ENABLE_OBSERVABILITY=false or OTEL_METRICS_ENABLED=false
 func DefaultConfig(serviceName string) Config {
+	// Check if observability is explicitly disabled
+	enableObservability := getEnvBoolOrDefault("ENABLE_OBSERVABILITY", true)
+	otelMetricsEnabled := getEnvBoolOrDefault("OTEL_METRICS_ENABLED", true)
+
+	// If either flag is false, disable observability
+	observabilityEnabled := enableObservability && otelMetricsEnabled
+
+	var exporterType string
+	if !observabilityEnabled {
+		exporterType = "none"
+		slog.Info("Observability disabled via environment variable",
+			"service", serviceName,
+			"ENABLE_OBSERVABILITY", enableObservability,
+			"OTEL_METRICS_ENABLED", otelMetricsEnabled)
+	} else {
+		exporterType = getEnvOrDefault("OTEL_METRICS_EXPORTER", "prometheus")
+	}
+
 	return Config{
-		ExporterType:     getEnvOrDefault("OTEL_METRICS_EXPORTER", "prometheus"),
+		ExporterType:     exporterType,
 		ServiceName:      serviceName,
 		ServiceVersion:   getEnvOrDefault("SERVICE_VERSION", "dev"),
 		OTLPEndpoint:     getEnvOrDefault("OTEL_EXPORTER_OTLP_ENDPOINT", ""),
diff --git a/observability/README.md b/observability/README.md
@@ -85,7 +85,7 @@ Run the verification script to start everything and verify metrics are working:
 
 ```bash
 cd observability
-./verification_steps.sh
+./verify_observability_setup.sh
 ```
 
 This script will:
diff --git a/observability/prometheus/prometheus.yml b/observability/prometheus/prometheus.yml
diff --git a/observability/verification_steps.sh b/observability/verification_steps.sh
diff --git a/observability/verify_observability_setup.sh b/observability/verify_observability_setup.sh
