wip

Signed-off-by: Attila Mészáros <a_meszaros@apple.com>

Author: csviri

micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetricsV2.java

@@ -66,7 +66,7 @@ public class MicrometerMetricsV2 implements Metrics {
   private static final String NUMBER_OF_RESOURCE_GAUGE = "custom_resources";
 
   private static final String RECONCILIATION_EXECUTION_DURATION =
-      RECONCILIATIONS + "execution.seconds";
+      RECONCILIATIONS + "execution.duration";
 
   private final MeterRegistry registry;
   private final Map<String, AtomicInteger> gauges = new ConcurrentHashMap<>();
@@ -195,6 +195,7 @@ public void receivedEvent(Event event, Map<String, Object> metadata) {
 
   @Override
   public void cleanupDoneFor(ResourceID resourceID, Map<String, Object> metadata) {
+    gauges.get(numberOfResourcesRefName(getControllerName(metadata))).decrementAndGet();
     incrementCounter(EVENTS_DELETE, resourceID.getNamespace().orElse(null), metadata);
   }
 

observability/josdk-operator-metrics-dashboard.json

@@ -513,26 +513,13 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
           },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "blue",
-                "value": null
-              },
-              {
-                "color": "green",
-                "value": 10
-              },
-              {
-                "color": "yellow",
-                "value": 100
-              }
-            ]
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 10
           },
+          "mappings": [],
           "unit": "short"
         },
         "overrides": []
@@ -545,16 +532,14 @@
       },
       "id": 13,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "values": false,
-          "calcs": ["lastNotNull"],
-          "fields": ""
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
         },
-        "textMode": "auto"
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        }
       },
       "pluginVersion": "10.0.0",
       "targets": [
@@ -571,7 +556,7 @@
         }
       ],
       "title": "Custom Resources Count",
-      "type": "stat"
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -654,7 +639,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.50, sum(rate(reconciliations_execution_seconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
+          "expr": "histogram_quantile(0.50, sum(rate(reconciliations_execution_duration_milliseconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
           "legendFormat": "p50 - {{controller_name}}",
           "range": true,
           "refId": "A"
@@ -665,7 +650,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.95, sum(rate(reconciliations_execution_seconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
+          "expr": "histogram_quantile(0.95, sum(rate(reconciliations_execution_duration_milliseconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
           "legendFormat": "p95 - {{controller_name}}",
           "range": true,
           "refId": "B"
@@ -676,7 +661,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum(rate(reconciliations_execution_seconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
+          "expr": "histogram_quantile(0.99, sum(rate(reconciliations_execution_duration_milliseconds_bucket{service_name=~\"$service_name\"}[5m])) by (le, controller_name))",
           "legendFormat": "p99 - {{controller_name}}",
           "range": true,
           "refId": "C"
@@ -780,7 +765,7 @@
         "type": "prometheus",
         "uid": "prometheus"
       },
-      "description": "Failures by controller",
+      "description": "Reconciliation failures",
       "fieldConfig": {
         "defaults": {
           "color": {

sample-operators/metrics-processing/src/main/resources/otlp-config.yaml

@@ -18,6 +18,6 @@ otlp:
   # OTLP Collector endpoint - see observability/install-observability.sh for setup
 #  url: "http://localhost:4318/v1/metrics"
   url: "http://otel-collector-collector.observability.svc.cluster.local:4318/v1/metrics"
-  step: 10s
+  step: 5s
   batchSize: 15000
   aggregationTemporality: "cumulative"

sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java

@@ -206,7 +206,7 @@ private void verifyPrometheusMetrics() {
     assertMetricPresent(prometheusUrl, "reconciliations_success_total", Duration.ofSeconds(30));
     assertMetricPresent(prometheusUrl, "reconciliations_failure_total", Duration.ofSeconds(30));
     assertMetricPresent(
-        prometheusUrl, "reconciliations_execution_seconds_count", Duration.ofSeconds(30));
+        prometheusUrl, "reconciliations_execution_duration_milliseconds_count", Duration.ofSeconds(30));
 
     log.info("All metrics verified successfully in Prometheus");
   }