From 70bdf58daba184a8d7c8776513f240539c7b2954 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 4 Jul 2026 11:43:59 +0700 Subject: [PATCH 1/3] feat: premade grafana dashboard --- grafana-dashboards/README.md | 38 + .../sentry-background-processing.json | 1851 +++++++++++++ .../sentry-errors-and-failures.json | 945 +++++++ grafana-dashboards/sentry-ingestion.json | 1876 +++++++++++++ grafana-dashboards/sentry-overview.json | 2434 +++++++++++++++++ 5 files changed, 7144 insertions(+) create mode 100644 grafana-dashboards/README.md create mode 100644 grafana-dashboards/sentry-background-processing.json create mode 100644 grafana-dashboards/sentry-errors-and-failures.json create mode 100644 grafana-dashboards/sentry-ingestion.json create mode 100644 grafana-dashboards/sentry-overview.json diff --git a/grafana-dashboards/README.md b/grafana-dashboards/README.md new file mode 100644 index 00000000000..e9738e5a696 --- /dev/null +++ b/grafana-dashboards/README.md @@ -0,0 +1,38 @@ +# Sentry Self-Hosted Grafana Dashboards + +A starting set of Grafana dashboards for monitoring Sentry self-hosted installations via the StatsD metrics emitted by Sentry services (compatible with Prometheus, VictoriaMetrics, or Graphite with query rewrites). + +## Dashboards + +| File | Purpose | +|---|---| +| `sentry-overview.json` | High-level system health: API volume, Snuba queries, emails, webhooks, workflow engine, nodestore, and ClickHouse connections. | +| `sentry-ingestion.json` | Event ingestion flow: accepted/rejected events, Relay health, event pipeline latency, Symbolicator, project config, and Relay/Redis. | +| `sentry-background-processing.json` | Workers, task brokers, Arroyo consumers/producers: task throughput, execution duration, RPC latency, Kafka queue health, and backpressure. | +| `sentry-errors-and-failures.json` | Failure signals: rejected events, dropped messages, task RPC errors, and Snuba query health. | + +## Metric name version + +These dashboards are written against the dot-separated StatsD metric names emitted by Sentry self-hosted, for example: + +- `stats.counters.sentry.relay.event.accepted.count` +- `stats.timers.sentry.events.since_received.median` +- `stats.counters.sentry.taskworker.worker.execute_task.count` + +If you are using a statsd-to-prometheus exporter that converts dots to underscores, update the queries accordingly. + +### Why `stats..*`? + +The `stats.counters.*`, `stats.timers.*`, and `stats.gauges.*` prefixes come from the default StatsD naming convention, not from Sentry itself. Sentry emits metric names like `sentry.relay.event.accepted`; the StatsD server then stores them as `stats.counters.sentry.relay.event.accepted.count`. This is the expected behavior when using the default StatsD settings. + +## Importing + +1. Open Grafana → **Dashboards** → **Import**. +2. Upload one of the JSON files from this directory. +3. Select your VictoriaMetrics/Prometheus data source. +4. The `host` variable will populate automatically from the `host` label. + +## Notes + +- Dashboards default to the last hour with a 30-second refresh. +- Several panels group by labels documented in Sentry's monitoring guide (e.g. `url_name`, `method`, `taskname`, `stage`). If your installation does not yet emit those tags, the panels will aggregate to a single series until the tags are present. diff --git a/grafana-dashboards/sentry-background-processing.json b/grafana-dashboards/sentry-background-processing.json new file mode 100644 index 00000000000..7ef7c679c6c --- /dev/null +++ b/grafana-dashboards/sentry-background-processing.json @@ -0,0 +1,1851 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Background workers, task brokers, and consumer health for Sentry self-hosted.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 101, + "panels": [], + "title": "Task Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "scheduled / sec", + "refId": "A" + } + ], + "title": "Tasks Scheduled / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "executed / sec", + "refId": "A" + } + ], + "title": "Tasks Executed / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "success / sec", + "refId": "A" + } + ], + "title": "Send Task Success / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.scheduler.delay_task.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "delayed / sec", + "refId": "A" + } + ], + "title": "Delayed Tasks / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "scheduled", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "executed", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "success", + "refId": "C" + } + ], + "title": "Task Schedule vs Execute vs Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "C" + } + ], + "title": "Task Execution Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 107, + "panels": [], + "title": "Taskworker RPC & Queue Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.get_task.rpc.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "get_task p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.get_task.rpc.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "get_task p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.update_task.rpc.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "update_task p50", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.taskworker.update_task.rpc.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "update_task p90", + "refId": "D" + } + ], + "title": "Task RPC Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.fetch_task.not_found.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "fetch not found", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.child_task_queue_empty.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "child queue empty", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.add_tasks.child_tasks_full.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "child tasks full", + "refId": "C" + } + ], + "title": "Queue Saturation Signals", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 109, + "panels": [], + "title": "Consumer Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "loops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "run count", + "refId": "A" + } + ], + "title": "Consumer Run Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + } + ], + "title": "Consumer Processing Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 110, + "panels": [], + "title": "Arroyo Consumers & Producers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 111, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "sentry consumer run", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.producer.produce_status.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "sentry producer status", + "refId": "B" + } + ], + "title": "Consumer Throughput (Sentry)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "snuba consumer run", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.producer.produce_status.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "snuba producer status", + "refId": "B" + } + ], + "title": "Consumer Throughput (Snuba)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba p50", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba p90", + "refId": "D" + } + ], + "title": "Consumer Processing Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 92 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry backpressure p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry backpressure p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba backpressure p50", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba backpressure p90", + "refId": "D" + } + ], + "title": "Consumer Poll & Backpressure Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 100 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry consumer queue", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.consumer.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba consumer queue", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.message_count{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry producer msgs", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.message_count{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba producer msgs", + "refId": "D" + } + ], + "title": "Kafka Queue Sizes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "latency", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 108 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_int_latency{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry internal latency", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "sentry outbuf latency", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.p99_int_latency{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba internal latency", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "snuba outbuf latency", + "refId": "D" + } + ], + "title": "Producer Latency (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "background-processing" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Sentry Self-Hosted: Background Processing", + "uid": "sentry-self-hosted-background-processing", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-errors-and-failures.json b/grafana-dashboards/sentry-errors-and-failures.json new file mode 100644 index 00000000000..f62c47b1e1b --- /dev/null +++ b/grafana-dashboards/sentry-errors-and-failures.json @@ -0,0 +1,945 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Failure signals across Relay, task brokers, consumers, and Snuba for Sentry self-hosted.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 101, + "panels": [], + "title": "Ingestion Failures", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Rejected / sec", + "refId": "A" + } + ], + "title": "Rejected Events / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Errors / sec", + "refId": "A" + } + ], + "title": "Error Events Processed / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.client.rpc_error.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC errors / sec", + "refId": "A" + } + ], + "title": "Taskworker RPC Errors / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "{{item_type}}", + "refId": "A" + } + ], + "title": "Rejected Events by Item Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "responses / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(status_code) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Relay Response Status Codes", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 106, + "panels": [], + "title": "Consumer & Processing Failures", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.strategies.filter.dropped_messages.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "dropped messages", + "refId": "A" + } + ], + "title": "Dropped Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "errors / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.checkin.handle_processing_error.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "errors", + "refId": "A" + } + ], + "title": "Monitor Checkin Processing Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 108, + "panels": [], + "title": "Snuba Query Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "queries / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "cache hit", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.db_query.cache_miss.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "cache miss", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.db_query.successful_query.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "successful", + "refId": "C" + } + ], + "title": "Snuba DB Query Hit/Miss", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.allocation_policy.bytes_scanned.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "bytes scanned", + "refId": "A" + } + ], + "title": "Snuba Allocation Policy Bytes Scanned", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "errors", + "failures" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Sentry Self-Hosted: Errors & Failures", + "uid": "sentry-self-hosted-errors-failures", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-ingestion.json b/grafana-dashboards/sentry-ingestion.json new file mode 100644 index 00000000000..1d66b1d9eeb --- /dev/null +++ b/grafana-dashboards/sentry-ingestion.json @@ -0,0 +1,1876 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Event ingestion flow for Sentry self-hosted. Tracks accepted/rejected events, Relay health, and pipeline latency.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 101, + "panels": [], + "title": "Ingestion Volume", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Events accepted / sec", + "refId": "A" + } + ], + "title": "Events Accepted / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Events rejected / sec", + "refId": "A" + } + ], + "title": "Events Rejected / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.server.http.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "HTTP accepted / sec", + "refId": "A" + } + ], + "title": "Relay HTTP Accepted / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.requests.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Requests / sec", + "refId": "A" + } + ], + "title": "Relay Requests / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "{{item_type}}", + "refId": "A" + } + ], + "title": "Event Items by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "rejected", + "refId": "B" + } + ], + "title": "Accepted vs Rejected Events", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 107, + "panels": [], + "title": "Relay Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "responses / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(status_code) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "title": "Relay Responses by Status Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.producer.produce_status.success.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "success", + "refId": "A" + } + ], + "title": "Relay Producer Success Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 109, + "panels": [], + "title": "Pipeline Latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.events.since_received.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "C" + } + ], + "title": "Event Pipeline Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 110, + "panels": [], + "title": "Symbolicator", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.events.symbolicator.create_task.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "tasks created", + "refId": "A" + } + ], + "title": "Symbolicator Tasks Created", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.events.symbolicator.response.completed.size.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.events.symbolicator.response.completed.size.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + } + ], + "title": "Symbolicator Response Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 111, + "panels": [], + "title": "Project Config", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "C" + } + ], + "title": "Project Config Fetch Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.relay.projectconfig_cache.size.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.timers.sentry.relay.projectconfig_cache.size.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + } + ], + "title": "Project Config Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.write.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "writes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "scheduled", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.invalidation.recompute.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "invalidations", + "refId": "C" + } + ], + "title": "Project Config Cache Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "configs / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.fetched.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "fetched", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "pending", + "refId": "B" + } + ], + "title": "Project Configs Fetched / Pending", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "id": 116, + "panels": [], + "title": "Relay & Redis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 110 + }, + "id": 117, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.connections{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "active", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "waiting", + "refId": "D" + } + ], + "title": "Redis Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "id": 118, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.redis.connection.create.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "created", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.redis.connection.recycle.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "recycled", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.project_state.redis.requests.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "project state requests", + "refId": "C" + } + ], + "title": "Redis Connection Operations", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "ingestion" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Sentry Self-Hosted: Ingestion", + "uid": "sentry-self-hosted-ingestion", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-overview.json b/grafana-dashboards/sentry-overview.json new file mode 100644 index 00000000000..e50fa60fd41 --- /dev/null +++ b/grafana-dashboards/sentry-overview.json @@ -0,0 +1,2434 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Sentry self-hosted monitoring dashboard for VictoriaMetrics. Visualizes notable metric keys from https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/#notable-metric-keys" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "API Request Volume", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "requests / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(url_name, method) (rate(stats.counters.sentry.view.response.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{method}} {{url_name}}", + "refId": "A" + } + ], + "title": "API Request Rate by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50 {{method}} {{url_name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90 {{method}} {{url_name}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max {{method}} {{url_name}}", + "refId": "C" + } + ], + "title": "API Request Duration by Endpoint", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 101, + "panels": [], + "title": "Task Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "scheduled: {{taskname}}", + "refId": "A" + } + ], + "title": "Tasks Scheduled Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "executed: {{taskname}}", + "refId": "A" + } + ], + "title": "Tasks Executed Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50 {{taskname}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper.90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90 {{taskname}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max {{taskname}}", + "refId": "C" + } + ], + "title": "Task Execution Duration by Task", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 102, + "panels": [], + "title": "Error Processing", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Events accepted / sec", + "refId": "A" + } + ], + "title": "Events Accepted Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 47 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Relay error events processed / sec", + "refId": "A" + } + ], + "title": "Relay Error Events Processed Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 51 + }, + "id": 8, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{item_type}}", + "refId": "A" + } + ], + "title": "Event Items by Type", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 59 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{item_type}}", + "refId": "A" + } + ], + "title": "Relay Event Items Rate by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.median{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p50 {{stage}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper.90{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "p90 {{stage}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "max {{stage}}", + "refId": "C" + } + ], + "title": "Event Pipeline Latency by Stage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 103, + "panels": [], + "title": "Snuba Queries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 72 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.clickhouse.native.connections{host=~\"$host\"})", + "interval": "", + "legendFormat": "connections", + "refId": "A" + } + ], + "title": "ClickHouse Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 76 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "run loops / sec", + "refId": "A" + } + ], + "title": "Subscription Executor Queries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 80 + }, + "id": 106, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.schedule.size{host=~\"$host\"})", + "interval": "", + "legendFormat": "scheduled subscriptions", + "refId": "A" + } + ], + "title": "Subscription Schedule Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 84 + }, + "id": 107, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.calculated_max_concurrent_queries{host=~\"$host\"})", + "interval": "", + "legendFormat": "max concurrent", + "refId": "A" + } + ], + "title": "Max Concurrent Queries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "consumer run loops", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.processing.strategies.healthcheck.touch.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "healthcheck touches", + "refId": "B" + } + ], + "title": "Snuba Subscription Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.tick_buffer.queue_size{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "tick buffer queue", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", + "hide": false, + "interval": "", + "legendFormat": "kafka queue", + "refId": "B" + } + ], + "title": "Snuba Subscription Queue Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 110, + "panels": [], + "title": "Emails", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 105 + }, + "id": 111, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "sent / sec", + "refId": "A" + } + ], + "title": "Emails Sent / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 109 + }, + "id": 112, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "queued / sec", + "refId": "A" + } + ], + "title": "Emails Queued / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "emails / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 113 + }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "sent", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "queued", + "refId": "B" + } + ], + "title": "Email Throughput", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 114, + "panels": [], + "title": "Webhooks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "webhooks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "started", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "success", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "prepare started", + "refId": "C" + } + ], + "title": "Sentry App Webhook Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "webhooks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 130 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.webhook.started.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "started", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.webhook.success.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "success", + "refId": "B" + } + ], + "title": "Integration Webhook Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 138 + }, + "id": 117, + "panels": [], + "title": "Workflow Engine", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 139 + }, + "id": 118, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "process workflows", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.data_condition.evaluation.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "condition evaluations", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "activities completed", + "refId": "C" + } + ], + "title": "Workflow Engine Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 147 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.hit.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "hit", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.miss.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "miss", + "refId": "B" + } + ], + "title": "Workflow Engine Cache Hit/Miss", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 155 + }, + "id": 120, + "panels": [], + "title": "Nodestore", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 156 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "get", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "get_multi", + "refId": "B" + } + ], + "title": "Nodestore Operations", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "monitoring" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Sentry Self-Hosted Monitoring", + "uid": "sentry-self-hosted-monitoring", + "version": 1, + "weekStart": "" +} From 4ab5fbb3339ac13af377873fb5907d4d2f3b2cd3 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 5 Jul 2026 16:26:55 +0700 Subject: [PATCH 2/3] feat: restructure everything --- grafana-dashboards/README.md | 25 +- .../sentry-background-workflows.json | 2128 +++++++++++++ .../sentry-errors-and-failures.json | 945 ------ .../sentry-event-processing.json | 1912 ++++++++++++ .../sentry-foundational-storage.json | 1592 ++++++++++ grafana-dashboards/sentry-ingestion.json | 2392 ++++++++++----- grafana-dashboards/sentry-overview.json | 2664 +++++++++++------ grafana-dashboards/sentry-tasks-manager.json | 2054 +++++++++++++ ...und-processing.json => sentry-ui-api.json} | 994 +++--- 9 files changed, 11516 insertions(+), 3190 deletions(-) create mode 100644 grafana-dashboards/sentry-background-workflows.json delete mode 100644 grafana-dashboards/sentry-errors-and-failures.json create mode 100644 grafana-dashboards/sentry-event-processing.json create mode 100644 grafana-dashboards/sentry-foundational-storage.json create mode 100644 grafana-dashboards/sentry-tasks-manager.json rename grafana-dashboards/{sentry-background-processing.json => sentry-ui-api.json} (67%) diff --git a/grafana-dashboards/README.md b/grafana-dashboards/README.md index e9738e5a696..61059465e4d 100644 --- a/grafana-dashboards/README.md +++ b/grafana-dashboards/README.md @@ -1,15 +1,20 @@ # Sentry Self-Hosted Grafana Dashboards -A starting set of Grafana dashboards for monitoring Sentry self-hosted installations via the StatsD metrics emitted by Sentry services (compatible with Prometheus, VictoriaMetrics, or Graphite with query rewrites). +A set of Grafana dashboards for monitoring Sentry self-hosted installations via the StatsD metrics emitted by Sentry services (compatible with Prometheus, VictoriaMetrics, or Graphite with query rewrites). + +Each dashboard is organized around an observability question — **what is this subsystem doing, and is it healthy?** — rather than around raw metrics. ## Dashboards -| File | Purpose | -|---|---| -| `sentry-overview.json` | High-level system health: API volume, Snuba queries, emails, webhooks, workflow engine, nodestore, and ClickHouse connections. | -| `sentry-ingestion.json` | Event ingestion flow: accepted/rejected events, Relay health, event pipeline latency, Symbolicator, project config, and Relay/Redis. | -| `sentry-background-processing.json` | Workers, task brokers, Arroyo consumers/producers: task throughput, execution duration, RPC latency, Kafka queue health, and backpressure. | -| `sentry-errors-and-failures.json` | Failure signals: rejected events, dropped messages, task RPC errors, and Snuba query health. | +| File | Title | Purpose | +|---|---|---| +| `sentry-overview.json` | **Self-Hosted Sentry: Overview** | The only dashboard you *need* to install. One summary row per subsystem so you can tell at a glance where to drill down. | +| `sentry-ui-api.json` | **Self-Hosted Sentry: UI & API** | Sentry API traffic, response codes, login attempts, rate limits, Snuba API queries, and backing-store health (Postgres, Memcached, Redis, ClickHouse). | +| `sentry-ingestion.json` | **Self-Hosted Sentry: Ingestion** | Relay event intake, project-config fetching, Kafka producer health, ingest consumers, and spans buffer. | +| `sentry-event-processing.json` | **Self-Hosted Sentry: Event Processing** | Pipeline latency, event normalization, Symbolicator, Snuba consumer inserts, and Arroyo backpressure. | +| `sentry-background-workflows.json` | **Self-Hosted Sentry: Background Workflows** | Workflow engine, alerts, email, weekly reports, outbound integrations/webhooks, and monitors. | +| `sentry-tasks-manager.json` | **Self-Hosted Sentry: Tasks Manager** | Taskbroker queue state and taskworker execution, RPC, and scheduling health. | +| `sentry-foundational-storage.json` | **Self-Hosted Sentry: Foundational Storage** | Nodestore, filestore, caches (Memcached, Redis), and Postgres query counters. | ## Metric name version @@ -18,6 +23,9 @@ These dashboards are written against the dot-separated StatsD metric names emitt - `stats.counters.sentry.relay.event.accepted.count` - `stats.timers.sentry.events.since_received.median` - `stats.counters.sentry.taskworker.worker.execute_task.count` +- `stats.counters.sentry.login.attempt.count` +- `stats.timers.sentry.relay.config.get_project_config.duration.median` +- `stats.counters.sentry.weekly_report.send_email.success.count` If you are using a statsd-to-prometheus exporter that converts dots to underscores, update the queries accordingly. @@ -28,7 +36,7 @@ The `stats.counters.*`, `stats.timers.*`, and `stats.gauges.*` prefixes come fro ## Importing 1. Open Grafana → **Dashboards** → **Import**. -2. Upload one of the JSON files from this directory. +2. Upload `sentry-overview.json` (and any subsystem dashboards you want). 3. Select your VictoriaMetrics/Prometheus data source. 4. The `host` variable will populate automatically from the `host` label. @@ -36,3 +44,4 @@ The `stats.counters.*`, `stats.timers.*`, and `stats.gauges.*` prefixes come fro - Dashboards default to the last hour with a 30-second refresh. - Several panels group by labels documented in Sentry's monitoring guide (e.g. `url_name`, `method`, `taskname`, `stage`). If your installation does not yet emit those tags, the panels will aggregate to a single series until the tags are present. +- Some metrics are feature-specific (e.g. Postgres queries only appear where Sentry emits them). Missing series simply produce empty panels until the corresponding feature is exercised. diff --git a/grafana-dashboards/sentry-background-workflows.json b/grafana-dashboards/sentry-background-workflows.json new file mode 100644 index 00000000000..043748f6b6a --- /dev/null +++ b/grafana-dashboards/sentry-background-workflows.json @@ -0,0 +1,2128 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Workflow engine, alerts, email, monitors, outbound integrations, and weekly reports.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1000, + "panels": [], + "title": "Workflow Engine", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Workflows Processed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.triggered_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Triggered Workflows / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.workflows_evaluated.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Workflows Evaluated / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Activities Completed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "process workflows" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.triggered_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "triggered workflows" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.workflows_evaluated.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "workflows evaluated" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "activities completed" + } + ], + "title": "Workflow Engine Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.workflow_engine.data_condition.evaluation_duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.workflow_engine.data_condition.evaluation_duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.workflow_engine.invoke_workflow_activity_handlers.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "invoke handlers p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.workflow_engine.invoke_workflow_activity_handlers.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "invoke handlers p90" + } + ], + "title": "Condition Evaluation Duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.hit.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "hit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.miss.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "miss" + } + ], + "title": "Workflow Cache Hit/Miss", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 7 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 1001, + "panels": [], + "title": "Email", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Emails Sent / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Emails Queued / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 22 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "emails / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "sent" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "queued" + } + ], + "title": "Email Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 10 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 1002, + "panels": [], + "title": "Outbound Integrations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "webhooks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "send started" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "send success" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "prepare started" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "prepare success" + } + ], + "title": "Sentry App Webhooks", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 11 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "webhooks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.http_request.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "http request" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.http_response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "http response" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.slo.started.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slo started" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.slo.failure.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slo failure" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.integrations.slo.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "slo success" + } + ], + "title": "Integration Webhooks & SLO", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 12 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 1003, + "panels": [], + "title": "Weekly Reports", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Weekly reports aggregate data from Postgres/ClickHouse and email multiple users once per week. Monitor scheduling, preparation, and delivery success.", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [], + "title": "", + "type": "text", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.organization.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Organizations Scheduled / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 46 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.prepare_organization_report.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Reports Prepared / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 46 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.send_email.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Emails Sent / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 46 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Weekly Report Emails / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 46 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "reports / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.organization.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "scheduled" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.prepare_organization_report.started.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "prepare started" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.prepare_organization_report.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "prepare success" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.send_email.started.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "send started" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.weekly_report.send_email.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "send success" + } + ], + "title": "Weekly Report Pipeline", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.weekly_report.create_context.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.weekly_report.create_context.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.weekly_report.deliver_reports.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "deliver p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.weekly_report.deliver_reports.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "deliver p90" + } + ], + "title": "Weekly Report Context Duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 19 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 1004, + "panels": [], + "title": "Monitors / Crons", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.checkin.result.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Check-in Results / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 59 + }, + "id": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.checkin.handle_processing_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Check-in Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 59 + }, + "id": 21 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.checkin.result.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "check-in results" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.checkin.handle_processing_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "processing errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.monitors.tasks.clock_tick.tick_decision.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "clock tick decisions" + } + ], + "title": "Monitor Health", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 22 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "delay", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.monitors.task.clock_delay{host=~\"$host\"})", + "refId": "A", + "legendFormat": "clock delay" + } + ], + "title": "Monitor Clock Delay", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 23 + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "workflows" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Self-Hosted Sentry: Background Workflows", + "uid": "sentry-self-hosted-background-workflows", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-errors-and-failures.json b/grafana-dashboards/sentry-errors-and-failures.json deleted file mode 100644 index f62c47b1e1b..00000000000 --- a/grafana-dashboards/sentry-errors-and-failures.json +++ /dev/null @@ -1,945 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Failure signals across Relay, task brokers, consumers, and Snuba for Sentry self-hosted.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [ - { - "title": "Sentry Monitoring Docs", - "type": "link", - "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 101, - "panels": [], - "title": "Ingestion Failures", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - }, - "unit": "events/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Rejected / sec", - "refId": "A" - } - ], - "title": "Rejected Events / sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - }, - "unit": "events/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Errors / sec", - "refId": "A" - } - ], - "title": "Error Events Processed / sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.taskworker.client.rpc_error.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "RPC errors / sec", - "refId": "A" - } - ], - "title": "Taskworker RPC Errors / sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "events/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "{{item_type}}", - "refId": "A" - } - ], - "title": "Rejected Events by Item Type", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "responses / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(status_code) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A" - } - ], - "title": "Relay Response Status Codes", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 106, - "panels": [], - "title": "Consumer & Processing Failures", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.strategies.filter.dropped_messages.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "dropped messages", - "refId": "A" - } - ], - "title": "Dropped Messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "errors / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.monitors.checkin.handle_processing_error.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "errors", - "refId": "A" - } - ], - "title": "Monitor Checkin Processing Errors", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 108, - "panels": [], - "title": "Snuba Query Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "queries / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "cache hit", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.snuba.db_query.cache_miss.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "cache miss", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.snuba.db_query.successful_query.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "successful", - "refId": "C" - } - ], - "title": "Snuba DB Query Hit/Miss", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.snuba.allocation_policy.bytes_scanned.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "bytes scanned", - "refId": "A" - } - ], - "title": "Snuba Allocation Policy Bytes Scanned", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "sentry", - "self-hosted", - "errors", - "failures" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "VictoriaMetrics", - "value": "VictoriaMetrics" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(host)", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "host", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(host)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Sentry Self-Hosted: Errors & Failures", - "uid": "sentry-self-hosted-errors-failures", - "version": 1, - "weekStart": "" -} diff --git a/grafana-dashboards/sentry-event-processing.json b/grafana-dashboards/sentry-event-processing.json new file mode 100644 index 00000000000..dfa3b8f3ede --- /dev/null +++ b/grafana-dashboards/sentry-event-processing.json @@ -0,0 +1,1912 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Event processing after ingestion: pipeline latency, normalization, Symbolicator, Snuba consumers/inserts, and Arroyo backpressure.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1000, + "panels": [], + "title": "Pipeline Latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50 {{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90 {{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max {{stage}}" + } + ], + "title": "Event Pipeline Latency by Stage", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 1001, + "panels": [], + "title": "Event Normalization", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.median{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Normalize Duration p50", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper_90{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Normalize Duration p90", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Normalize Duration Max", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" + } + ], + "title": "Event Normalization Duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 5 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 1002, + "panels": [], + "title": "Symbolicator", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.to-symbolicate.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Events to Symbolicate / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Symbolicator Responses / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.request_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Symbolicator Request Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.status_code.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Symbolicator Status Codes / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.to-symbolicate.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "to-symbolicate" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "responses" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.request_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "request errors" + } + ], + "title": "Symbolicator Demand", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.create_task.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "create_task p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.create_task.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "create_task p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.query_task.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "query_task p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.query_task.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "query_task p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.session.request.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "session.request p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.events.symbolicator.session.request.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "session.request p90" + } + ], + "title": "Symbolicator Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 11 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 1003, + "panels": [], + "title": "Snuba Consumers & Inserts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.insertions.batch_write_msgs.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Batch Write Msgs / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.insertions.batch_write_bytes.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Batch Write Bytes / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.rust_consumer.clickhouse_insert_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "CH Insert Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.snuba.clickhouse.native.connections{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "ClickHouse Connections", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "writes / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.insertions.batch_write_msgs.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "msgs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.insertions.item_type_count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "item types" + } + ], + "title": "Snuba Insert Volume", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.batch_write_ms.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "batch_write p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.batch_write_ms.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "batch_write p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.latency_ms.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "insert latency p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.latency_ms.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "insert latency p90" + } + ], + "title": "Snuba Insert Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.end_to_end_latency_ms.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.end_to_end_latency_ms.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.insertions.max_end_to_end_latency_ms.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max p50" + } + ], + "title": "End-to-End Insert Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 18 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 1004, + "panels": [], + "title": "Arroyo Consumer Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "loops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "snuba run loops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.replacer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "replacer run loops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "sentry run loops" + } + ], + "title": "Consumer Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 19 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.strategies.run_task_in_threads.giving_backpressure.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "giving backpressure" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.strategies.run_task_in_threads.got_backpressure.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "got backpressure" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "backpressure p90" + } + ], + "title": "Arroyo Backpressure & Strategies", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 20 + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "processing" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Self-Hosted Sentry: Event Processing", + "uid": "sentry-self-hosted-event-processing", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-foundational-storage.json b/grafana-dashboards/sentry-foundational-storage.json new file mode 100644 index 00000000000..208f3e560d7 --- /dev/null +++ b/grafana-dashboards/sentry-foundational-storage.json @@ -0,0 +1,1592 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Nodestore, filestore, caches (memcached, Redis), and Postgres query counters.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1000, + "panels": [], + "title": "Nodestore", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Nodestore Get / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Nodestore Get Multi / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.median{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Nodestore p50", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.upper_90{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Nodestore p90", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "get" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "get_multi" + } + ], + "title": "Nodestore Operations", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" + } + ], + "title": "Nodestore Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 6 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.set_bytes.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "set_bytes p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.nodestore.set_bytes.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "set_bytes p90" + } + ], + "title": "Nodestore Payload Size", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 7 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 1001, + "panels": [], + "title": "Filestore", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.read.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Filestore Read / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 22 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.save.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Filestore Save / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 22 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.read.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.save.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "save" + } + ], + "title": "Filestore Operations", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.filestore.read.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "read p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.filestore.read.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "read p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.filestore.save.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "save p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.filestore.save.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "save p90" + } + ], + "title": "Filestore Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 11 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 1002, + "panels": [], + "title": "Caches", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Memcache Reconnects / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 35 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Metrics Indexer Memcache / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 35 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "reconnects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "metrics indexer memcache" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.fetch.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "metrics indexer fetch" + } + ], + "title": "Memcached Activity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "connections" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "idle" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", + "refId": "A", + "legendFormat": "waiting" + } + ], + "title": "Relay Redis Pool", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 15 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 1003, + "panels": [], + "title": "Postgres", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Sentry does not emit generic Postgres connection metrics via StatsD. These counters show feature-specific query load.", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [], + "title": "", + "type": "text", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "queries / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.postgres.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "metrics indexer postgres" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.dynamic_sampling.boost_low_volume_transactions.query.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "dynamic sampling queries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.release_monitor.fetch_project_release_health_totals.query.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "release monitor queries" + } + ], + "title": "Postgres Query Counters", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 17 + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "storage" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Self-Hosted Sentry: Foundational Storage", + "uid": "sentry-self-hosted-foundational-storage", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-ingestion.json b/grafana-dashboards/sentry-ingestion.json index 1d66b1d9eeb..a3d2b9c2e99 100644 --- a/grafana-dashboards/sentry-ingestion.json +++ b/grafana-dashboards/sentry-ingestion.json @@ -15,7 +15,7 @@ } ] }, - "description": "Event ingestion flow for Sentry self-hosted. Tracks accepted/rejected events, Relay health, and pipeline latency.", + "description": "Event ingestion flow: Relay health, project config fetching, Kafka producers, ingest consumers, and spans buffer.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -36,7 +36,7 @@ "x": 0, "y": 0 }, - "id": 101, + "id": 1000, "panels": [], "title": "Ingestion Volume", "type": "row" @@ -65,13 +65,6 @@ }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, "options": { "colorMode": "value", "graphMode": "area", @@ -97,13 +90,18 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Events accepted / sec", "refId": "A" } ], "title": "Events Accepted / sec", - "type": "stat" + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1 }, { "datasource": { @@ -129,13 +127,6 @@ }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 5 - }, - "id": 2, "options": { "colorMode": "value", "graphMode": "area", @@ -161,13 +152,18 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Events rejected / sec", "refId": "A" } ], "title": "Events Rejected / sec", - "type": "stat" + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2 }, { "datasource": { @@ -184,22 +180,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "orange", "value": null } ] }, - "unit": "reqps" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 9 - }, - "id": 3, "options": { "colorMode": "value", "graphMode": "area", @@ -224,14 +213,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.server.http.accepted.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "HTTP accepted / sec", + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Relay HTTP Accepted / sec", - "type": "stat" + "title": "Error Events / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3 }, { "datasource": { @@ -253,17 +247,10 @@ } ] }, - "unit": "reqps" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 13 - }, - "id": 4, "options": { "colorMode": "value", "graphMode": "area", @@ -288,14 +275,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.requests.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Requests / sec", + "expr": "sum(rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Relay Requests / sec", - "type": "stat" + "title": "Event Items / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4 }, { "datasource": { @@ -334,14 +326,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -351,17 +342,11 @@ } ] }, - "unit": "events/s" + "unit": "events/s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 5, "options": { "legend": { "calcs": [ @@ -379,21 +364,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "{{item_type}}", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "accepted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "rejected" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "error processed" } ], - "title": "Event Items by Type", - "type": "timeseries" + "title": "Event Intake", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5 }, { "datasource": { @@ -406,69 +415,37 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" } }, "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "events/s" + "unit": "short" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 6, "options": { + "displayLabels": [ + "name" + ], "legend": { - "calcs": [ - "mean", - "max" - ], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, "tooltip": { "hideZeros": false, @@ -477,32 +454,27 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "accepted", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "rejected", - "refId": "B" + "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{item_type}}" } ], - "title": "Accepted vs Rejected Events", - "type": "timeseries" + "title": "Event Items by Type", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 5 + }, + "id": 6 }, { "collapsed": false, @@ -510,9 +482,9 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 13 }, - "id": 107, + "id": 1001, "panels": [], "title": "Relay Health", "type": "row" @@ -525,43 +497,9 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "responses / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -575,45 +513,43 @@ }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 7, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(status_code) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "{{status_code}}", + "expr": "sum(rate(stats.counters.sentry.relay.server.http.accepted.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Relay Responses by Status Code", - "type": "timeseries" + "title": "Relay HTTP Accepted / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 7 }, { "datasource": { @@ -623,108 +559,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "mps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 42 - }, - "id": 8, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.producer.produce_status.success.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "success", + "expr": "sum(rate(stats.counters.sentry.relay.requests.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Relay Producer Success Rate", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Relay Requests / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 + "h": 4, + "w": 6, + "x": 6, + "y": 14 }, - "id": 109, - "panels": [], - "title": "Pipeline Latency", - "type": "row" + "id": 8 }, { "datasource": { @@ -734,130 +621,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "yellow", "value": null } ] }, - "unit": "s" + "unit": "ops" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 51 - }, - "id": 9, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.events.since_received.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", + "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.fetched.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "C" } ], - "title": "Event Pipeline Latency", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Project Configs Fetched / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 59 + "h": 4, + "w": 6, + "x": 12, + "y": 14 }, - "id": 110, - "panels": [], - "title": "Symbolicator", - "type": "row" + "id": 9 }, { "datasource": { @@ -867,95 +683,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "tasks / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "orange", "value": null } ] }, - "unit": "tps" + "unit": "short" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 60 - }, - "id": 10, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.timers.sentry.events.symbolicator.create_task.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "tasks created", + "expr": "sum(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"})", "refId": "A" } ], - "title": "Symbolicator Tasks Created", - "type": "timeseries" + "title": "Project Configs Pending", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 10 }, { "datasource": { @@ -971,7 +751,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "requests / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1001,7 +781,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1011,17 +790,11 @@ } ] }, - "unit": "bytes" + "unit": "reqps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "id": 11, "options": { "legend": { "calcs": [ @@ -1039,45 +812,36 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.events.symbolicator.response.completed.size.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.relay.requests.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "requests" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.events.symbolicator.response.completed.size.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "responses" } ], - "title": "Symbolicator Response Size", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Relay Requests & Responses", + "type": "timeseries", "gridPos": { - "h": 1, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 76 + "y": 18 }, - "id": 111, - "panels": [], - "title": "Project Config", - "type": "row" + "id": 11 }, { "datasource": { @@ -1093,7 +857,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "duration", + "axisLabel": "responses / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1116,14 +880,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1133,17 +896,11 @@ } ] }, - "unit": "s" + "unit": "reqps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 77 - }, - "id": 112, "options": { "legend": { "calcs": [ @@ -1161,43 +918,1140 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", - "refId": "A" - }, + "expr": "sum by(status_code) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{status_code}}" + } + ], + "title": "Relay Response Status Codes", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 12 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 1002, + "panels": [], + "title": "Project Config Flow", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Relay fetches project configs from Sentry, which are fed onto Redis by the taskworker. Watch for pending configs growing or fetch latency spiking.", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [], + "title": "", + "type": "text", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.api.endpoints.relay.project_configs.post.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Sentry Project Config Endpoint / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.fetched.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Relay Configs Fetched / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Relay Configs Pending", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 29 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.config.get_quotas.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Get Quotas / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 29 + }, + "id": 17 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.relay.config.get_project_config.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.relay.config.get_project_config.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.relay.config.get_project_config.duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" + } + ], + "title": "Project Config Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.fetched.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "fetched / sec" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"})", + "refId": "A", + "legendFormat": "pending" + } + ], + "title": "Project Config Fetch vs Pending", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 19 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 1003, + "panels": [], + "title": "Kafka Producer Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.producer.produce_status.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "success" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.producer.produce_status.error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "error" + } + ], + "title": "Relay Produce Status", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "latency", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_int_latency{host=~\"$host\"})", + "refId": "A", + "legendFormat": "internal latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", + "refId": "A", + "legendFormat": "outbuf latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_rtt{host=~\"$host\"})", + "refId": "A", + "legendFormat": "rtt" + } + ], + "title": "Kafka Producer Latency (p99)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 21 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 1004, + "panels": [], + "title": "Ingest Consumers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Consumer Run Loops / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 22 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.strategies.filter.dropped_messages.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Dropped Messages / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 23 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.invalid_message.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Invalid Messages / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 51 + }, + "id": 24 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.consumer.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Kafka Queue Size", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 51 + }, + "id": 25 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "run loops" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.relay.config.get_project_config.duration.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "C" + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.processing.strategies.healthcheck.touch.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "healthcheck touches" } ], - "title": "Project Config Fetch Duration", - "type": "timeseries" + "title": "Consumer Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 26 }, { "datasource": { @@ -1213,7 +2067,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1243,70 +2097,222 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "processing p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "processing p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "backpressure p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "backpressure p90" + } + ], + "title": "Consumer Processing & Backpressure", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 27 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 1005, + "panels": [], + "title": "Spans Buffer", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.spans.buffer.process_spans.count_spans.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Spans Processed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 64 + }, + "id": 28 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", "value": null } ] }, - "unit": "bytes" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 85 - }, - "id": 113, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.relay.projectconfig_cache.size.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", + "expr": "sum(rate(stats.counters.sentry.spans.buffer.empty_segments.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.relay.projectconfig_cache.size.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" } ], - "title": "Project Config Cache Size", - "type": "timeseries" + "title": "Empty Segments / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 64 + }, + "id": 29 }, { "datasource": { @@ -1316,48 +2322,14 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null } ] @@ -1366,67 +2338,43 @@ }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 93 - }, - "id": 114, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.write.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "writes", + "expr": "sum(rate(stats.counters.sentry.spans.buffer.flusher_unhealthy.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.scheduled.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "scheduled", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.projectconfig_cache.invalidation.recompute.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "invalidations", - "refId": "C" } ], - "title": "Project Config Cache Activity", - "type": "timeseries" + "title": "Flusher Unhealthy / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 64 + }, + "id": 30 }, { "datasource": { @@ -1436,48 +2384,14 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "configs / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] @@ -1486,69 +2400,43 @@ }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 101 - }, - "id": 115, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.fetched.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "fetched", + "expr": "sum(rate(stats.counters.sentry.spans.consumers.process_segments.outcome_emitted.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "pending", - "refId": "B" } ], - "title": "Project Configs Fetched / Pending", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Process Segments Outcomes / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 109 + "h": 4, + "w": 6, + "x": 18, + "y": 64 }, - "id": 116, - "panels": [], - "title": "Relay & Redis", - "type": "row" + "id": 31 }, { "datasource": { @@ -1564,7 +2452,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "events / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1594,7 +2482,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1604,17 +2491,11 @@ } ] }, - "unit": "short" + "unit": "events/s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 110 - }, - "id": 117, "options": { "legend": { "calcs": [ @@ -1632,54 +2513,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.relay.redis.pool.connections{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "active", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "idle", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.spans.buffer.process_spans.count_spans.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "spans processed" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "C" + "expr": "sum(rate(stats.counters.sentry.spans.buffer.empty_segments.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "empty segments" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "waiting", - "refId": "D" + "expr": "sum(rate(stats.counters.sentry.spans.buffer.flusher_unhealthy.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "flusher unhealthy" } ], - "title": "Redis Connections", - "type": "timeseries" + "title": "Spans Buffer Activity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 32 }, { "datasource": { @@ -1695,7 +2567,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "ops / sec", + "axisLabel": "duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1725,7 +2597,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1735,17 +2606,11 @@ } ] }, - "unit": "ops" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 118 - }, - "id": 118, "options": { "legend": { "calcs": [ @@ -1763,43 +2628,54 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.redis.connection.create.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "created", - "refId": "A" + "expr": "sum (stats.timers.sentry.spans.buffer.process_spans.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_spans p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.spans.buffer.process_spans.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_spans p90" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.redis.connection.recycle.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "recycled", - "refId": "B" + "expr": "sum (stats.timers.sentry.spans.buffer.process_batch.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_batch p50" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.project_state.redis.requests.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "project state requests", - "refId": "C" + "expr": "sum (stats.timers.sentry.spans.buffer.process_batch.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_batch p90" } ], - "title": "Redis Connection Operations", - "type": "timeseries" + "title": "Spans Buffer Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 33 } ], "refresh": "30s", @@ -1869,7 +2745,7 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Sentry Self-Hosted: Ingestion", + "title": "Self-Hosted Sentry: Ingestion", "uid": "sentry-self-hosted-ingestion", "version": 1, "weekStart": "" diff --git a/grafana-dashboards/sentry-overview.json b/grafana-dashboards/sentry-overview.json index e50fa60fd41..e784e445eb9 100644 --- a/grafana-dashboards/sentry-overview.json +++ b/grafana-dashboards/sentry-overview.json @@ -15,7 +15,7 @@ } ] }, - "description": "Sentry self-hosted monitoring dashboard for VictoriaMetrics. Visualizes notable metric keys from https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/", + "description": "Single-pane health summary for Sentry self-hosted. Pulls one signal from each subsystem so you know where to drill down.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -24,21 +24,51 @@ { "title": "Sentry Monitoring Docs", "type": "link", - "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/#notable-metric-keys" + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" } ], "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard answers: **Is my Sentry healthy?** Each row summarizes one subsystem. If a number looks wrong, open the matching subsystem dashboard.", + "mode": "markdown" + }, + "pluginVersion": "11.0.0", + "targets": [], + "title": "", + "type": "text", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1 + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 2 }, - "id": 100, + "id": 1000, "panels": [], - "title": "API Request Volume", + "title": "Ingestion: Is Sentry receiving events?", "type": "row" }, { @@ -49,43 +79,9 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "requests / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -95,33 +91,26 @@ } ] }, - "unit": "reqps" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 1, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ @@ -130,14 +119,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(url_name, method) (rate(stats.counters.sentry.view.response.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{method}} {{url_name}}", + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "API Request Rate by Endpoint", - "type": "timeseries" + "title": "Events Accepted / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 2 }, { "datasource": { @@ -147,130 +141,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null } ] }, - "unit": "s" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 2, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50 {{method}} {{url_name}}", + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90 {{method}} {{url_name}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(url_name, method) (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max {{method}} {{url_name}}", - "refId": "C" } ], - "title": "API Request Duration by Endpoint", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Events Rejected / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 + "h": 4, + "w": 6, + "x": 6, + "y": 3 }, - "id": 101, - "panels": [], - "title": "Task Throughput", - "type": "row" + "id": 3 }, { "datasource": { @@ -280,94 +203,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "tasks / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "orange", "value": null } ] }, - "unit": "tps" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 3, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(taskname) (rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "scheduled: {{taskname}}", + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Tasks Scheduled Rate", - "type": "timeseries" + "title": "Error Events / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 4 }, { "datasource": { @@ -377,94 +265,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "tasks / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "tps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 4, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(taskname) (rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "executed: {{taskname}}", + "expr": "sum(rate(stats.counters.sentry.relay.server.http.accepted.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Tasks Executed Rate", - "type": "timeseries" + "title": "Relay HTTP Accepted / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 5 }, { "datasource": { @@ -480,7 +333,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "duration", + "axisLabel": "events / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -510,7 +363,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -520,17 +372,11 @@ } ] }, - "unit": "s" + "unit": "events/s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 5, "options": { "legend": { "calcs": [ @@ -548,43 +394,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50 {{taskname}}", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "accepted" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper.90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90 {{taskname}}", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "rejected" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max {{taskname}}", - "refId": "C" + "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "error processed" } ], - "title": "Task Execution Duration by Task", - "type": "timeseries" + "title": "Event Intake", + "type": "timeseries", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 6 }, { "collapsed": false, @@ -592,11 +440,11 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 13 }, - "id": 102, + "id": 1001, "panels": [], - "title": "Error Processing", + "title": "UI & API: Can users reach Sentry?", "type": "row" }, { @@ -614,22 +462,77 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "events/s" + "unit": "reqps" }, "overrides": [] }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.view.response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "API Requests / sec", + "type": "stat", "gridPos": { "h": 4, - "w": 8, + "w": 6, "x": 0, - "y": 43 + "y": 14 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] }, - "id": 6, "options": { "colorMode": "value", "graphMode": "area", @@ -654,14 +557,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.event.accepted.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Events accepted / sec", + "expr": "sum(rate(stats.counters.sentry.login.attempt.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Events Accepted Rate", - "type": "stat" + "title": "Login Attempts / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 8 }, { "datasource": { @@ -678,22 +586,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "purple", "value": null } ] }, - "unit": "events/s" + "unit": "short" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 47 - }, - "id": 7, "options": { "colorMode": "value", "graphMode": "area", @@ -718,14 +619,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Relay error events processed / sec", + "expr": "sum(stats.gauges.snuba.clickhouse.native.connections{host=~\"$host\"})", "refId": "A" } ], - "title": "Relay Error Events Processed Rate", - "type": "stat" + "title": "ClickHouse Connections", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 9 }, { "datasource": { @@ -735,41 +641,28 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "mode": "thresholds" }, "mappings": [], - "unit": "events/s" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 51 - }, - "id": 8, "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "donut", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -777,27 +670,30 @@ "fields": "", "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{item_type}}", + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Event Items by Type", - "type": "piechart" + "title": "Tasks Executed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 10 }, { "datasource": { @@ -813,7 +709,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "events / sec", + "axisLabel": "duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -836,14 +732,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -853,17 +748,11 @@ } ] }, - "unit": "events/s" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 16, - "x": 0, - "y": 59 - }, - "id": 9, "options": { "legend": { "calcs": [ @@ -881,20 +770,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(item_type) (rate(stats.counters.sentry.relay.event.items.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{item_type}}", - "refId": "A" + "expr": "sum (stats.timers.sentry.view.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" } ], - "title": "Relay Event Items Rate by Type", - "type": "timeseries" + "title": "API Request Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 11 }, { "datasource": { @@ -910,7 +824,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "duration", + "axisLabel": "queries / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -940,7 +854,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -950,17 +863,11 @@ } ] }, - "unit": "s" + "unit": "ops", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 63 - }, - "id": 10, "options": { "legend": { "calcs": [ @@ -978,43 +885,27 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by(stage) (stats.timers.sentry.events.since_received.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50 {{stage}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper.90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90 {{stage}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max {{stage}}", - "refId": "C" + "expr": "sum(rate(stats.counters.snuba.api.execute.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "queries / sec" } ], - "title": "Event Pipeline Latency by Stage", - "type": "timeseries" + "title": "Snuba API Queries", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 12 }, { "collapsed": false, @@ -1022,11 +913,11 @@ "h": 1, "w": 24, "x": 0, - "y": 71 + "y": 26 }, - "id": 103, + "id": 1002, "panels": [], - "title": "Snuba Queries", + "title": "Event Processing: Is the pipeline flowing?", "type": "row" }, { @@ -1049,17 +940,10 @@ } ] }, - "unit": "short" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 72 - }, - "id": 104, "options": { "colorMode": "value", "graphMode": "area", @@ -1084,14 +968,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.clickhouse.native.connections{host=~\"$host\"})", - "interval": "", - "legendFormat": "connections", + "expr": "sum(rate(stats.counters.sentry.events.symbolicator.response.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "ClickHouse Connections", - "type": "stat" + "title": "Symbolicator Responses / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 13 }, { "datasource": { @@ -1108,22 +997,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "yellow", "value": null } ] }, - "unit": "ops" + "unit": "events/s" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 76 - }, - "id": 105, "options": { "colorMode": "value", "graphMode": "area", @@ -1148,14 +1030,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "run loops / sec", + "expr": "sum(rate(stats.counters.sentry.events.to-symbolicate.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Subscription Executor Queries", - "type": "stat" + "title": "Events to Symbolicate / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 14 }, { "datasource": { @@ -1172,22 +1059,15 @@ "mode": "absolute", "steps": [ { - "color": "yellow", + "color": "green", "value": null } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 80 - }, - "id": 106, "options": { "colorMode": "value", "graphMode": "area", @@ -1212,14 +1092,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.schedule.size{host=~\"$host\"})", - "interval": "", - "legendFormat": "scheduled subscriptions", + "expr": "sum(rate(stats.counters.snuba.consumer.insertions.batch_write_msgs.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Subscription Schedule Size", - "type": "stat" + "title": "Snuba Inserts / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 15 }, { "datasource": { @@ -1236,22 +1121,15 @@ "mode": "absolute", "steps": [ { - "color": "orange", + "color": "red", "value": null } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 84 - }, - "id": 107, "options": { "colorMode": "value", "graphMode": "area", @@ -1276,14 +1154,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.calculated_max_concurrent_queries{host=~\"$host\"})", - "interval": "", - "legendFormat": "max concurrent", + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.invalid_message.count.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Max Concurrent Queries", - "type": "stat" + "title": "Consumer Invalid Messages / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 16 }, { "datasource": { @@ -1299,7 +1182,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "events / sec", + "axisLabel": "duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1329,7 +1212,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1339,17 +1221,11 @@ } ] }, - "unit": "ops" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 88 - }, - "id": 108, "options": { "legend": { "calcs": [ @@ -1367,32 +1243,58 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "consumer run loops", - "refId": "A" + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50 {{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90 {{stage}}" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.snuba.subscriptions.scheduler_executor.arroyo.processing.strategies.healthcheck.touch.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "healthcheck touches", - "refId": "B" + "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max {{stage}}" } ], - "title": "Snuba Subscription Throughput", - "type": "timeseries" + "title": "Event Pipeline Latency by Stage", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 17 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 1003, + "panels": [], + "title": "Tasks Manager: Are tasks keeping up?", + "type": "row" }, { "datasource": { @@ -1402,119 +1304,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "short" + "unit": "tps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 96 - }, - "id": 109, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.tick_buffer.queue_size{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "tick buffer queue", + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.gauges.snuba.subscriptions.scheduler_executor.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "kafka queue", - "refId": "B" } ], - "title": "Snuba Subscription Queue Size", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Tasks Scheduled / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, + "h": 4, + "w": 6, "x": 0, - "y": 104 + "y": 40 }, - "id": 110, - "panels": [], - "title": "Emails", - "type": "row" + "id": 18 }, { "datasource": { @@ -1536,17 +1378,10 @@ } ] }, - "unit": "mps" + "unit": "tps" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 105 - }, - "id": 111, "options": { "colorMode": "value", "graphMode": "area", @@ -1571,14 +1406,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "sent / sec", + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Emails Sent / sec", - "type": "stat" + "title": "Tasks Executed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 19 }, { "datasource": { @@ -1595,22 +1435,15 @@ "mode": "absolute", "steps": [ { - "color": "blue", + "color": "yellow", "value": null } ] }, - "unit": "mps" + "unit": "short" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 109 - }, - "id": 112, "options": { "colorMode": "value", "graphMode": "area", @@ -1635,14 +1468,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "queued / sec", + "expr": "sum(stats.gauges.taskbroker.upkeep.current_pending_tasks{host=~\"$host\"})", "refId": "A" } ], - "title": "Emails Queued / sec", - "type": "stat" + "title": "Broker Pending Tasks", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 40 + }, + "id": 20 }, { "datasource": { @@ -1652,119 +1490,59 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "emails / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null } ] }, - "unit": "mps" + "unit": "tps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 113 - }, - "id": 113, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "sent", + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.processing_deadline_exceeded.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "queued", - "refId": "B" } ], - "title": "Email Throughput", - "type": "timeseries" - }, - { - "collapsed": false, + "title": "Processing Deadline Exceeded / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 121 + "h": 4, + "w": 6, + "x": 18, + "y": 40 }, - "id": 114, - "panels": [], - "title": "Webhooks", - "type": "row" + "id": 21 }, { "datasource": { @@ -1780,7 +1558,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "webhooks / sec", + "axisLabel": "tasks / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1810,7 +1588,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1820,17 +1597,11 @@ } ] }, - "unit": "ops" + "unit": "tps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 122 - }, - "id": 115, "options": { "legend": { "calcs": [ @@ -1848,43 +1619,36 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "started", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "success", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "scheduled" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "prepare started", - "refId": "C" + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "executed" } ], - "title": "Sentry App Webhook Rate", - "type": "timeseries" + "title": "Task Scheduled vs Executed", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 22 }, { "datasource": { @@ -1900,7 +1664,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "webhooks / sec", + "axisLabel": "duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1930,7 +1694,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1940,17 +1703,11 @@ } ] }, - "unit": "ops" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 116, "options": { "legend": { "calcs": [ @@ -1968,32 +1725,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.integrations.webhook.started.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "started", - "refId": "A" + "expr": "sum (stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.integrations.webhook.success.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "success", - "refId": "B" + "expr": "sum (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" } ], - "title": "Integration Webhook Rate", - "type": "timeseries" + "title": "Task Execution Duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 23 }, { "collapsed": false, @@ -2001,11 +1771,11 @@ "h": 1, "w": 24, "x": 0, - "y": 138 + "y": 52 }, - "id": 117, + "id": 1004, "panels": [], - "title": "Workflow Engine", + "title": "Background Workflows: Are notifications and rules firing?", "type": "row" }, { @@ -2016,43 +1786,9 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2062,71 +1798,47 @@ } ] }, - "unit": "ops" + "unit": "mps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 139 - }, - "id": 118, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "process workflows", + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.workflow_engine.data_condition.evaluation.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "condition evaluations", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "activities completed", - "refId": "C" } ], - "title": "Workflow Engine Throughput", - "type": "timeseries" + "title": "Emails Sent / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 53 + }, + "id": 24 }, { "datasource": { @@ -2136,119 +1848,183 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "ops" + "unit": "mps" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 147 - }, - "id": 119, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.hit.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "hit", + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.miss.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "miss", - "refId": "B" } ], - "title": "Workflow Engine Cache Hit/Miss", - "type": "timeseries" + "title": "Emails Queued / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 53 + }, + "id": 25 }, { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Workflows Processed / sec", + "type": "stat", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 155 + "h": 4, + "w": 6, + "x": 12, + "y": 53 }, - "id": 120, - "panels": [], - "title": "Nodestore", - "type": "row" + "id": 26 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Activities Completed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 53 + }, + "id": 27 }, { "datasource": { @@ -2264,7 +2040,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "ops / sec", + "axisLabel": "emails / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -2294,7 +2070,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2304,17 +2079,117 @@ } ] }, - "unit": "ops" + "unit": "mps", + "min": 0 }, "overrides": [] }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.sent.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "sent" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "queued" + } + ], + "title": "Email Throughput", + "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 156 + "y": 57 + }, + "id": 28 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] }, - "id": 121, "options": { "legend": { "calcs": [ @@ -2332,32 +2207,921 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "get", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "process workflows" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "get_multi", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.workflow_engine.data_condition.evaluation.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "condition evaluations" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "activities completed" } ], - "title": "Nodestore Operations", - "type": "timeseries" + "title": "Workflow Engine Throughput", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 29 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 1005, + "panels": [], + "title": "Foundational Storage: Is storage healthy?", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Nodestore Get / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 66 + }, + "id": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.read.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Filestore Read / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 66 + }, + "id": 31 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.timers.sentry.filestore.save.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Filestore Save / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 66 + }, + "id": 32 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Memcache Reconnects / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 66 + }, + "id": 33 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "ops / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "get" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "get_multi" + } + ], + "title": "Nodestore Operations", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 34 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "connections" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "idle" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", + "refId": "A", + "legendFormat": "waiting" + } + ], + "title": "Redis Pool Health (Relay)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 35 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 1006, + "panels": [], + "title": "Failures: Where should I look first?", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "events/s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Rejected Events / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 79 + }, + "id": 36 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.client.rpc_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Task RPC Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 79 + }, + "id": 37 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.processing_deadline_exceeded.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Processing Deadline Exceeded / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 79 + }, + "id": 38 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.invalid_message.count.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Consumer Invalid Messages / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 79 + }, + "id": 39 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.snuba.consumer.rust_consumer.clickhouse_insert_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Snuba CH Insert Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 79 + }, + "id": 40 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Memcache Reconnects / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 79 + }, + "id": 41 } ], "refresh": "30s", @@ -2365,7 +3129,7 @@ "tags": [ "sentry", "self-hosted", - "monitoring" + "overview" ], "templating": { "list": [ @@ -2427,8 +3191,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Sentry Self-Hosted Monitoring", - "uid": "sentry-self-hosted-monitoring", + "title": "Self-Hosted Sentry: Overview", + "uid": "sentry-self-hosted-overview", "version": 1, "weekStart": "" } diff --git a/grafana-dashboards/sentry-tasks-manager.json b/grafana-dashboards/sentry-tasks-manager.json new file mode 100644 index 00000000000..4e8f5bde95e --- /dev/null +++ b/grafana-dashboards/sentry-tasks-manager.json @@ -0,0 +1,2054 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Taskbroker queue state and taskworker execution, RPC, and scheduling health.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "Sentry Monitoring Docs", + "type": "link", + "url": "https://develop.sentry.dev/self-hosted/production-enhancements/monitoring/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1000, + "panels": [], + "title": "Task Production & Execution", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Tasks Scheduled / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Tasks Sent / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Tasks Executed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.scheduler.delay_task.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Tasks Delayed / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "scheduled" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "send success" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "executed" + } + ], + "title": "Task Production vs Execution", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50 {{taskname}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90 {{taskname}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max {{taskname}}" + } + ], + "title": "Task Execution Duration by Task", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 6 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 1001, + "panels": [], + "title": "Stuck / Dropped Tasks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.fetch_task.not_found.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Fetch Task Not Found / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 7 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.processing_deadline_exceeded.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Processing Deadline Exceeded / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 8 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.add_tasks.child_tasks_full.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Child Tasks Full / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 9 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "tps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.max_task_count_reached.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Max Task Count Reached / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "tps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.fetch_task.not_found.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "fetch not found" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.processing_deadline_exceeded.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "deadline exceeded" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.child_task_queue_empty.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "child queue empty" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.max_task_count_reached.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "max task count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.worker.add_tasks.child_tasks_full.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "child tasks full" + } + ], + "title": "Task Problems", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 11 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 1002, + "panels": [], + "title": "Taskbroker Queue State", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_pending_tasks{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Pending Tasks", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 12 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_processing_tasks{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Processing Tasks", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_claimed_tasks{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Claimed Tasks", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 14 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_delayed_tasks{host=~\"$host\"})", + "refId": "A" + } + ], + "title": "Delayed Tasks", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 15 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "tasks", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_pending_tasks{host=~\"$host\"})", + "refId": "A", + "legendFormat": "pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_processing_tasks{host=~\"$host\"})", + "refId": "A", + "legendFormat": "processing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_claimed_tasks{host=~\"$host\"})", + "refId": "A", + "legendFormat": "claimed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.current_delayed_tasks{host=~\"$host\"})", + "refId": "A", + "legendFormat": "delayed" + } + ], + "title": "Taskbroker Queue Size", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 16 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "lag", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.pending_activation.max_lag.sec{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max lag" + } + ], + "title": "Taskbroker Pending Lag", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 17 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 1003, + "panels": [], + "title": "Taskbroker Storage Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "events", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_hit_total{host=~\"$host\"})", + "refId": "A", + "legendFormat": "cache hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_miss_total{host=~\"$host\"})", + "refId": "A", + "legendFormat": "cache misses" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_spill_total{host=~\"$host\"})", + "refId": "A", + "legendFormat": "cache spills" + } + ], + "title": "SQLite Cache Activity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 18 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.db_file_size.bytes{host=~\"$host\"})", + "refId": "A", + "legendFormat": "db file size" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.upkeep.wal_file_size.bytes{host=~\"$host\"})", + "refId": "A", + "legendFormat": "wal file size" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_used_bytes{host=~\"$host\"})", + "refId": "A", + "legendFormat": "cache used" + } + ], + "title": "Taskbroker Storage Size", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 19 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 1004, + "panels": [], + "title": "Worker \u2194 Broker RPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.taskworker.client.rpc_error.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" + } + ], + "title": "Taskworker RPC Errors / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.taskworker.get_task.rpc.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "get_task p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.taskworker.get_task.rpc.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "get_task p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.taskworker.update_task.rpc.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "update_task p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.taskworker.update_task.rpc.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "update_task p90" + } + ], + "title": "RPC Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 21 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.get_task.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "get_task p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.get_task.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "get_task p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.set_status.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "set_status p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.set_status.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "set_status p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.fetch_next.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "fetch_next p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.taskbroker.grpc_server.fetch_next.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "fetch_next p90" + } + ], + "title": "Broker gRPC Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 22 + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "sentry", + "self-hosted", + "tasks" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(host)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Self-Hosted Sentry: Tasks Manager", + "uid": "sentry-self-hosted-tasks-manager", + "version": 1, + "weekStart": "" +} diff --git a/grafana-dashboards/sentry-background-processing.json b/grafana-dashboards/sentry-ui-api.json similarity index 67% rename from grafana-dashboards/sentry-background-processing.json rename to grafana-dashboards/sentry-ui-api.json index 7ef7c679c6c..35e3a0fd4d2 100644 --- a/grafana-dashboards/sentry-background-processing.json +++ b/grafana-dashboards/sentry-ui-api.json @@ -15,7 +15,7 @@ } ] }, - "description": "Background workers, task brokers, and consumer health for Sentry self-hosted.", + "description": "Deep-dive on Sentry API traffic, response codes, rate limits, login attempts, Snuba API queries, and backing-store health.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -36,9 +36,9 @@ "x": 0, "y": 0 }, - "id": 101, + "id": 1000, "panels": [], - "title": "Task Throughput", + "title": "API Request Volume", "type": "row" }, { @@ -56,22 +56,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] }, - "unit": "tps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 1, "options": { "colorMode": "value", "graphMode": "area", @@ -96,14 +89,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "scheduled / sec", + "expr": "sum(rate(stats.counters.sentry.view.response.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Tasks Scheduled / sec", - "type": "stat" + "title": "API Requests / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1 }, { "datasource": { @@ -125,17 +123,10 @@ } ] }, - "unit": "tps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 5 - }, - "id": 2, "options": { "colorMode": "value", "graphMode": "area", @@ -160,14 +151,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "executed / sec", + "expr": "sum(rate(stats.counters.sentry.response.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Tasks Executed / sec", - "type": "stat" + "title": "Sentry Responses / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2 }, { "datasource": { @@ -184,22 +180,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "purple", "value": null } ] }, - "unit": "tps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 9 - }, - "id": 3, "options": { "colorMode": "value", "graphMode": "area", @@ -224,14 +213,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "success / sec", + "expr": "sum(rate(stats.counters.sentry.relay.requests.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Send Task Success / sec", - "type": "stat" + "title": "Relay Requests / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3 }, { "datasource": { @@ -253,17 +247,10 @@ } ] }, - "unit": "tps" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 13 - }, - "id": 4, "options": { "colorMode": "value", "graphMode": "area", @@ -288,14 +275,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.scheduler.delay_task.count{host=~\"$host\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "delayed / sec", + "expr": "sum(rate(stats.counters.sentry.login.attempt.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" } ], - "title": "Delayed Tasks / sec", - "type": "stat" + "title": "Login Attempts / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4 }, { "datasource": { @@ -311,7 +303,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "tasks / sec", + "axisLabel": "requests / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -334,14 +326,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -351,17 +342,11 @@ } ] }, - "unit": "tps" + "unit": "reqps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 5, "options": { "legend": { "calcs": [ @@ -379,43 +364,27 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.scheduled.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "scheduled", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "executed", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "success", - "refId": "C" + "expr": "sum by(url_name,method) (rate(stats.counters.sentry.view.response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{method}} {{url_name}}" } ], - "title": "Task Schedule vs Execute vs Success", - "type": "timeseries" + "title": "API Request Rate by Endpoint", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5 }, { "datasource": { @@ -461,7 +430,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -471,17 +439,11 @@ } ] }, - "unit": "s" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 6, "options": { "legend": { "calcs": [ @@ -499,43 +461,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", - "refId": "A" + "expr": "sum by(url_name,method) (stats.timers.sentry.view.duration.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p50 {{method}} {{url_name}}" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" + "expr": "sum by(url_name,method) (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "p90 {{method}} {{url_name}}" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "C" + "expr": "sum by(url_name,method) (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max {{method}} {{url_name}}" } ], - "title": "Task Execution Duration", - "type": "timeseries" + "title": "API Request Duration by Endpoint", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 6 }, { "collapsed": false, @@ -543,11 +507,11 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 13 }, - "id": 107, + "id": 1001, "panels": [], - "title": "Taskworker RPC & Queue Health", + "title": "HTTP Status Codes", "type": "row" }, { @@ -564,7 +528,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "duration", + "axisLabel": "responses / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -587,14 +551,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -604,17 +567,11 @@ } ] }, - "unit": "s" + "unit": "reqps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 7, "options": { "legend": { "calcs": [ @@ -632,54 +589,27 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.taskworker.get_task.rpc.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "get_task p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.taskworker.get_task.rpc.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "get_task p90", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.taskworker.update_task.rpc.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "update_task p50", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.taskworker.update_task.rpc.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "update_task p90", - "refId": "D" + "expr": "sum by(status_code) (rate(stats.counters.sentry.response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{status_code}}" } ], - "title": "Task RPC Latency", - "type": "timeseries" + "title": "Sentry Response Status Codes", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 7 }, { "datasource": { @@ -695,7 +625,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "events / sec", + "axisLabel": "responses / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -718,14 +648,13 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -735,17 +664,11 @@ } ] }, - "unit": "ops" + "unit": "reqps", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 42 - }, - "id": 8, "options": { "legend": { "calcs": [ @@ -763,43 +686,27 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.taskworker.worker.fetch_task.not_found.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "fetch not found", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.taskworker.worker.child_task_queue_empty.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "child queue empty", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(rate(stats.counters.sentry.taskworker.worker.add_tasks.child_tasks_full.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "child tasks full", - "refId": "C" + "expr": "sum by(status_code,method,route) (rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{status_code}} {{method}} {{route}}" } ], - "title": "Queue Saturation Signals", - "type": "timeseries" + "title": "Relay Response Status Codes", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8 }, { "collapsed": false, @@ -807,11 +714,11 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 22 }, - "id": 109, + "id": 1002, "panels": [], - "title": "Consumer Health", + "title": "Rate Limits & Auth", "type": "row" }, { @@ -828,7 +735,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "loops / sec", + "axisLabel": "invocations / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -858,7 +765,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -868,17 +774,11 @@ } ] }, - "unit": "ops" + "unit": "ops", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 9, "options": { "legend": { "calcs": [ @@ -896,21 +796,36 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "run count", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.middleware.ratelimit.process_view.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "process_view" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(stats.counters.sentry.middleware.ratelimit.process_response.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "process_response" } ], - "title": "Consumer Run Count", - "type": "timeseries" + "title": "Rate-Limit Middleware Activity", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 9 }, { "datasource": { @@ -956,7 +871,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -966,17 +880,11 @@ } ] }, - "unit": "s" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "id": 10, "options": { "legend": { "calcs": [ @@ -994,32 +902,54 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p50", - "refId": "A" + "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_view.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_view p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_view.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_view p90" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "p90", - "refId": "B" + "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_response.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_response p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_response.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_response p90" } ], - "title": "Consumer Processing Time", - "type": "timeseries" + "title": "Rate-Limit Middleware Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 10 }, { "collapsed": false, @@ -1027,11 +957,11 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 31 }, - "id": 110, + "id": 1003, "panels": [], - "title": "Arroyo Consumers & Producers", + "title": "Snuba API & Queries", "type": "row" }, { @@ -1042,48 +972,14 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": null } ] @@ -1092,56 +988,105 @@ }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 68 - }, - "id": 111, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "sentry consumer run", + "expr": "sum(rate(stats.counters.snuba.api.execute.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" + } + ], + "title": "Snuba API Executes / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 11 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.producer.produce_status.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "sentry producer status", - "refId": "B" + "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A" } ], - "title": "Consumer Throughput (Sentry)", - "type": "timeseries" + "title": "Snuba Query Cache Hits / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 32 + }, + "id": 12 }, { "datasource": { @@ -1151,48 +1096,14 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "events / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "yellow", "value": null } ] @@ -1201,56 +1112,105 @@ }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 112, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": [ - "mean", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "snuba consumer run", + "expr": "sum(rate(stats.counters.snuba.db_query.cache_miss.count{host=~\"$host\"}[$__rate_interval]))", "refId": "A" + } + ], + "title": "Snuba Query Cache Misses / sec", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 32 + }, + "id": 13 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.producer.produce_status.count{host=~\"$host\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "snuba producer status", - "refId": "B" + "expr": "sum(stats.gauges.snuba.clickhouse.native.connections{host=~\"$host\"})", + "refId": "A" } ], - "title": "Consumer Throughput (Snuba)", - "type": "timeseries" + "title": "ClickHouse Connections", + "type": "stat", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 14 }, { "datasource": { @@ -1266,7 +1226,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "duration", + "axisLabel": "queries / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1296,7 +1256,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1306,17 +1265,11 @@ } ] }, - "unit": "s" + "unit": "ops", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 84 - }, - "id": 113, "options": { "legend": { "calcs": [ @@ -1334,54 +1287,45 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry p90", - "refId": "B" + "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "hits" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.processing.time.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba p50", - "refId": "C" + "expr": "sum(rate(stats.counters.snuba.db_query.cache_miss.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "misses" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba p90", - "refId": "D" + "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit_simple.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "hits simple" } ], - "title": "Consumer Processing Time", - "type": "timeseries" + "title": "Snuba Query Cache", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 15 }, { "datasource": { @@ -1427,7 +1371,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1437,17 +1380,11 @@ } ] }, - "unit": "s" + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 92 - }, - "id": 114, "options": { "legend": { "calcs": [ @@ -1465,54 +1402,67 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry backpressure p50", - "refId": "A" + "expr": "sum (stats.timers.sentry.snuba.client.snql_query.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "snql_query p50" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry backpressure p90", - "refId": "B" + "expr": "sum (stats.timers.sentry.snuba.client.snql_query.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "snql_query p90" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba backpressure p50", - "refId": "C" + "expr": "sum (stats.timers.sentry.snuba.client.process_result.median{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_result p50" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba backpressure p90", - "refId": "D" + "expr": "sum (stats.timers.sentry.snuba.client.process_result.upper_90{host=~\"$host\"})", + "refId": "A", + "legendFormat": "process_result p90" } ], - "title": "Consumer Poll & Backpressure Time", - "type": "timeseries" + "title": "Sentry Snuba Client Latency", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 16 + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 1004, + "panels": [], + "title": "Backing Stores", + "type": "row" }, { "datasource": { @@ -1528,7 +1478,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "messages", + "axisLabel": "ops / sec", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1558,7 +1508,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1568,17 +1517,11 @@ } ] }, - "unit": "short" + "unit": "ops", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 100 - }, - "id": 115, "options": { "legend": { "calcs": [ @@ -1596,54 +1539,54 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.consumer.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry consumer queue", - "refId": "A" + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.postgres.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "metrics indexer postgres" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.consumer.arroyo.consumer.librdkafka.total_queue_size{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba consumer queue", - "refId": "B" + "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "metrics indexer memcache" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.message_count{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry producer msgs", - "refId": "C" + "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "memcache reconnect" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.message_count{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba producer msgs", - "refId": "D" + "expr": "sum(rate(stats.counters.sentry.dynamic_sampling.boost_low_volume_transactions.query.count{host=~\"$host\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "dynamic sampling queries" } ], - "title": "Kafka Queue Sizes", - "type": "timeseries" + "title": "Postgres & Memcache Query Counters", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 17 }, { "datasource": { @@ -1659,7 +1602,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "latency", + "axisLabel": "connections", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1689,7 +1632,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1699,17 +1641,11 @@ } ] }, - "unit": "\u00b5s" + "unit": "short", + "min": 0 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 108 - }, - "id": 116, "options": { "legend": { "calcs": [ @@ -1727,54 +1663,54 @@ "sort": "desc" } }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_int_latency{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry internal latency", - "refId": "A" + "expr": "sum(stats.gauges.sentry.relay.redis.pool.connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "connections" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "sentry outbuf latency", - "refId": "B" + "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "idle" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.p99_int_latency{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba internal latency", - "refId": "C" + "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", + "refId": "A", + "legendFormat": "max" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(stats.gauges.snuba.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", - "hide": false, - "interval": "", - "legendFormat": "snuba outbuf latency", - "refId": "D" + "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", + "refId": "A", + "legendFormat": "waiting" } ], - "title": "Producer Latency (p99)", - "type": "timeseries" + "title": "Relay Redis Pool", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 18 } ], "refresh": "30s", @@ -1782,7 +1718,7 @@ "tags": [ "sentry", "self-hosted", - "background-processing" + "api" ], "templating": { "list": [ @@ -1844,8 +1780,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Sentry Self-Hosted: Background Processing", - "uid": "sentry-self-hosted-background-processing", + "title": "Self-Hosted Sentry: UI & API", + "uid": "sentry-self-hosted-ui-api", "version": 1, "weekStart": "" } From 5eca13114aee94f19db48863f19f2a1bc67dbb13 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Mon, 6 Jul 2026 08:04:51 +0700 Subject: [PATCH 3/3] fix: handle "Multiple queries using the same RefId is not allowed" errors --- .../sentry-background-workflows.json | 48 +++++++++---------- .../sentry-event-processing.json | 42 ++++++++-------- .../sentry-foundational-storage.json | 30 ++++++------ grafana-dashboards/sentry-ingestion.json | 36 +++++++------- grafana-dashboards/sentry-overview.json | 32 ++++++------- grafana-dashboards/sentry-tasks-manager.json | 46 +++++++++--------- grafana-dashboards/sentry-ui-api.json | 34 ++++++------- 7 files changed, 134 insertions(+), 134 deletions(-) diff --git a/grafana-dashboards/sentry-background-workflows.json b/grafana-dashboards/sentry-background-workflows.json index 043748f6b6a..40a675c2f7a 100644 --- a/grafana-dashboards/sentry-background-workflows.json +++ b/grafana-dashboards/sentry-background-workflows.json @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.triggered_workflows.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "triggered workflows" }, { @@ -390,7 +390,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.process_workflows.workflows_evaluated.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "workflows evaluated" }, { @@ -399,7 +399,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "activities completed" } ], @@ -505,7 +505,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.workflow_engine.data_condition.evaluation_duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -514,7 +514,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.workflow_engine.invoke_workflow_activity_handlers.duration.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "invoke handlers p50" }, { @@ -523,7 +523,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.workflow_engine.invoke_workflow_activity_handlers.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "invoke handlers p90" } ], @@ -629,7 +629,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.workflow_engine.cache.processing_workflow.miss.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "miss" } ], @@ -872,7 +872,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "queued" } ], @@ -991,7 +991,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_app.send_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "send success" }, { @@ -1000,7 +1000,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.started.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "prepare started" }, { @@ -1009,7 +1009,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_app.prepare_webhook.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "prepare success" } ], @@ -1115,7 +1115,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.integrations.http_response.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "http response" }, { @@ -1124,7 +1124,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.integrations.slo.started.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "slo started" }, { @@ -1133,7 +1133,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.integrations.slo.failure.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "slo failure" }, { @@ -1142,7 +1142,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.integrations.slo.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "E", "legendFormat": "slo success" } ], @@ -1539,7 +1539,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.weekly_report.prepare_organization_report.started.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "prepare started" }, { @@ -1548,7 +1548,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.weekly_report.prepare_organization_report.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "prepare success" }, { @@ -1557,7 +1557,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.weekly_report.send_email.started.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "send started" }, { @@ -1566,7 +1566,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.weekly_report.send_email.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "E", "legendFormat": "send success" } ], @@ -1672,7 +1672,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.weekly_report.create_context.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -1681,7 +1681,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.weekly_report.deliver_reports.duration.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "deliver p50" }, { @@ -1690,7 +1690,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.weekly_report.deliver_reports.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "deliver p90" } ], @@ -1933,7 +1933,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.monitors.checkin.handle_processing_error.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "processing errors" }, { @@ -1942,7 +1942,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.monitors.tasks.clock_tick.tick_decision.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "clock tick decisions" } ], diff --git a/grafana-dashboards/sentry-event-processing.json b/grafana-dashboards/sentry-event-processing.json index dfa3b8f3ede..4d51211c3f2 100644 --- a/grafana-dashboards/sentry-event-processing.json +++ b/grafana-dashboards/sentry-event-processing.json @@ -133,7 +133,7 @@ "uid": "${datasource}" }, "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90 {{stage}}" }, { @@ -142,7 +142,7 @@ "uid": "${datasource}" }, "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max {{stage}}" } ], @@ -447,7 +447,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -456,7 +456,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.store.normalize.duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" } ], @@ -823,7 +823,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.events.symbolicator.response.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "responses" }, { @@ -832,7 +832,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.events.symbolicator.request_error.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "request errors" } ], @@ -938,7 +938,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.symbolicator.create_task.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "create_task p90" }, { @@ -947,7 +947,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.symbolicator.query_task.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "query_task p50" }, { @@ -956,7 +956,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.symbolicator.query_task.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "query_task p90" }, { @@ -965,7 +965,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.symbolicator.session.request.median{host=~\"$host\"})", - "refId": "A", + "refId": "E", "legendFormat": "session.request p50" }, { @@ -974,7 +974,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.events.symbolicator.session.request.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "F", "legendFormat": "session.request p90" } ], @@ -1341,7 +1341,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.snuba.consumer.insertions.item_type_count.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "item types" } ], @@ -1447,7 +1447,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.insertions.batch_write_ms.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "batch_write p90" }, { @@ -1456,7 +1456,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.insertions.latency_ms.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "insert latency p50" }, { @@ -1465,7 +1465,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.insertions.latency_ms.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "insert latency p90" } ], @@ -1571,7 +1571,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.insertions.end_to_end_latency_ms.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -1580,7 +1580,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.insertions.max_end_to_end_latency_ms.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max p50" } ], @@ -1699,7 +1699,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.snuba.replacer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "replacer run loops" }, { @@ -1708,7 +1708,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.consumer.run.count.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "sentry run loops" } ], @@ -1814,7 +1814,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.snuba.consumer.arroyo.strategies.run_task_in_threads.got_backpressure.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "got backpressure" }, { @@ -1823,7 +1823,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.snuba.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "backpressure p90" } ], diff --git a/grafana-dashboards/sentry-foundational-storage.json b/grafana-dashboards/sentry-foundational-storage.json index 208f3e560d7..e0f86ef2bef 100644 --- a/grafana-dashboards/sentry-foundational-storage.json +++ b/grafana-dashboards/sentry-foundational-storage.json @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "get_multi" } ], @@ -487,7 +487,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.nodestore.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -496,7 +496,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.nodestore.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" } ], @@ -602,7 +602,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.nodestore.set_bytes.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "set_bytes p90" } ], @@ -845,7 +845,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.timers.sentry.filestore.save.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "save" } ], @@ -951,7 +951,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.filestore.read.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "read p90" }, { @@ -960,7 +960,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.filestore.save.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "save p50" }, { @@ -969,7 +969,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.filestore.save.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "save p90" } ], @@ -1212,7 +1212,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "metrics indexer memcache" }, { @@ -1221,7 +1221,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.fetch.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "metrics indexer fetch" } ], @@ -1327,7 +1327,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "idle" }, { @@ -1336,7 +1336,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" }, { @@ -1345,7 +1345,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "waiting" } ], @@ -1494,7 +1494,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.dynamic_sampling.boost_low_volume_transactions.query.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "dynamic sampling queries" }, { @@ -1503,7 +1503,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.release_monitor.fetch_project_release_health_totals.query.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "release monitor queries" } ], diff --git a/grafana-dashboards/sentry-ingestion.json b/grafana-dashboards/sentry-ingestion.json index a3d2b9c2e99..03c7a7c82ef 100644 --- a/grafana-dashboards/sentry-ingestion.json +++ b/grafana-dashboards/sentry-ingestion.json @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "rejected" }, { @@ -390,7 +390,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "error processed" } ], @@ -829,7 +829,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.responses.status_codes.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "responses" } ], @@ -1323,7 +1323,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.relay.config.get_project_config.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -1332,7 +1332,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.relay.config.get_project_config.duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" } ], @@ -1438,7 +1438,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.counters.sentry.relay.project_configs.post_v3.pending.count{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "pending" } ], @@ -1557,7 +1557,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.producer.produce_status.error.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "error" } ], @@ -1663,7 +1663,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_outbuf_latency{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "outbuf latency" }, { @@ -1672,7 +1672,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.consumer.arroyo.producer.librdkafka.p99_rtt{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "rtt" } ], @@ -2039,7 +2039,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.consumer.arroyo.processing.strategies.healthcheck.touch.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "healthcheck touches" } ], @@ -2145,7 +2145,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.processing.time.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "processing p90" }, { @@ -2154,7 +2154,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "backpressure p50" }, { @@ -2163,7 +2163,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.consumer.arroyo.consumer.backpressure.time.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "backpressure p90" } ], @@ -2530,7 +2530,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.spans.buffer.empty_segments.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "empty segments" }, { @@ -2539,7 +2539,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.spans.buffer.flusher_unhealthy.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "flusher unhealthy" } ], @@ -2645,7 +2645,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.spans.buffer.process_spans.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "process_spans p90" }, { @@ -2654,7 +2654,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.spans.buffer.process_batch.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "process_batch p50" }, { @@ -2663,7 +2663,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.spans.buffer.process_batch.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "process_batch p90" } ], diff --git a/grafana-dashboards/sentry-overview.json b/grafana-dashboards/sentry-overview.json index e784e445eb9..9ea3fbba54a 100644 --- a/grafana-dashboards/sentry-overview.json +++ b/grafana-dashboards/sentry-overview.json @@ -411,7 +411,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.rejected.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "rejected" }, { @@ -420,7 +420,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.relay.event.error.processed.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "error processed" } ], @@ -787,7 +787,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -796,7 +796,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" } ], @@ -1260,7 +1260,7 @@ "uid": "${datasource}" }, "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90 {{stage}}" }, { @@ -1269,7 +1269,7 @@ "uid": "${datasource}" }, "expr": "sum by(stage) (stats.timers.sentry.events.since_received.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max {{stage}}" } ], @@ -1636,7 +1636,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "executed" } ], @@ -1742,7 +1742,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90" }, { @@ -1751,7 +1751,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" } ], @@ -2118,7 +2118,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.email.queued.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "queued" } ], @@ -2224,7 +2224,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.data_condition.evaluation.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "condition evaluations" }, { @@ -2233,7 +2233,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.workflow_engine.activity_handler.complete.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "activities completed" } ], @@ -2600,7 +2600,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.nodestore.get_multi.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "get_multi" } ], @@ -2706,7 +2706,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "idle" }, { @@ -2715,7 +2715,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" }, { @@ -2724,7 +2724,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "waiting" } ], diff --git a/grafana-dashboards/sentry-tasks-manager.json b/grafana-dashboards/sentry-tasks-manager.json index 4e8f5bde95e..fb62666ff7d 100644 --- a/grafana-dashboards/sentry-tasks-manager.json +++ b/grafana-dashboards/sentry-tasks-manager.json @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.registry.send_task.success.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "send success" }, { @@ -390,7 +390,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.execute_task.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "executed" } ], @@ -496,7 +496,7 @@ "uid": "${datasource}" }, "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90 {{taskname}}" }, { @@ -505,7 +505,7 @@ "uid": "${datasource}" }, "expr": "sum by(taskname) (stats.timers.sentry.taskworker.worker.execution_duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max {{taskname}}" } ], @@ -872,7 +872,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.processing_deadline_exceeded.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "deadline exceeded" }, { @@ -881,7 +881,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.child_task_queue_empty.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "child queue empty" }, { @@ -890,7 +890,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.max_task_count_reached.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "max task count" }, { @@ -899,7 +899,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.taskworker.worker.add_tasks.child_tasks_full.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "E", "legendFormat": "child tasks full" } ], @@ -1266,7 +1266,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.upkeep.current_processing_tasks{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "processing" }, { @@ -1275,7 +1275,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.upkeep.current_claimed_tasks{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "claimed" }, { @@ -1284,7 +1284,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.upkeep.current_delayed_tasks{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "delayed" } ], @@ -1500,7 +1500,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_miss_total{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "cache misses" }, { @@ -1509,7 +1509,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_spill_total{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "cache spills" } ], @@ -1615,7 +1615,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.upkeep.wal_file_size.bytes{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "wal file size" }, { @@ -1624,7 +1624,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.taskbroker.sqlite.db.cache_used_bytes{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "cache used" } ], @@ -1805,7 +1805,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.taskworker.get_task.rpc.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "get_task p90" }, { @@ -1814,7 +1814,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.taskworker.update_task.rpc.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "update_task p50" }, { @@ -1823,7 +1823,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.taskworker.update_task.rpc.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "update_task p90" } ], @@ -1929,7 +1929,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.taskbroker.grpc_server.get_task.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "get_task p90" }, { @@ -1938,7 +1938,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.taskbroker.grpc_server.set_status.duration.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "set_status p50" }, { @@ -1947,7 +1947,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.taskbroker.grpc_server.set_status.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "set_status p90" }, { @@ -1956,7 +1956,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.taskbroker.grpc_server.fetch_next.duration.median{host=~\"$host\"})", - "refId": "A", + "refId": "E", "legendFormat": "fetch_next p50" }, { @@ -1965,7 +1965,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.taskbroker.grpc_server.fetch_next.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "F", "legendFormat": "fetch_next p90" } ], diff --git a/grafana-dashboards/sentry-ui-api.json b/grafana-dashboards/sentry-ui-api.json index 35e3a0fd4d2..dbe60b39805 100644 --- a/grafana-dashboards/sentry-ui-api.json +++ b/grafana-dashboards/sentry-ui-api.json @@ -478,7 +478,7 @@ "uid": "${datasource}" }, "expr": "sum by(url_name,method) (stats.timers.sentry.view.duration.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "p90 {{method}} {{url_name}}" }, { @@ -487,7 +487,7 @@ "uid": "${datasource}" }, "expr": "sum by(url_name,method) (stats.timers.sentry.view.duration.upper{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max {{method}} {{url_name}}" } ], @@ -813,7 +813,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.middleware.ratelimit.process_response.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "process_response" } ], @@ -919,7 +919,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_view.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "process_view p90" }, { @@ -928,7 +928,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_response.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "process_response p50" }, { @@ -937,7 +937,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.middleware.ratelimit.process_response.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "process_response p90" } ], @@ -1304,7 +1304,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.snuba.db_query.cache_miss.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "misses" }, { @@ -1313,7 +1313,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.snuba.db_query.cache_hit_simple.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "hits simple" } ], @@ -1419,7 +1419,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.snuba.client.snql_query.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "snql_query p90" }, { @@ -1428,7 +1428,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.snuba.client.process_result.median{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "process_result p50" }, { @@ -1437,7 +1437,7 @@ "uid": "${datasource}" }, "expr": "sum (stats.timers.sentry.snuba.client.process_result.upper_90{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "process_result p90" } ], @@ -1556,7 +1556,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.sentry_metrics.indexer.memcache.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "B", "legendFormat": "metrics indexer memcache" }, { @@ -1565,7 +1565,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.cache.memcache.reconnect.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "C", "legendFormat": "memcache reconnect" }, { @@ -1574,7 +1574,7 @@ "uid": "${datasource}" }, "expr": "sum(rate(stats.counters.sentry.dynamic_sampling.boost_low_volume_transactions.query.count{host=~\"$host\"}[$__rate_interval]))", - "refId": "A", + "refId": "D", "legendFormat": "dynamic sampling queries" } ], @@ -1680,7 +1680,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.idle_connections{host=~\"$host\"})", - "refId": "A", + "refId": "B", "legendFormat": "idle" }, { @@ -1689,7 +1689,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.max_connections{host=~\"$host\"})", - "refId": "A", + "refId": "C", "legendFormat": "max" }, { @@ -1698,7 +1698,7 @@ "uid": "${datasource}" }, "expr": "sum(stats.gauges.sentry.relay.redis.pool.waiting_for_connection{host=~\"$host\"})", - "refId": "A", + "refId": "D", "legendFormat": "waiting" } ],