From 7670d18136606cf4eff107f4bf851cfa596144be Mon Sep 17 00:00:00 2001 From: 0xghost42 Date: Wed, 13 May 2026 15:59:14 +0530 Subject: [PATCH 1/2] fix(price-pusher): provision Loki + Promtail in sample monitoring stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2520. The bundled grafana-dashboard.sample.json references a Loki data source (uid: ads9ouz3jh4hsa) for three panels — Tx Hash, All Logs, Error Logs — and uses it to populate the `chain` template variable. The sample stack only provisioned Prometheus, so on a fresh docker-compose up the dashboard rendered with an unconfigured-data-source banner across all log panels and an empty chain selector. This change makes the sample stack actually run the dashboard as-is: - datasource.sample.yml: add a Loki entry pinned to the UID the dashboard already expects. - docker-compose.metrics.sample.yaml: add a Loki service and a Promtail sidecar that ships Docker container logs into Loki. - promtail.sample.yml: new file. Scrapes container logs via the Docker socket and labels each stream with `namespace=`, which is exactly the label the dashboard's `{namespace=~"$chain"}` filter selects on. JSON pino logs are parsed so that the Error Logs panel's `detected_level = error` matcher works against pino's level field. - README: update the 'Running Locally with Docker' section to describe the new services and the one-container-per-chain pattern the dashboard expects. --- apps/price_pusher/README.md | 6 ++- apps/price_pusher/datasource.sample.yml | 8 ++++ .../docker-compose.metrics.sample.yaml | 23 +++++++++++ apps/price_pusher/promtail.sample.yml | 41 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 apps/price_pusher/promtail.sample.yml diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md index 4df9dadaf8..d0e4620602 100644 --- a/apps/price_pusher/README.md +++ b/apps/price_pusher/README.md @@ -293,7 +293,7 @@ pnpm run dev evm --config config.evm.mainnet.json --metrics-port 9091 ### Running Locally with Docker -You can run the monitoring stack (Prometheus and Grafana) using the provided docker-compose configuration: +You can run the monitoring stack (Prometheus, Loki, Promtail, and Grafana) using the provided docker-compose configuration: 1. Use the sample docker-compose file for metrics: @@ -303,9 +303,11 @@ docker-compose -f docker-compose.metrics.sample.yaml up This will start: - Prometheus server on port 9090 with the alerts configured in alerts.sample.yml +- Loki server on port 3100 (log aggregation) +- Promtail, configured by promtail.sample.yml to ship every Docker container's logs into Loki and tag them with `namespace=` - Grafana server on port 3000 with default credentials (admin/admin) -The docker-compose.metrics.sample.yaml file includes a pre-configured Grafana dashboard (see the [Dashboard](#dashboard) section below) that displays all the metrics mentioned above. This dashboard provides monitoring of your price pusher operations with panels for configured feeds, active feeds, wallet balance, update statistics, and error tracking. The dashboard is automatically provisioned when you start the stack with docker-compose. +The docker-compose.metrics.sample.yaml file includes a pre-configured Grafana dashboard (see the [Dashboard](#dashboard) section below) that displays all the metrics mentioned above plus log panels (Tx Hash, All Logs, Error Logs) backed by Loki. The dashboard is automatically provisioned when you start the stack with docker-compose; the `chain` template variable lists every container name Promtail has seen, so running one pusher container per chain gives you per-chain log views out of the box. ### Example Grafana Queries diff --git a/apps/price_pusher/datasource.sample.yml b/apps/price_pusher/datasource.sample.yml index 86fd3465e1..bcaf05fd5d 100644 --- a/apps/price_pusher/datasource.sample.yml +++ b/apps/price_pusher/datasource.sample.yml @@ -6,3 +6,11 @@ datasources: access: proxy url: http://prometheus:9090 isDefault: true + # The Loki UID is referenced by the provided grafana-dashboard.sample.json + # (Tx Hash / All Logs / Error Logs panels and the `chain` template variable). + # Keep the UID in sync with the dashboard if it ever changes. + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: ads9ouz3jh4hsa diff --git a/apps/price_pusher/docker-compose.metrics.sample.yaml b/apps/price_pusher/docker-compose.metrics.sample.yaml index b210b9e8e3..e368d1e80d 100644 --- a/apps/price_pusher/docker-compose.metrics.sample.yaml +++ b/apps/price_pusher/docker-compose.metrics.sample.yaml @@ -17,6 +17,28 @@ services: networks: - monitoring + loki: + image: grafana/loki:latest + container_name: loki + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - monitoring + + promtail: + image: grafana/promtail:latest + container_name: promtail + volumes: + - ./promtail.sample.yml:/etc/promtail/config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml + depends_on: + - loki + networks: + - monitoring + grafana: image: grafana/grafana:latest container_name: grafana @@ -33,6 +55,7 @@ services: - GF_USERS_ALLOW_SIGN_UP=false depends_on: - prometheus + - loki networks: - monitoring diff --git a/apps/price_pusher/promtail.sample.yml b/apps/price_pusher/promtail.sample.yml new file mode 100644 index 0000000000..70e3d567a3 --- /dev/null +++ b/apps/price_pusher/promtail.sample.yml @@ -0,0 +1,41 @@ +# Sample Promtail config that ships Docker container logs to the local Loki +# instance defined in docker-compose.metrics.sample.yaml. It tags every line +# with `namespace=` so the `chain` template variable in +# grafana-dashboard.sample.json (which filters `{namespace=~"$chain"}`) picks +# up one entry per pusher container you run alongside this stack. +# +# Run one pusher container per chain (e.g. `price-pusher-evm`, +# `price-pusher-sui`) and the dashboard's chain selector will list them. + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + # Use the container name (without the leading slash Docker prepends) as + # the `namespace` label that the Grafana dashboard filters on. + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: namespace + - source_labels: ["__meta_docker_container_id"] + target_label: container_id + pipeline_stages: + # Surface pino's standard `level` field so the Error Logs panel + # (`detected_level = error`) can detect log levels emitted as JSON. + - json: + expressions: + level: level + msg: msg + - labels: + level: From d154baf27e755720dd555128ae4b960d7e11d491 Mon Sep 17 00:00:00 2001 From: 0xghost42 Date: Thu, 14 May 2026 15:55:44 +0530 Subject: [PATCH 2/2] fix(price-pusher): map pino numeric levels + pin Prometheus UID Follow-ups from review feedback on the sample monitoring stack: - promtail.sample.yml: add a `template` stage between `json` and `labels` that maps pino's numeric severities (10/20/30/40/50/60) to the string forms Loki's `detected_level` recognises. Without this the Error Logs panel's `detected_level = error` filter never matches because pino emits `level=50`, not `level=error`. - datasource.sample.yml: set an explicit `uid: edryyydtht14wa` on the Prometheus datasource so dashboards that hardcode the UID resolve in a fresh provision. Previously only the Loki UID was pinned, leaving every Prometheus-backed panel to rely on Grafana auto-assignment. --- apps/price_pusher/datasource.sample.yml | 10 +++++++--- apps/price_pusher/promtail.sample.yml | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/price_pusher/datasource.sample.yml b/apps/price_pusher/datasource.sample.yml index bcaf05fd5d..ceffb06aac 100644 --- a/apps/price_pusher/datasource.sample.yml +++ b/apps/price_pusher/datasource.sample.yml @@ -1,14 +1,18 @@ apiVersion: 1 datasources: + # Both UIDs below are referenced from grafana-dashboard.sample.json (the + # Prometheus UID by every metric panel, the Loki UID by Tx Hash / All Logs + # / Error Logs and the `chain` template variable). Without explicit UIDs + # Grafana auto-generates per-provision random values and the dashboard's + # hardcoded references fail to resolve. Keep them in sync if the dashboard + # is regenerated. - name: Prometheus type: prometheus access: proxy url: http://prometheus:9090 + uid: edryyydtht14wa isDefault: true - # The Loki UID is referenced by the provided grafana-dashboard.sample.json - # (Tx Hash / All Logs / Error Logs panels and the `chain` template variable). - # Keep the UID in sync with the dashboard if it ever changes. - name: Loki type: loki access: proxy diff --git a/apps/price_pusher/promtail.sample.yml b/apps/price_pusher/promtail.sample.yml index 70e3d567a3..57fd4e0bcf 100644 --- a/apps/price_pusher/promtail.sample.yml +++ b/apps/price_pusher/promtail.sample.yml @@ -33,9 +33,15 @@ scrape_configs: pipeline_stages: # Surface pino's standard `level` field so the Error Logs panel # (`detected_level = error`) can detect log levels emitted as JSON. + # Pino emits numeric severities by default (10=trace, 20=debug, + # 30=info, 40=warn, 50=error, 60=fatal) and Loki's detected_level + # only recognises the string forms, so translate before labelling. - json: expressions: level: level msg: msg + - template: + source: level + template: '{{ if eq .Value "10" }}trace{{ else if eq .Value "20" }}debug{{ else if eq .Value "30" }}info{{ else if eq .Value "40" }}warn{{ else if eq .Value "50" }}error{{ else if eq .Value "60" }}fatal{{ else }}{{ .Value }}{{ end }}' - labels: level: