diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md index 4df9dadaf8..d0e4620602 100644 --- a/apps/price_pusher/README.md +++ b/apps/price_pusher/README.md @@ -293,7 +293,7 @@ pnpm run dev evm --config config.evm.mainnet.json --metrics-port 9091 ### Running Locally with Docker -You can run the monitoring stack (Prometheus and Grafana) using the provided docker-compose configuration: +You can run the monitoring stack (Prometheus, Loki, Promtail, and Grafana) using the provided docker-compose configuration: 1. Use the sample docker-compose file for metrics: @@ -303,9 +303,11 @@ docker-compose -f docker-compose.metrics.sample.yaml up This will start: - Prometheus server on port 9090 with the alerts configured in alerts.sample.yml +- Loki server on port 3100 (log aggregation) +- Promtail, configured by promtail.sample.yml to ship every Docker container's logs into Loki and tag them with `namespace=` - Grafana server on port 3000 with default credentials (admin/admin) -The docker-compose.metrics.sample.yaml file includes a pre-configured Grafana dashboard (see the [Dashboard](#dashboard) section below) that displays all the metrics mentioned above. This dashboard provides monitoring of your price pusher operations with panels for configured feeds, active feeds, wallet balance, update statistics, and error tracking. The dashboard is automatically provisioned when you start the stack with docker-compose. +The docker-compose.metrics.sample.yaml file includes a pre-configured Grafana dashboard (see the [Dashboard](#dashboard) section below) that displays all the metrics mentioned above plus log panels (Tx Hash, All Logs, Error Logs) backed by Loki. The dashboard is automatically provisioned when you start the stack with docker-compose; the `chain` template variable lists every container name Promtail has seen, so running one pusher container per chain gives you per-chain log views out of the box. ### Example Grafana Queries diff --git a/apps/price_pusher/datasource.sample.yml b/apps/price_pusher/datasource.sample.yml index 86fd3465e1..ceffb06aac 100644 --- a/apps/price_pusher/datasource.sample.yml +++ b/apps/price_pusher/datasource.sample.yml @@ -1,8 +1,20 @@ apiVersion: 1 datasources: + # Both UIDs below are referenced from grafana-dashboard.sample.json (the + # Prometheus UID by every metric panel, the Loki UID by Tx Hash / All Logs + # / Error Logs and the `chain` template variable). Without explicit UIDs + # Grafana auto-generates per-provision random values and the dashboard's + # hardcoded references fail to resolve. Keep them in sync if the dashboard + # is regenerated. - name: Prometheus type: prometheus access: proxy url: http://prometheus:9090 + uid: edryyydtht14wa isDefault: true + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: ads9ouz3jh4hsa diff --git a/apps/price_pusher/docker-compose.metrics.sample.yaml b/apps/price_pusher/docker-compose.metrics.sample.yaml index b210b9e8e3..e368d1e80d 100644 --- a/apps/price_pusher/docker-compose.metrics.sample.yaml +++ b/apps/price_pusher/docker-compose.metrics.sample.yaml @@ -17,6 +17,28 @@ services: networks: - monitoring + loki: + image: grafana/loki:latest + container_name: loki + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - monitoring + + promtail: + image: grafana/promtail:latest + container_name: promtail + volumes: + - ./promtail.sample.yml:/etc/promtail/config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml + depends_on: + - loki + networks: + - monitoring + grafana: image: grafana/grafana:latest container_name: grafana @@ -33,6 +55,7 @@ services: - GF_USERS_ALLOW_SIGN_UP=false depends_on: - prometheus + - loki networks: - monitoring diff --git a/apps/price_pusher/promtail.sample.yml b/apps/price_pusher/promtail.sample.yml new file mode 100644 index 0000000000..57fd4e0bcf --- /dev/null +++ b/apps/price_pusher/promtail.sample.yml @@ -0,0 +1,47 @@ +# Sample Promtail config that ships Docker container logs to the local Loki +# instance defined in docker-compose.metrics.sample.yaml. It tags every line +# with `namespace=` so the `chain` template variable in +# grafana-dashboard.sample.json (which filters `{namespace=~"$chain"}`) picks +# up one entry per pusher container you run alongside this stack. +# +# Run one pusher container per chain (e.g. `price-pusher-evm`, +# `price-pusher-sui`) and the dashboard's chain selector will list them. + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + # Use the container name (without the leading slash Docker prepends) as + # the `namespace` label that the Grafana dashboard filters on. + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: namespace + - source_labels: ["__meta_docker_container_id"] + target_label: container_id + pipeline_stages: + # Surface pino's standard `level` field so the Error Logs panel + # (`detected_level = error`) can detect log levels emitted as JSON. + # Pino emits numeric severities by default (10=trace, 20=debug, + # 30=info, 40=warn, 50=error, 60=fatal) and Loki's detected_level + # only recognises the string forms, so translate before labelling. + - json: + expressions: + level: level + msg: msg + - template: + source: level + template: '{{ if eq .Value "10" }}trace{{ else if eq .Value "20" }}debug{{ else if eq .Value "30" }}info{{ else if eq .Value "40" }}warn{{ else if eq .Value "50" }}error{{ else if eq .Value "60" }}fatal{{ else }}{{ .Value }}{{ end }}' + - labels: + level: