feat: implement Grafana and Glowroot alerting with Telegram and Slack support

deploy/inventory/host_vars/monitor.template

@@ -0,0 +1,11 @@
+---
+# Variables for the monitoring host.
+# Copy to 'monitor': cp monitor.template monitor
+
+# alerting_enabled: true
+# alerting_telegram_bot_token: "123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11"
+# alerting_telegram_chat_id: "-1001234567890"
+
+# alerting_default_contact_point: slack
+# alerting_slack_webhook_url: "https://hooks.slack.com/services/T.../B.../xxx"
+# alerting_slack_channel: "#dhis2-alerts"

deploy/inventory/hosts.template

@@ -53,6 +53,12 @@ postgresql_version=16
 server_monitoring=munin
 app_monitoring=glowroot
 
+# alerting (requires server_monitoring=grafana, see docs/alerting.md)
+# alerting_enabled=false
+# alerting_default_contact_point=telegram
+# alerting_telegram_bot_token=
+# alerting_telegram_chat_id=
+# glowroot_alerting_enabled=false
 
 # lxd
 lxd_network=172.19.2.1/24 

deploy/roles/create-instance/defaults/main/glowroot_alerting.yml

@@ -0,0 +1,25 @@
+---
+glowroot_alerting_enabled: false
+glowroot_telegram_forwarder_port: '9099'
+
+# Slack (native Glowroot)
+# glowroot_slack_webhook_url: ""
+# glowroot_slack_webhook_display: "DHIS2 Alerts"
+
+# Email / SMTP (native Glowroot)
+# glowroot_smtp_host: ""
+# glowroot_smtp_port: 587
+# glowroot_smtp_connection_security: "STARTTLS"
+# glowroot_smtp_username: ""
+# glowroot_smtp_password: ""
+# glowroot_smtp_from_address: ""
+# glowroot_smtp_from_name: "Glowroot DHIS2"
+# glowroot_alert_email_addresses: []
+
+# Thresholds
+glowroot_alert_p95_threshold_ms: 10000
+glowroot_alert_p95_time_period_seconds: 600
+glowroot_alert_error_rate_threshold: 10.0
+glowroot_alert_error_rate_time_period_seconds: 300
+glowroot_alert_heartbeat_seconds: 300
+glowroot_alert_min_transaction_count: 10

deploy/roles/create-instance/handlers/main.yml

@@ -49,3 +49,9 @@
   ansible.builtin.service:
     name: munin-node
     state: restarted
+
+- name: Restart Glowroot Telegram Forwarder
+  ansible.builtin.systemd:
+    name: glowroot-telegram-forwarder
+    state: restarted
+    daemon_reload: true

deploy/roles/create-instance/tasks/glowroot.yml

@@ -102,6 +102,7 @@
     owner: root
     group: tomcat
     mode: "0660"
+  no_log: true
   when: not glowroot_admin_file_status.stat.exists
   notify: Restart Tomcat
 
@@ -112,3 +113,95 @@
     line: '\1"contextPath": "{{ "/glowroot" if dhis2_base_path | default(inventory_hostname) | to_fixed_string == "ROOT" else "/" + dhis2_base_path | default(inventory_hostname) | to_fixed_string + "-glowroot" }}",'
     backrefs: true
   notify: Restart Tomcat
+
+# Glowroot alerting pre-flight check
+- name: Glowroot | Assert at least one notification channel is configured
+  ansible.builtin.assert:
+    that: >-
+      (alerting_telegram_bot_token is defined and alerting_telegram_chat_id is defined) or
+      (glowroot_slack_webhook_url is defined) or
+      (glowroot_smtp_host is defined)
+    fail_msg: >-
+      glowroot_alerting_enabled is true but no notification channel is configured.
+      Set alerting_telegram_bot_token + alerting_telegram_chat_id,
+      or glowroot_slack_webhook_url, or glowroot_smtp_host.
+  when: glowroot_alerting_enabled | default(false) | bool
+
+# Telegram forwarder for Glowroot alerts
+- name: Glowroot | Create forwarder system group
+  ansible.builtin.group:
+    name: glowroot_forwarder
+    system: true
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - alerting_telegram_bot_token is defined
+    - alerting_telegram_chat_id is defined
+
+- name: Glowroot | Create forwarder system user
+  ansible.builtin.user:
+    name: glowroot_forwarder
+    shell: /bin/false
+    create_home: false
+    system: true
+    group: glowroot_forwarder
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - alerting_telegram_bot_token is defined
+    - alerting_telegram_chat_id is defined
+
+- name: Glowroot | Deploy Telegram forwarder script
+  ansible.builtin.template:
+    src: glowroot-telegram-forwarder.py.j2
+    dest: /opt/glowroot/glowroot-telegram-forwarder.py
+    owner: root
+    group: glowroot_forwarder
+    mode: "0640"
+  no_log: true
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - alerting_telegram_bot_token is defined
+    - alerting_telegram_chat_id is defined
+  notify: Restart Glowroot Telegram Forwarder
+
+- name: Glowroot | Deploy Telegram forwarder systemd service
+  ansible.builtin.template:
+    src: glowroot-telegram-forwarder.service.j2
+    dest: /etc/systemd/system/glowroot-telegram-forwarder.service
+    owner: root
+    group: root
+    mode: "0644"
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - alerting_telegram_bot_token is defined
+    - alerting_telegram_chat_id is defined
+  notify: Restart Glowroot Telegram Forwarder
+
+- name: Glowroot | Enable and start Telegram forwarder
+  ansible.builtin.systemd:
+    name: glowroot-telegram-forwarder
+    state: started
+    enabled: true
+    daemon_reload: true
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - alerting_telegram_bot_token is defined
+    - alerting_telegram_chat_id is defined
+
+# Glowroot alert rules config
+- name: Glowroot | Check if config.json exists
+  ansible.builtin.stat:
+    path: /opt/glowroot/config.json
+  register: glowroot_config_file_status
+  when: glowroot_alerting_enabled | default(false) | bool
+
+- name: Glowroot | Deploy config.json with alert rules
+  ansible.builtin.template:
+    src: glowroot_config.json.j2
+    dest: /opt/glowroot/config.json
+    owner: root
+    group: tomcat
+    mode: "0660"
+  when:
+    - glowroot_alerting_enabled | default(false) | bool
+    - not (glowroot_config_file_status.stat.exists | default(false))
+  notify: Restart Tomcat

deploy/roles/create-instance/templates/glowroot-telegram-forwarder.py.j2

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# {{ ansible_managed }}
+"""Slack-to-Telegram webhook forwarder for Glowroot alerts.
+
+Listens on localhost for Slack-formatted webhook POSTs from Glowroot
+and forwards them to a Telegram chat via the Bot API.
+
+Zero external dependencies -- uses only Python 3 stdlib.
+"""
+
+import json
+import sys
+import urllib.request
+import urllib.error
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+LISTEN_HOST = "127.0.0.1"
+LISTEN_PORT = {{ glowroot_telegram_forwarder_port | default(9099) }}
+TELEGRAM_BOT_TOKEN = "{{ alerting_telegram_bot_token }}"
+TELEGRAM_CHAT_ID = "{{ alerting_telegram_chat_id }}"
+TELEGRAM_API = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
+
+STATUS_ICONS = {"danger": "\u26a0\ufe0f", "good": "\u2705", "warning": "\u26a0\ufe0f"}
+
+
+def slack_to_telegram_html(payload):
+    """Convert a Slack attachment payload to Telegram HTML message."""
+    attachments = payload.get("attachments", [])
+    if not attachments:
+        text = payload.get("text", "Unknown alert")
+        return f"<b>Glowroot Alert</b>\n{text}"
+
+    att = attachments[0]
+    color = att.get("color", "warning")
+    icon = STATUS_ICONS.get(color, "\u2139\ufe0f")
+    pretext = att.get("pretext", "Alert")
+    body = att.get("text", "")
+    fallback = att.get("fallback", "")
+
+    lines = [f"{icon} <b>{pretext}</b>"]
+    if body:
+        lines.append(f"\n{body}")
+    elif fallback:
+        lines.append(f"\n{fallback}")
+    return "\n".join(lines)
+
+
+def send_telegram(message):
+    """POST a message to the Telegram Bot API."""
+    data = json.dumps({
+        "chat_id": TELEGRAM_CHAT_ID,
+        "text": message[:4096],
+        "parse_mode": "HTML",
+    }).encode("utf-8")
+    req = urllib.request.Request(
+        TELEGRAM_API,
+        data=data,
+        headers={"Content-Type": "application/json"},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            return resp.status
+    except urllib.error.URLError as exc:
+        print(f"Telegram API error: {exc}", file=sys.stderr)
+        return None
+
+
+class WebhookHandler(BaseHTTPRequestHandler):
+    """Handle incoming Slack-formatted webhook POST requests."""
+
+    def do_POST(self):  # noqa: N802
+        length = int(self.headers.get("Content-Length", 0))
+        raw = self.rfile.read(length)
+        try:
+            payload = json.loads(raw)
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            self.send_response(400)
+            self.end_headers()
+            return
+
+        message = slack_to_telegram_html(payload)
+        status = send_telegram(message)
+        code = 200 if status == 200 else 502
+        self.send_response(code)
+        self.end_headers()
+
+    def log_message(self, fmt, *args):
+        """Silence default stderr logging in production."""
+
+
+if __name__ == "__main__":
+    server = HTTPServer((LISTEN_HOST, LISTEN_PORT), WebhookHandler)
+    print(f"Listening on {LISTEN_HOST}:{LISTEN_PORT}")
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        server.server_close()

deploy/roles/create-instance/templates/glowroot-telegram-forwarder.service.j2‎

@@ -0,0 +1,20 @@
+{{ ansible_managed | comment }}
+[Unit]
+Description=Glowroot Slack-to-Telegram webhook forwarder
+After=network.target
+
+[Service]
+Type=simple
+User=glowroot_forwarder
+Group=glowroot_forwarder
+ExecStart=/usr/bin/python3 /opt/glowroot/glowroot-telegram-forwarder.py
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectSystem=strict
+ReadOnlyPaths=/opt/glowroot/glowroot-telegram-forwarder.py
+ProtectHome=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

deploy/roles/create-instance/templates/glowroot_admin.json

@@ -49,4 +49,46 @@
     ],
     "traceCappedDatabaseSizeMb": 500
   }
+  {% if glowroot_alerting_enabled | default(false) | bool %}
+{% if alerting_telegram_bot_token is defined and alerting_telegram_chat_id is defined %}
+  ,"slack": {
+    "webhooks": [
+      {
+        "url": "http://127.0.0.1:{{ glowroot_telegram_forwarder_port | default(9099) }}",
+        "display": "Telegram (via forwarder)"
+      }
+    ]
+  }
+{% elif glowroot_slack_webhook_url is defined and glowroot_slack_webhook_url | length > 0 %}
+  ,"slack": {
+    "webhooks": [
+      {
+        "url": "{{ glowroot_slack_webhook_url }}",
+        "display": "{{ glowroot_slack_webhook_display | default('DHIS2 Alerts') }}"
+      }
+    ]
+  }
+{% endif %}
+{% if glowroot_smtp_host is defined and glowroot_smtp_host | length > 0 %}
+  ,"smtp": {
+    "host": "{{ glowroot_smtp_host }}",
+    "port": {{ glowroot_smtp_port | default(587) }},
+    "connectionSecurity": "{{ glowroot_smtp_connection_security | default('STARTTLS') }}",
+    "username": "{{ glowroot_smtp_username | default('') }}",
+    "password": "{{ glowroot_smtp_password | default('') }}",
+    "fromEmailAddress": "{{ glowroot_smtp_from_address | default('') }}",
+    "fromDisplayName": "{{ glowroot_smtp_from_name | default('Glowroot DHIS2') }}"
+  }
+{% endif %}
+{% if glowroot_pagerduty_key is defined and glowroot_pagerduty_key | length > 0 %}
+  ,"pagerDuty": {
+    "integrationKeys": [
+      {
+        "key": "{{ glowroot_pagerduty_key }}",
+        "display": "{{ glowroot_pagerduty_display | default('DHIS2') }}"
+      }
+    ]
+  }
+{% endif %}
+{% endif %}
 }