From cf0e13d92fcb459dd2a6ea27dd9bb33ef2ef6e79 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Thu, 30 Oct 2025 15:51:25 +0100 Subject: [PATCH 01/21] JE-66040 --- addons/monitoring/manifest.yml | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 addons/monitoring/manifest.yml diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml new file mode 100644 index 00000000..295818de --- /dev/null +++ b/addons/monitoring/manifest.yml @@ -0,0 +1,62 @@ +type: update +name: Database Monitoring +id: db-monitoring + +description: + text: The Database Monitoring add-on performs a comprehensive monitoring of your MySQL/MariaDB/Percona Database to detect data corruption and inconsistency in the components. Add-on will **temporarily stop all database services** for the duration of the diagnostic to ensure accurate results. Detected issues will be listed in the recovery log. + short: The add-on checks your database for corrupted or inconsistent data in the components. + +logo: /images/database-monitoring.png + +baseUrl: https://cdn.jsdelivr.net/gh/jelastic-jps/mysql-cluster@master/addons/monitoring + +mixins: + - https://cdn.jsdelivr.net/gh/jelastic-jps/mysql-cluster@3.0.0/scripts/common.yml + +targetNodes: + nodeType: + - mysql + - mariadb-dockerized + - mariadb + - perconadb + +settings: + submitUnchanged: true + fields: + - name: user + caption: User + type: string + required: true + - name: password + caption: Password + type: string + inputType: password + required: true + +onAfterClone: + install: ${baseUrl}/manifest.yml?_r=${fn.random} + envName: ${event.response.env.envName} + nodeGroup: ${targetNodes.nodeGroup} + settings: + install: true + user: ${settings.user} + password: ${settings.password} + +onInstall: + - if (!${settings.install:false}): authValidate + - getReplicaUser + - if (!${settings.install:false}): init + +actions: + authValidate: + - forEach(i:nodes.sqldb): + - cmd[${@i.id}]: mysqladmin ping -u${settings.user} -p${settings.password} 2>/dev/null 1>/dev/null; MYSQLD_RUNNING=${?}; echo ${MYSQLD_RUNNING}; + - if ('${response.out}' == '0'): + - cmd[${@i.id}]: mysql -u${settings.user} -p${settings.password} -e "EXIT" 2>/dev/null 1>/dev/null; MYSQLD_RUNNING=${?}; echo ${MYSQLD_RUNNING}; + - if ('${response.out}' != '0'): + return: + type: warning + message: Authentication failed, please check User/Password. + + init: + log: Monitoring is initialized \ No newline at end of file From 3af555d23d24b9e8c7adda38f63efbe2d4af99a3 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Thu, 30 Oct 2025 16:56:47 +0100 Subject: [PATCH 02/21] JE-66040 --- addons/monitoring/manifest.yml | 13 +- addons/monitoring/scripts/db-monitoring.sh | 156 +++++++++++++++++++++ 2 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 addons/monitoring/scripts/db-monitoring.sh diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 295818de..591d9e11 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -8,7 +8,7 @@ description: logo: /images/database-monitoring.png -baseUrl: https://cdn.jsdelivr.net/gh/jelastic-jps/mysql-cluster@master/addons/monitoring +baseUrl: https://raw.githubusercontent.com/sych74/mysql-cluster/JE-66040/addons/monitoring mixins: - https://cdn.jsdelivr.net/gh/jelastic-jps/mysql-cluster@3.0.0/scripts/common.yml @@ -46,6 +46,7 @@ onInstall: - if (!${settings.install:false}): authValidate - getReplicaUser - if (!${settings.install:false}): init + - setupCron actions: authValidate: @@ -59,4 +60,12 @@ actions: message: Authentication failed, please check User/Password. init: - log: Monitoring is initialized \ No newline at end of file + log: Monitoring is initialized + setupCron: + - forEach(i:nodes.sqldb): + - cmd[${@i.id}]: |- + curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/bin/db-monitoring.sh && chmod +x /usr/local/bin/db-monitoring.sh + - cmd[${@i.id}]: |- + echo "* * * * * root /usr/local/bin/db-monitoring.sh ${settings.user} ${settings.password} >/dev/null 2>&1" > /etc/cron.d/db-monitoring && chmod 644 /etc/cron.d/db-monitoring + - cmd[${@i.id}]: |- + if command -v systemctl >/dev/null 2>&1; then systemctl reload crond 2>/dev/null || systemctl reload cron 2>/dev/null || true; else service crond reload 2>/dev/null || service cron reload 2>/dev/null || true; fi \ No newline at end of file diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh new file mode 100644 index 00000000..75997e95 --- /dev/null +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +DB_USER=$1 +DB_PASSWORD=$2 +MONITORING_LOG=/var/log/db-monitoring.log +STATUS_FILE=/var/tmp/db-monitoring.status + +# email notification via Jelastic API +function sendEmailNotification(){ + if [ -e "/usr/lib/jelastic/modules/api.module" ]; then + [ -e "/var/run/jem.pid" ] && return 0 + CURRENT_PLATFORM_MAJOR_VERSION=$(jem api apicall -s --connect-timeout 3 --max-time 15 [API_DOMAIN]/1.0/statistic/system/rest/getversion 2>/dev/null | jq .version | grep -o [0-9.]* | awk -F . '{print $1}') + if [ "${CURRENT_PLATFORM_MAJOR_VERSION}" -ge "7" ]; then + echo $(date) ${ENV_NAME} "Sending e-mail notification about high DB connections usage" | tee -a $MONITORING_LOG + SUBJECT="${ENV_NAME}: MySQL connections usage reached threshold" + BODY="$1" + jem api apicall -s --connect-timeout 3 --max-time 15 [API_DOMAIN]/1.0/message/email/rest/send --data-urlencode "session=$USER_SESSION" --data-urlencode "to=$USER_EMAIL" --data-urlencode "subject=$SUBJECT" --data-urlencode "body=$BODY" + if [[ $? != 0 ]]; then + echo $(date) ${ENV_NAME} "Sending of e-mail notification failed" | tee -a $MONITORING_LOG + else + echo $(date) ${ENV_NAME} "E-mail notification is sent successfully" | tee -a $MONITORING_LOG + fi + elif [ -z "${CURRENT_PLATFORM_MAJOR_VERSION}" ]; then + echo $(date) ${ENV_NAME} "Error when checking the platform version" | tee -a $MONITORING_LOG + else + echo $(date) ${ENV_NAME} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG + fi + else + echo $(date) ${ENV_NAME} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG + fi +} + +# status helpers: send email once per status change +get_last_status(){ + [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" 2>/dev/null || echo "" +} + +set_status(){ + echo "$1" > "$STATUS_FILE" 2>/dev/null || true +} + +send_on_status_change(){ + local new_status="$1" + shift + local body="$*" + local last_status="$(get_last_status)" + if [ "$new_status" != "$last_status" ]; then + sendEmailNotification "$body" + set_status "$new_status" + else + echo "$(date) ${ENV_NAME} Status '$new_status' unchanged, skipping email" >> $MONITORING_LOG + fi +} + +echo "Monitoring started at $(date)" >> $MONITORING_LOG + +# Collect metrics using mysqladmin status +STATUS_RAW=$(mysqladmin status -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) +RET=$? +if [ $RET -ne 0 ] || [ -z "$STATUS_RAW" ]; then + BODY="MySQL/MariaDB monitoring error on ${ENV_NAME} + +Action: mysqladmin status +Exit code: $RET +Output:\n$STATUS_RAW +Timestamp: $(date)" + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "STATUS_ERROR" "$BODY" + echo "Monitoring finished at $(date)" >> $MONITORING_LOG + exit 1 +fi +STATUS="$STATUS_RAW" + +# Example STATUS: "Uptime: 12345 Threads: 12 Questions: 34567 Slow queries: 0 Opens: 132 Flush tables: 1 Open tables: 64 Queries per second avg: 2.80" +UPTIME=$(echo "$STATUS" | awk -F'Uptime: | ' '{print $2}') +THREADS=$(echo "$STATUS" | awk -F'Threads: | ' '{print $3}') +QUESTIONS=$(echo "$STATUS" | awk -F'Questions: | ' '{print $4}') +SLOW=$(echo "$STATUS" | awk -F'Slow queries: | ' '{print $5}') +OPENS=$(echo "$STATUS" | awk -F'Opens: | ' '{print $6}') +FLUSHES=$(echo "$STATUS" | awk -F'Flush tables: | ' '{print $7}') +OPEN_TABLES=$(echo "$STATUS" | awk -F'Open tables: | ' '{print $8}') +QPS=$(echo "$STATUS" | awk -F'Queries per second avg: ' '{print $2}') + +# Get max_connections (prefer mysqladmin variables to stay within mysqladmin tooling) +VARS_RAW=$(mysqladmin variables -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) +VARS_RC=$? +if [ $VARS_RC -eq 0 ]; then + MAX_CONNECTIONS=$(echo "$VARS_RAW" | awk -F'|' '/max_connections/ {gsub(/ /,"",$0); print $3; exit}') +fi + +if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then + # fallback to mysql client if parsing failed or command failed + FALLBACK_RAW=$(mysql -Nse "SHOW VARIABLES LIKE 'max_connections';" -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) + FALLBACK_RC=$? + if [ $FALLBACK_RC -eq 0 ]; then + MAX_CONNECTIONS=$(echo "$FALLBACK_RAW" | awk '{print $2}') + fi +fi + +if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then + BODY="MySQL/MariaDB monitoring error on ${ENV_NAME} + +Issue: Unable to determine max_connections +mysqladmin variables exit: $VARS_RC +mysqladmin variables output:\n$VARS_RAW +mysql fallback exit: ${FALLBACK_RC:-not_executed} +mysql fallback output:\n${FALLBACK_RAW:-N/A} +Timestamp: $(date)" + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "MAXCONN_ERROR" "$BODY" + # continue with MAX_CONNECTIONS=0 to avoid division errors + MAX_CONNECTIONS=0 +fi + +# Calculate usage percentage (integer) +USAGE_PCT=0 +if [ "$MAX_CONNECTIONS" -gt 0 ]; then + USAGE_PCT=$(awk -v th="$THREADS" -v max="$MAX_CONNECTIONS" 'BEGIN { if (max>0) printf("%d", (th*100)/max); else print 0 }') +fi + +# Compose metrics body +METRICS_BODY="MySQL/MariaDB connections usage alert on ${ENV_NAME} + +Status: $STATUS +max_connections: $MAX_CONNECTIONS +Current threads (connections): $THREADS +Usage: ${USAGE_PCT}% +Uptime: $UPTIME sec +Questions: $QUESTIONS +Slow queries: $SLOW +Opens: $OPENS +Flush tables: $FLUSHES +Open tables: $OPEN_TABLES +QPS (avg): $QPS +Timestamp: $(date)" + +echo "$METRICS_BODY" >> $MONITORING_LOG + +# Determine status and send only on change +THRESHOLD=70 +if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then + send_on_status_change "THRESHOLD" "$METRICS_BODY" +else + OK_BODY="MySQL/MariaDB connections back to normal on ${ENV_NAME} + +Status: $STATUS +max_connections: $MAX_CONNECTIONS +Current threads (connections): $THREADS +Usage: ${USAGE_PCT}% +Timestamp: $(date)" + send_on_status_change "OK" "$OK_BODY" +fi + +echo "Monitoring finished at $(date)" >> $MONITORING_LOG + +exit 0 \ No newline at end of file From 8e5faf2d756718d8d4c2bca1ce0d1dd5f7e8fb24 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Thu, 30 Oct 2025 17:26:37 +0100 Subject: [PATCH 03/21] JE-66040 --- addons/monitoring/manifest.yml | 11 +++--- addons/monitoring/scripts/db-monitoring.sh | 40 +++++++++++++++++++--- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 591d9e11..0ca30919 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -61,11 +61,8 @@ actions: init: log: Monitoring is initialized + setupCron: - - forEach(i:nodes.sqldb): - - cmd[${@i.id}]: |- - curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/bin/db-monitoring.sh && chmod +x /usr/local/bin/db-monitoring.sh - - cmd[${@i.id}]: |- - echo "* * * * * root /usr/local/bin/db-monitoring.sh ${settings.user} ${settings.password} >/dev/null 2>&1" > /etc/cron.d/db-monitoring && chmod 644 /etc/cron.d/db-monitoring - - cmd[${@i.id}]: |- - if command -v systemctl >/dev/null 2>&1; then systemctl reload crond 2>/dev/null || systemctl reload cron 2>/dev/null || true; else service crond reload 2>/dev/null || service cron reload 2>/dev/null || true; fi \ No newline at end of file + - cmd[sqldb]: |- + curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh && echo "* * * * * root /usr/local/sbin/db-monitoring.sh >/dev/null 2>&1" > /etc/cron.d/db-monitoring && chmod 644 /etc/cron.d/db-monitoring && (systemctl reload crond 2>/dev/null || systemctl reload cron 2>/dev/null || service crond reload 2>/dev/null || service cron reload 2>/dev/null || true) + user: root diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 75997e95..61704925 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -1,9 +1,10 @@ #!/bin/bash -DB_USER=$1 -DB_PASSWORD=$2 +DB_USER="" +DB_PASSWORD="" MONITORING_LOG=/var/log/db-monitoring.log STATUS_FILE=/var/tmp/db-monitoring.status +BODY_ERROR_PREFIX="MySQL/MariaDB monitoring error on ${ENV_NAME}" # email notification via Jelastic API function sendEmailNotification(){ @@ -54,11 +55,42 @@ send_on_status_change(){ echo "Monitoring started at $(date)" >> $MONITORING_LOG +# Read credentials from /.jelenv (REPLICA_USER/REPLICA_PSWD) only; if absent -> notify and exit +if [ ! -f "/.jelenv" ]; then + BODY="${BODY_ERROR_PREFIX} + +Issue: Credentials file /.jelenv not found +Action required: Ensure REPLICA_USER/REPLICA_PSWD are provisioned in /.jelenv +Timestamp: $(date)" + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "CREDENTIALS_MISSING" "$BODY" + echo "Monitoring finished at $(date)" >> $MONITORING_LOG + exit 1 +fi + +source "/.jelenv" + +DB_USER="$REPLICA_USER" +DB_PASSWORD="$REPLICA_PSWD" + +if [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then + BODY="${BODY_ERROR_PREFIX} + +Issue: Missing REPLICA_USER or REPLICA_PSWD in /.jelenv +Observed values: REPLICA_USER='${REPLICA_USER:-EMPTY}', REPLICA_PSWD='${REPLICA_PSWD:+SET}' +Action required: Populate both variables in /.jelenv +Timestamp: $(date)" + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "CREDENTIALS_MISSING" "$BODY" + echo "Monitoring finished at $(date)" >> $MONITORING_LOG + exit 1 +fi + # Collect metrics using mysqladmin status STATUS_RAW=$(mysqladmin status -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) RET=$? if [ $RET -ne 0 ] || [ -z "$STATUS_RAW" ]; then - BODY="MySQL/MariaDB monitoring error on ${ENV_NAME} + BODY="${BODY_ERROR_PREFIX} Action: mysqladmin status Exit code: $RET @@ -98,7 +130,7 @@ if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then fi if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then - BODY="MySQL/MariaDB monitoring error on ${ENV_NAME} + BODY="${BODY_ERROR_PREFIX} Issue: Unable to determine max_connections mysqladmin variables exit: $VARS_RC From e6deb556a1ae2083f1f9aa4f4702f81598fbd1c3 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Tue, 4 Nov 2025 14:59:30 +0100 Subject: [PATCH 04/21] JE-66040 --- addons/monitoring/manifest.yml | 146 ++++++++++++++---- addons/monitoring/scripts/db-monitoring.sh | 5 + .../monitoring/scripts/monitoring-runner.js | 33 ++++ 3 files changed, 155 insertions(+), 29 deletions(-) create mode 100644 addons/monitoring/scripts/monitoring-runner.js diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 0ca30919..8c264aa8 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -21,17 +21,37 @@ targetNodes: - perconadb settings: - submitUnchanged: true - fields: - - name: user - caption: User - type: string - required: true - - name: password - caption: Password - type: string - inputType: password - required: true + main: + submitUnchanged: true + fields: + - name: monitorInterval + caption: Monitoring interval + type: list + editable: false + values: + - value: 5 + caption: Every 5 minutes + - value: 10 + caption: Every 10 minutes + - value: 15 + caption: Every 15 minutes + - value: 20 + caption: Every 20 minutes + - value: 30 + caption: Every 30 minutes + - value: 40 + caption: Every 40 minutes + - value: 50 + caption: Every 50 minutes + default: 10 + +buttons: + - name: Configure + caption: Configure + action: configure + settings: main + loadingText: Configuring... + successText: The monitoring configs have been updated successfully. onAfterClone: install: ${baseUrl}/manifest.yml?_r=${fn.random} @@ -39,30 +59,98 @@ onAfterClone: nodeGroup: ${targetNodes.nodeGroup} settings: install: true - user: ${settings.user} - password: ${settings.password} + monitorInterval: ${settings.monitorInterval} onInstall: - - if (!${settings.install:false}): authValidate - - getReplicaUser - if (!${settings.install:false}): init - - setupCron + - downloadMonitoringScript + - createMonitoringScript + - createMonitoringSchedule + +onUninstall: + - removeMonitoringSchedule + - removeMonitoringScript + - removeMonitoringBinary + +onBeforeDelete: + - removeMonitoringSchedule + - removeMonitoringScript + - removeMonitoringBinary actions: - authValidate: - - forEach(i:nodes.sqldb): - - cmd[${@i.id}]: mysqladmin ping -u${settings.user} -p${settings.password} 2>/dev/null 1>/dev/null; MYSQLD_RUNNING=${?}; echo ${MYSQLD_RUNNING}; - - if ('${response.out}' == '0'): - - cmd[${@i.id}]: mysql -u${settings.user} -p${settings.password} -e "EXIT" 2>/dev/null 1>/dev/null; MYSQLD_RUNNING=${?}; echo ${MYSQLD_RUNNING}; - - if ('${response.out}' != '0'): - return: - type: warning - message: Authentication failed, please check User/Password. - init: log: Monitoring is initialized - - setupCron: + + downloadMonitoringScript: + - cmd[sqldb]: |- + curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh + user: root + + createMonitoringScript: + script: | + let Response = com.hivext.api.Response; + let Transport = com.hivext.api.core.utils.Transport; + let scriptBody = new Transport().get("${baseUrl}/scripts/monitoring-runner.js"); + let scriptName = "${env.name}-dbMonitoringRunner"; + + var resp = api.dev.scripting.GetScript(appid, session, scriptName); + if (resp.result == Response.OK) { + api.dev.scripting.DeleteScript(appid, session, scriptName); + } + + resp = api.dev.scripting.CreateScript(appid, session, scriptName, "js", scriptBody); + if (resp.result != 0) return resp; + java.lang.Thread.sleep(500); + resp = api.dev.scripting.Build(appid, session, scriptName); + if (resp.result != 0) return resp; + return { result: 0 }; + + createMonitoringSchedule: + script: | + let scriptName = "${env.name}-dbMonitoringRunner"; + let trigger = "cron:0 0/${settings.monitorInterval} * ? * * *"; + + var tasks = api.utils.scheduler.GetTasks({ appid: appid, session: session }).objects; + for (var i = 0, l = tasks.length; i < l; i++) { + if (tasks[i].script == scriptName) { + api.utils.scheduler.RemoveTask({ appid: appid, session: session, id: tasks[i].id }); + } + } + + return api.utils.scheduler.CreateEnvTask({ + appid: appid, + envName: "${env.name}", + session: session, + script: scriptName, + trigger: trigger, + description: "Run DB monitoring", + params: { envName: "${env.name}" } + }); + + removeMonitoringSchedule: + script: | + let scriptName = "${env.name}-dbMonitoringRunner"; + let tasks = api.utils.scheduler.GetTasks({ appid: appid, session: session }).objects; + for (var i = 0, l = tasks.length; i < l; i++) { + if (tasks[i].script == scriptName) { + api.utils.scheduler.RemoveTask({ appid: appid, session: session, id: tasks[i].id }); + } + } + return { result: 0 }; + + removeMonitoringScript: + script: | + let scriptName = "${env.name}-dbMonitoringRunner"; + var resp = api.dev.scripting.GetScript(appid, session, scriptName); + if (resp.result == com.hivext.api.Response.OK) { + api.dev.scripting.DeleteScript(appid, session, scriptName); + } + return { result: 0 }; + + removeMonitoringBinary: - cmd[sqldb]: |- - curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh && echo "* * * * * root /usr/local/sbin/db-monitoring.sh >/dev/null 2>&1" > /etc/cron.d/db-monitoring && chmod 644 /etc/cron.d/db-monitoring && (systemctl reload crond 2>/dev/null || systemctl reload cron 2>/dev/null || service crond reload 2>/dev/null || service cron reload 2>/dev/null || true) + rm -f /usr/local/sbin/db-monitoring.sh /etc/cron.d/db-monitoring || true user: root + + configure: + - createMonitoringSchedule diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 61704925..f4d09435 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -6,6 +6,11 @@ MONITORING_LOG=/var/log/db-monitoring.log STATUS_FILE=/var/tmp/db-monitoring.status BODY_ERROR_PREFIX="MySQL/MariaDB monitoring error on ${ENV_NAME}" + +# Accept USER_SESSION and USER_EMAIL via positional args if provided +if [ -n "$1" ]; then USER_SESSION="$1"; fi +if [ -n "$2" ]; then USER_EMAIL="$2"; fi + # email notification via Jelastic API function sendEmailNotification(){ if [ -e "/usr/lib/jelastic/modules/api.module" ]; then diff --git a/addons/monitoring/scripts/monitoring-runner.js b/addons/monitoring/scripts/monitoring-runner.js new file mode 100644 index 00000000..97fc7e4f --- /dev/null +++ b/addons/monitoring/scripts/monitoring-runner.js @@ -0,0 +1,33 @@ +//@auth + +var ROOT = "root"; +var envName = getParam("envName", "${env.envName}"); + +function run() { + var info = jelastic.env.control.GetEnvInfo(envName, session); + if (info.result != 0) return info; + + var nodes = info.nodes || [], node, resp; + var userEmail = user.email; + var userSession = session; + // pass USER_SESSION and USER_EMAIL as positional arguments + var command = "/usr/local/sbin/db-monitoring.sh '" + userSession + "' '" + userEmail + "'"; + + for (var i = 0, n = nodes.length; i < n; i++) { + node = nodes[i]; + if (node.nodeGroup == "sqldb") { + resp = jelastic.env.control.ExecCmdById(envName, session, node.id, toJSON([{ command: command }]), true, ROOT); + if (resp.result != 0) return resp; + } + } + + return { result: 0 }; +} + +try { + return run(); +} catch (ex) { + return { result: com.hivext.api.Response.ERROR_UNKNOWN, error: "Error: " + toJSON(ex) }; +} + + From c2084d468aceaad3d7452cede197bfbea646bb00 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Tue, 4 Nov 2025 15:46:00 +0100 Subject: [PATCH 05/21] JE-66040 --- addons/monitoring/manifest.yml | 15 +++++++++------ .../{monitoring-runner.js => db-monitoring.js} | 0 2 files changed, 9 insertions(+), 6 deletions(-) rename addons/monitoring/scripts/{monitoring-runner.js => db-monitoring.js} (100%) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 8c264aa8..67d68bc7 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -53,6 +53,9 @@ buttons: loadingText: Configuring... successText: The monitoring configs have been updated successfully. +globals: + scriptSufix: db-monitoring + onAfterClone: install: ${baseUrl}/manifest.yml?_r=${fn.random} envName: ${event.response.env.envName} @@ -90,8 +93,8 @@ actions: script: | let Response = com.hivext.api.Response; let Transport = com.hivext.api.core.utils.Transport; - let scriptBody = new Transport().get("${baseUrl}/scripts/monitoring-runner.js"); - let scriptName = "${env.name}-dbMonitoringRunner"; + let scriptBody = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); + let scriptName = "${env.name}-${globals.scriptSufix}"; var resp = api.dev.scripting.GetScript(appid, session, scriptName); if (resp.result == Response.OK) { @@ -107,7 +110,7 @@ actions: createMonitoringSchedule: script: | - let scriptName = "${env.name}-dbMonitoringRunner"; + let scriptName = "${env.name}-${globals.scriptSufix}"; let trigger = "cron:0 0/${settings.monitorInterval} * ? * * *"; var tasks = api.utils.scheduler.GetTasks({ appid: appid, session: session }).objects; @@ -129,7 +132,7 @@ actions: removeMonitoringSchedule: script: | - let scriptName = "${env.name}-dbMonitoringRunner"; + let scriptName = "${env.name}-${globals.scriptSufix}"; let tasks = api.utils.scheduler.GetTasks({ appid: appid, session: session }).objects; for (var i = 0, l = tasks.length; i < l; i++) { if (tasks[i].script == scriptName) { @@ -140,7 +143,7 @@ actions: removeMonitoringScript: script: | - let scriptName = "${env.name}-dbMonitoringRunner"; + let scriptName = "${env.name}-${globals.scriptSufix}"; var resp = api.dev.scripting.GetScript(appid, session, scriptName); if (resp.result == com.hivext.api.Response.OK) { api.dev.scripting.DeleteScript(appid, session, scriptName); @@ -149,7 +152,7 @@ actions: removeMonitoringBinary: - cmd[sqldb]: |- - rm -f /usr/local/sbin/db-monitoring.sh /etc/cron.d/db-monitoring || true + rm -f /usr/local/sbin/db-monitoring.sh || true user: root configure: diff --git a/addons/monitoring/scripts/monitoring-runner.js b/addons/monitoring/scripts/db-monitoring.js similarity index 100% rename from addons/monitoring/scripts/monitoring-runner.js rename to addons/monitoring/scripts/db-monitoring.js From 3c0e918666dc799168e3508fbcddad2ce1faf9c8 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Wed, 5 Nov 2025 15:36:28 +0100 Subject: [PATCH 06/21] JE-66040 --- addons/monitoring/manifest.yml | 15 +- addons/monitoring/scripts/db-monitoring.sh | 248 ++++++++++----------- 2 files changed, 125 insertions(+), 138 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 67d68bc7..dae483e3 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -45,6 +45,16 @@ settings: caption: Every 50 minutes default: 10 + - name: user + caption: DB User + type: string + required: true + - name: password + caption: DB Password + type: string + inputType: password + required: true + buttons: - name: Configure caption: Configure @@ -65,7 +75,7 @@ onAfterClone: monitorInterval: ${settings.monitorInterval} onInstall: - - if (!${settings.install:false}): init + - getReplicaUser - downloadMonitoringScript - createMonitoringScript - createMonitoringSchedule @@ -81,9 +91,6 @@ onBeforeDelete: - removeMonitoringBinary actions: - init: - log: Monitoring is initialized - downloadMonitoringScript: - cmd[sqldb]: |- curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index f4d09435..732f51d0 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -1,190 +1,170 @@ #!/bin/bash - -DB_USER="" -DB_PASSWORD="" +USER_SESSION="$1" +USER_EMAIL="$2" +THRESHOLD=70 MONITORING_LOG=/var/log/db-monitoring.log STATUS_FILE=/var/tmp/db-monitoring.status -BODY_ERROR_PREFIX="MySQL/MariaDB monitoring error on ${ENV_NAME}" - - -# Accept USER_SESSION and USER_EMAIL via positional args if provided -if [ -n "$1" ]; then USER_SESSION="$1"; fi -if [ -n "$2" ]; then USER_EMAIL="$2"; fi - -# email notification via Jelastic API +HOSTNAME_SHORT=$(hostname -s 2>/dev/null || hostname) +BODY_ERROR_PREFIX="DataBase monitoring error on ${HOSTNAME_SHORT}" +# email notification via Virtuozzo API function sendEmailNotification(){ if [ -e "/usr/lib/jelastic/modules/api.module" ]; then [ -e "/var/run/jem.pid" ] && return 0 - CURRENT_PLATFORM_MAJOR_VERSION=$(jem api apicall -s --connect-timeout 3 --max-time 15 [API_DOMAIN]/1.0/statistic/system/rest/getversion 2>/dev/null | jq .version | grep -o [0-9.]* | awk -F . '{print $1}') - if [ "${CURRENT_PLATFORM_MAJOR_VERSION}" -ge "7" ]; then - echo $(date) ${ENV_NAME} "Sending e-mail notification about high DB connections usage" | tee -a $MONITORING_LOG - SUBJECT="${ENV_NAME}: MySQL connections usage reached threshold" - BODY="$1" - jem api apicall -s --connect-timeout 3 --max-time 15 [API_DOMAIN]/1.0/message/email/rest/send --data-urlencode "session=$USER_SESSION" --data-urlencode "to=$USER_EMAIL" --data-urlencode "subject=$SUBJECT" --data-urlencode "body=$BODY" - if [[ $? != 0 ]]; then - echo $(date) ${ENV_NAME} "Sending of e-mail notification failed" | tee -a $MONITORING_LOG - else - echo $(date) ${ENV_NAME} "E-mail notification is sent successfully" | tee -a $MONITORING_LOG - fi - elif [ -z "${CURRENT_PLATFORM_MAJOR_VERSION}" ]; then - echo $(date) ${ENV_NAME} "Error when checking the platform version" | tee -a $MONITORING_LOG + echo $(date) ${HOSTNAME_SHORT} "Sending e-mail notification about high DB connections usage" | tee -a $MONITORING_LOG + SUBJECT="${HOSTNAME_SHORT}: MySQL connections usage reached threshold" + BODY="$1" + jem api apicall -s --connect-timeout 3 --max-time 15 [API_DOMAIN]/1.0/message/email/rest/send \ + --data-urlencode "session=$USER_SESSION" \ + --data-urlencode "to=$USER_EMAIL" \ + --data-urlencode "subject=$SUBJECT" \ + --data-urlencode body@- <<< "$BODY" + if [[ $? != 0 ]]; then + echo $(date) ${HOSTNAME_SHORT} "Sending of e-mail notification failed" | tee -a $MONITORING_LOG else - echo $(date) ${ENV_NAME} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG + echo $(date) ${HOSTNAME_SHORT} "E-mail notification is sent successfully" | tee -a $MONITORING_LOG fi else - echo $(date) ${ENV_NAME} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG + echo $(date) ${HOSTNAME_SHORT} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG fi } # status helpers: send email once per status change -get_last_status(){ +function get_last_status(){ [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" 2>/dev/null || echo "" } -set_status(){ +function set_status(){ echo "$1" > "$STATUS_FILE" 2>/dev/null || true } -send_on_status_change(){ +# Build reusable metrics body +function build_metrics_body(){ + local title="$1" + cat < +Database connections ${title} on ${HOSTNAME_SHORT}
+
+Status
+Uptime: $UPTIME_HUMAN
+Threads: $THREADS
+Questions: $QUESTIONS
+Slow queries: $SLOW
+Opens: $OPENS
+Open tables: $OPEN_TABLES
+QPS: $QPS
+
+max_connections: $MAX_CONNECTIONS
+Current threads (connections): $THREADS
+Usage: ${USAGE_PCT}%
+Timestamp: $(date) + +EOF +} + +function send_on_status_change(){ local new_status="$1" - shift - local body="$*" + local body="$2" local last_status="$(get_last_status)" if [ "$new_status" != "$last_status" ]; then sendEmailNotification "$body" set_status "$new_status" else - echo "$(date) ${ENV_NAME} Status '$new_status' unchanged, skipping email" >> $MONITORING_LOG + echo "$(date) ${HOSTNAME_SHORT} Status '$new_status' unchanged, skipping email" >> $MONITORING_LOG fi } -echo "Monitoring started at $(date)" >> $MONITORING_LOG - -# Read credentials from /.jelenv (REPLICA_USER/REPLICA_PSWD) only; if absent -> notify and exit -if [ ! -f "/.jelenv" ]; then - BODY="${BODY_ERROR_PREFIX} - -Issue: Credentials file /.jelenv not found -Action required: Ensure REPLICA_USER/REPLICA_PSWD are provisioned in /.jelenv -Timestamp: $(date)" - echo "$BODY" >> $MONITORING_LOG - send_on_status_change "CREDENTIALS_MISSING" "$BODY" - echo "Monitoring finished at $(date)" >> $MONITORING_LOG - exit 1 -fi - -source "/.jelenv" - -DB_USER="$REPLICA_USER" -DB_PASSWORD="$REPLICA_PSWD" +# credentials check and load +function check_credentials(){ + source "/.jelenv" + DB_USER="$REPLICA_USER" + DB_PASSWORD="$REPLICA_PSWD" -if [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then - BODY="${BODY_ERROR_PREFIX} + if [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then + BODY="${BODY_ERROR_PREFIX} -Issue: Missing REPLICA_USER or REPLICA_PSWD in /.jelenv +Issue: Missing REPLICA_USER or REPLICA_PSWD in environment variables Observed values: REPLICA_USER='${REPLICA_USER:-EMPTY}', REPLICA_PSWD='${REPLICA_PSWD:+SET}' -Action required: Populate both variables in /.jelenv +Action required: Populate both variables in environment variables Timestamp: $(date)" - echo "$BODY" >> $MONITORING_LOG - send_on_status_change "CREDENTIALS_MISSING" "$BODY" - echo "Monitoring finished at $(date)" >> $MONITORING_LOG - exit 1 -fi + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "CREDENTIALS_MISSING" "$BODY" + echo "Monitoring finished at $(date)" >> $MONITORING_LOG + exit 1 + fi +} -# Collect metrics using mysqladmin status -STATUS_RAW=$(mysqladmin status -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) -RET=$? -if [ $RET -ne 0 ] || [ -z "$STATUS_RAW" ]; then - BODY="${BODY_ERROR_PREFIX} +# collect DB metrics using mysqladmin and mysql client +function collect_metrics(){ + STATUS_RAW=$(mysqladmin status -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) + RET=$? + if [ $RET -ne 0 ] || [ -z "$STATUS_RAW" ]; then + BODY="${BODY_ERROR_PREFIX} Action: mysqladmin status Exit code: $RET Output:\n$STATUS_RAW Timestamp: $(date)" - echo "$BODY" >> $MONITORING_LOG - send_on_status_change "STATUS_ERROR" "$BODY" - echo "Monitoring finished at $(date)" >> $MONITORING_LOG - exit 1 -fi -STATUS="$STATUS_RAW" - -# Example STATUS: "Uptime: 12345 Threads: 12 Questions: 34567 Slow queries: 0 Opens: 132 Flush tables: 1 Open tables: 64 Queries per second avg: 2.80" -UPTIME=$(echo "$STATUS" | awk -F'Uptime: | ' '{print $2}') -THREADS=$(echo "$STATUS" | awk -F'Threads: | ' '{print $3}') -QUESTIONS=$(echo "$STATUS" | awk -F'Questions: | ' '{print $4}') -SLOW=$(echo "$STATUS" | awk -F'Slow queries: | ' '{print $5}') -OPENS=$(echo "$STATUS" | awk -F'Opens: | ' '{print $6}') -FLUSHES=$(echo "$STATUS" | awk -F'Flush tables: | ' '{print $7}') -OPEN_TABLES=$(echo "$STATUS" | awk -F'Open tables: | ' '{print $8}') -QPS=$(echo "$STATUS" | awk -F'Queries per second avg: ' '{print $2}') - -# Get max_connections (prefer mysqladmin variables to stay within mysqladmin tooling) -VARS_RAW=$(mysqladmin variables -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) -VARS_RC=$? -if [ $VARS_RC -eq 0 ]; then - MAX_CONNECTIONS=$(echo "$VARS_RAW" | awk -F'|' '/max_connections/ {gsub(/ /,"",$0); print $3; exit}') -fi + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "STATUS_ERROR" "$BODY" + echo "Monitoring finished at $(date)" >> $MONITORING_LOG + exit 1 + fi + STATUS="$STATUS_RAW" + + UPTIME=$(echo "$STATUS" | grep -o 'Uptime: [0-9]\+' | awk '{print $2}') + THREADS=$(echo "$STATUS" | grep -o 'Threads: [0-9]\+' | awk '{print $2}') + QUESTIONS=$(echo "$STATUS" | grep -o 'Questions: [0-9]\+' | awk '{print $2}') + SLOW=$(echo "$STATUS" | grep -o 'Slow queries: [0-9]\+' | awk '{print $3}') + OPENS=$(echo "$STATUS" | grep -o 'Opens: [0-9]\+' | awk '{print $2}') + FLUSHES=$(echo "$STATUS" | grep -o 'Flush tables: [0-9]\+' | awk '{print $3}') + OPEN_TABLES=$(echo "$STATUS" | grep -o 'Open tables: [0-9]\+' | awk '{print $3}') + QPS=$(echo "$STATUS" | sed -n 's/.*Queries per second avg: \([0-9.]*\).*/\1/p') + + UPTIME_HUMAN="$UPTIME" + if [[ "$UPTIME" =~ ^[0-9]+$ ]]; then + D=$((UPTIME/86400)) + H=$(((UPTIME%86400)/3600)) + M=$(((UPTIME%3600)/60)) + UPTIME_HUMAN="${D} days ${H} hours ${M} minutes" + fi -if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then - # fallback to mysql client if parsing failed or command failed - FALLBACK_RAW=$(mysql -Nse "SHOW VARIABLES LIKE 'max_connections';" -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) - FALLBACK_RC=$? - if [ $FALLBACK_RC -eq 0 ]; then - MAX_CONNECTIONS=$(echo "$FALLBACK_RAW" | awk '{print $2}') + VAR_RAW=$(mysql -Nse "SHOW VARIABLES LIKE 'max_connections';" -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) + VAR_RC=$? + if [ $VAR_RC -eq 0 ]; then + MAX_CONNECTIONS=$(echo "$VAR_RAW" | awk '{print $2}') fi -fi -if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then - BODY="${BODY_ERROR_PREFIX} + if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then + BODY="${BODY_ERROR_PREFIX} Issue: Unable to determine max_connections -mysqladmin variables exit: $VARS_RC -mysqladmin variables output:\n$VARS_RAW -mysql fallback exit: ${FALLBACK_RC:-not_executed} -mysql fallback output:\n${FALLBACK_RAW:-N/A} +mysql SHOW VARIABLES exit: ${VAR_RC} +mysql SHOW VARIABLES output:\n${VAR_RAW} Timestamp: $(date)" - echo "$BODY" >> $MONITORING_LOG - send_on_status_change "MAXCONN_ERROR" "$BODY" - # continue with MAX_CONNECTIONS=0 to avoid division errors - MAX_CONNECTIONS=0 -fi + echo "$BODY" >> $MONITORING_LOG + send_on_status_change "MAXCONN_ERROR" "$BODY" + # continue with MAX_CONNECTIONS=0 to avoid division errors + MAX_CONNECTIONS=0 + fi -# Calculate usage percentage (integer) -USAGE_PCT=0 -if [ "$MAX_CONNECTIONS" -gt 0 ]; then - USAGE_PCT=$(awk -v th="$THREADS" -v max="$MAX_CONNECTIONS" 'BEGIN { if (max>0) printf("%d", (th*100)/max); else print 0 }') -fi + USAGE_PCT=0 + if [ "$MAX_CONNECTIONS" -gt 0 ]; then + USAGE_PCT=$(awk -v th="$THREADS" -v max="$MAX_CONNECTIONS" 'BEGIN { if (max>0) printf("%d", (th*100)/max); else print 0 }') + fi +} -# Compose metrics body -METRICS_BODY="MySQL/MariaDB connections usage alert on ${ENV_NAME} - -Status: $STATUS -max_connections: $MAX_CONNECTIONS -Current threads (connections): $THREADS -Usage: ${USAGE_PCT}% -Uptime: $UPTIME sec -Questions: $QUESTIONS -Slow queries: $SLOW -Opens: $OPENS -Flush tables: $FLUSHES -Open tables: $OPEN_TABLES -QPS (avg): $QPS -Timestamp: $(date)" +echo "Monitoring started at $(date)" >> $MONITORING_LOG +check_credentials +collect_metrics +METRICS_BODY=$(build_metrics_body "usage alert") echo "$METRICS_BODY" >> $MONITORING_LOG # Determine status and send only on change -THRESHOLD=70 if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then send_on_status_change "THRESHOLD" "$METRICS_BODY" else - OK_BODY="MySQL/MariaDB connections back to normal on ${ENV_NAME} - -Status: $STATUS -max_connections: $MAX_CONNECTIONS -Current threads (connections): $THREADS -Usage: ${USAGE_PCT}% -Timestamp: $(date)" + OK_BODY=$(build_metrics_body "back to normal") send_on_status_change "OK" "$OK_BODY" fi From 93b840235d37db3f8217faf7354e7bbf9bbf3119 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Wed, 5 Nov 2025 15:52:35 +0100 Subject: [PATCH 07/21] JE-66040 --- addons/monitoring/manifest.yml | 73 +++++++++++++--------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index dae483e3..38c8b086 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -76,91 +76,74 @@ onAfterClone: onInstall: - getReplicaUser - - downloadMonitoringScript - - createMonitoringScript - - createMonitoringSchedule + - applyMonitoring onUninstall: - - removeMonitoringSchedule - - removeMonitoringScript - - removeMonitoringBinary + - cleanupMonitoring onBeforeDelete: - - removeMonitoringSchedule - - removeMonitoringScript - - removeMonitoringBinary + - cleanupMonitoring actions: - downloadMonitoringScript: - - cmd[sqldb]: |- - curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh - user: root - - createMonitoringScript: + applyMonitoring: script: | - let Response = com.hivext.api.Response; + let envName = "${env.name}"; + let scriptName = envName + "-${globals.scriptSufix}"; let Transport = com.hivext.api.core.utils.Transport; - let scriptBody = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); - let scriptName = "${env.name}-${globals.scriptSufix}"; + let Response = com.hivext.api.Response; + + var downloadCmd = "curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh"; + var cmdResp = api.env.control.ExecCmdByGroup(envName, session, "sqldb", toJSON([{command: downloadCmd}]), true, false, "root"); + if (cmdResp.result != 0) return cmdResp; + var body = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); var resp = api.dev.scripting.GetScript(appid, session, scriptName); if (resp.result == Response.OK) { api.dev.scripting.DeleteScript(appid, session, scriptName); } - - resp = api.dev.scripting.CreateScript(appid, session, scriptName, "js", scriptBody); + resp = api.dev.scripting.CreateScript(appid, session, scriptName, "js", body); if (resp.result != 0) return resp; java.lang.Thread.sleep(500); resp = api.dev.scripting.Build(appid, session, scriptName); if (resp.result != 0) return resp; - return { result: 0 }; - - createMonitoringSchedule: - script: | - let scriptName = "${env.name}-${globals.scriptSufix}"; - let trigger = "cron:0 0/${settings.monitorInterval} * ? * * *"; - var tasks = api.utils.scheduler.GetTasks({ appid: appid, session: session }).objects; - for (var i = 0, l = tasks.length; i < l; i++) { + var tasks = api.utils.scheduler.GetTasks({appid: appid, session: session}).objects; + for (var i=0; i Date: Wed, 5 Nov 2025 16:44:52 +0100 Subject: [PATCH 08/21] JE-66040 --- addons/monitoring/scripts/db-monitoring.sh | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 732f51d0..631af863 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -24,11 +24,10 @@ function sendEmailNotification(){ echo $(date) ${HOSTNAME_SHORT} "E-mail notification is sent successfully" | tee -a $MONITORING_LOG fi else - echo $(date) ${HOSTNAME_SHORT} "Email notification is not sent because this functionality is unavailable for current platform version." | tee -a $MONITORING_LOG + echo $(date) ${HOSTNAME_SHORT} "Email notification is not sent because this functionality is unavailable for current platform." | tee -a $MONITORING_LOG fi } -# status helpers: send email once per status change function get_last_status(){ [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" 2>/dev/null || echo "" } @@ -44,17 +43,15 @@ function build_metrics_body(){
Database connections ${title} on ${HOSTNAME_SHORT}

-Status
+STATUS
Uptime: $UPTIME_HUMAN
Threads: $THREADS
-Questions: $QUESTIONS
Slow queries: $SLOW
-Opens: $OPENS
Open tables: $OPEN_TABLES
QPS: $QPS

-max_connections: $MAX_CONNECTIONS
-Current threads (connections): $THREADS
+Max connections: $MAX_CONNECTIONS
+Current connections: $THREADS
Usage: ${USAGE_PCT}%
Timestamp: $(date)
@@ -143,7 +140,6 @@ mysql SHOW VARIABLES output:\n${VAR_RAW} Timestamp: $(date)" echo "$BODY" >> $MONITORING_LOG send_on_status_change "MAXCONN_ERROR" "$BODY" - # continue with MAX_CONNECTIONS=0 to avoid division errors MAX_CONNECTIONS=0 fi @@ -153,12 +149,10 @@ Timestamp: $(date)" fi } - echo "Monitoring started at $(date)" >> $MONITORING_LOG check_credentials collect_metrics METRICS_BODY=$(build_metrics_body "usage alert") -echo "$METRICS_BODY" >> $MONITORING_LOG # Determine status and send only on change if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then From 6780416a8f264d1643f0ca92e26ce7a3de1d1a17 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Wed, 5 Nov 2025 16:48:56 +0100 Subject: [PATCH 09/21] JE-66040 --- addons/monitoring/scripts/db-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 631af863..3b0f3064 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -48,7 +48,7 @@ function build_metrics_body(){ Threads: $THREADS
Slow queries: $SLOW
Open tables: $OPEN_TABLES
-QPS: $QPS
+Queries per second avg: $QPS

Max connections: $MAX_CONNECTIONS
Current connections: $THREADS
From db44e8199a004d4cb1269a1eb353a657b6e5d39c Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Tue, 11 Nov 2025 11:48:38 +0200 Subject: [PATCH 10/21] JE-66040 --- addons/monitoring/manifest.yml | 6 ++++-- addons/monitoring/scripts/db-monitoring.sh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 38c8b086..e6e754f8 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -3,8 +3,8 @@ name: Database Monitoring id: db-monitoring description: - text: The Database Monitoring add-on performs a comprehensive monitoring of your MySQL/MariaDB/Percona Database to detect data corruption and inconsistency in the components. Add-on will **temporarily stop all database services** for the duration of the diagnostic to ensure accurate results. Detected issues will be listed in the recovery log. - short: The add-on checks your database for corrupted or inconsistent data in the components. + text: The Database Monitoring add-on tracks MySQL/MariaDB/Percona connection usage and sends single-on-state-change email alerts. It periodically collects mysqladmin status and max_connections, alerts when usage reaches 70%, and includes detailed metrics with human‑readable uptime. The check runs online without stopping database services and supports configurable intervals. + short: Email alerts for DB connection usage with detailed metrics. logo: /images/database-monitoring.png @@ -29,6 +29,8 @@ settings: type: list editable: false values: + - value: 2 + caption: Every 2 minutes - value: 5 caption: Every 5 minutes - value: 10 diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 3b0f3064..dd852401 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -33,7 +33,8 @@ function get_last_status(){ } function set_status(){ - echo "$1" > "$STATUS_FILE" 2>/dev/null || true + local status="$1" + echo "$status" > "$STATUS_FILE" 2>/dev/null || true } # Build reusable metrics body @@ -154,7 +155,6 @@ check_credentials collect_metrics METRICS_BODY=$(build_metrics_body "usage alert") -# Determine status and send only on change if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then send_on_status_change "THRESHOLD" "$METRICS_BODY" else From dcca9474ca49eaf8415a0a56fb0d9af0e438ba5c Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Tue, 11 Nov 2025 13:16:05 +0200 Subject: [PATCH 11/21] JE-66040 --- addons/monitoring/README.md | 49 ++++++++++++++++++++ addons/monitoring/manifest.yml | 2 - addons/monitoring/scripts/db-monitoring.sh | 53 ++++++++++++++-------- 3 files changed, 84 insertions(+), 20 deletions(-) create mode 100644 addons/monitoring/README.md diff --git a/addons/monitoring/README.md b/addons/monitoring/README.md new file mode 100644 index 00000000..c4e31043 --- /dev/null +++ b/addons/monitoring/README.md @@ -0,0 +1,49 @@ +# Database Monitoring Add-On (Connections Usage Alerts) + +Detailed description of the add-on that monitors MySQL/MariaDB/Percona connections and sends email alerts. + +## Purpose +The add-on tracks database connection usage and sends an email alert when usage reaches 70% of `max_connections`. Alerts are stateful (sent once per state change) to avoid spam. Emails include detailed metrics. + +## Key features +- Monitors connection usage (Usage = Threads / max_connections). +- Default threshold: 70% (state `THRESHOLD`). +- Stateful notifications (one email per state change): + - `OK` — Usage < 70% (back to normal). + - `THRESHOLD` — Usage ≥ 70% (threshold exceeded). + - `STATUS_ERROR` — failed to get `mysqladmin status`. + - `MAXCONN_ERROR` — failed to get `max_connections` (`SHOW VARIABLES`). +- Metrics from `mysqladmin status` and `SHOW VARIABLES LIKE 'max_connections'`. +- HTML email: monospace block, bold labels, `
` line breaks, hostname in subject/body, uptime shown as “X days Y hours Z minutes”. +- Configurable schedule (Quartz cron) via add-on settings: 2, 5, 10, 15, 20, 30, 40, 50 minutes. +- Online check (does not stop database services). Runs on all `sqldb` nodes. + +## How it works +1. Install: + - Downloads `/usr/local/sbin/db-monitoring.sh` to `sqldb` nodes. + - Creates a script runner `db-monitoring.js` that executes the shell script on all `sqldb` nodes, passing `USER_SESSION` and `USER_EMAIL`. + - Creates a scheduler task with Quartz trigger `cron:0 0/N * ? * * *` (N is the chosen interval). +2. Runtime: + - Reads DB credentials from `/.jelenv`: `REPLICA_USER`/`REPLICA_PSWD`. + - Collects metrics: `mysqladmin status` and `SHOW VARIABLES LIKE 'max_connections'`. + - Calculates usage and determines state (OK/THRESHOLD/ERROR). + - Stores the last state in `/var/tmp/db-monitoring.status` and sends email only on state changes. + - Logs to `/var/log/db-monitoring.log`. + +## Email content and metrics +Emails are HTML with bold labels and `
` line breaks. Included: +- Status: + - Uptime — node uptime (days/hours/minutes). + - Threads — current number of active connections. + - Slow queries — number of slow queries. + - Open tables — tables currently open. + - Queries per second avg — average queries per second since start. +- max_connections — maximum concurrent connections. +- Current threads (connections) — current connections count. +- Usage — share of used connections (percent). +- Timestamp — report time. + +## Logs and artifacts +- Monitoring log: `/var/log/db-monitoring.log` (start/finish, email send, errors). +- State file: `/var/tmp/db-monitoring.status` (last state to suppress duplicate emails). + diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index e6e754f8..59036aa0 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -29,8 +29,6 @@ settings: type: list editable: false values: - - value: 2 - caption: Every 2 minutes - value: 5 caption: Every 5 minutes - value: 10 diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index dd852401..dccc2087 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -78,12 +78,17 @@ function check_credentials(){ DB_PASSWORD="$REPLICA_PSWD" if [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then - BODY="${BODY_ERROR_PREFIX} - -Issue: Missing REPLICA_USER or REPLICA_PSWD in environment variables -Observed values: REPLICA_USER='${REPLICA_USER:-EMPTY}', REPLICA_PSWD='${REPLICA_PSWD:+SET}' -Action required: Populate both variables in environment variables -Timestamp: $(date)" + BODY=$(cat < +${BODY_ERROR_PREFIX}
+
+Issue: Missing REPLICA_USER or REPLICA_PSWD in environment variables
+Observed values: REPLICA_USER='${REPLICA_USER:-EMPTY}', REPLICA_PSWD='${REPLICA_PSWD:+SET}'
+Action required: Populate both variables in environment variables
+Timestamp: $(date) + +EOF +) echo "$BODY" >> $MONITORING_LOG send_on_status_change "CREDENTIALS_MISSING" "$BODY" echo "Monitoring finished at $(date)" >> $MONITORING_LOG @@ -96,12 +101,18 @@ function collect_metrics(){ STATUS_RAW=$(mysqladmin status -u"$DB_USER" -p"$DB_PASSWORD" 2>&1) RET=$? if [ $RET -ne 0 ] || [ -z "$STATUS_RAW" ]; then - BODY="${BODY_ERROR_PREFIX} - -Action: mysqladmin status -Exit code: $RET -Output:\n$STATUS_RAW -Timestamp: $(date)" + OUT_ESC=$(printf '%s' "$STATUS_RAW" | sed -e 's/&/\&/g' -e 's//\>/g' | sed ':a;N;$!ba;s/\n//g') + BODY=$(cat < +${BODY_ERROR_PREFIX}
+
+Action: mysqladmin status
+Exit code: $RET
+Output: $OUT_ESC
+Timestamp: $(date) + +EOF +) echo "$BODY" >> $MONITORING_LOG send_on_status_change "STATUS_ERROR" "$BODY" echo "Monitoring finished at $(date)" >> $MONITORING_LOG @@ -133,12 +144,18 @@ Timestamp: $(date)" fi if ! [[ "$MAX_CONNECTIONS" =~ ^[0-9]+$ ]]; then - BODY="${BODY_ERROR_PREFIX} - -Issue: Unable to determine max_connections -mysql SHOW VARIABLES exit: ${VAR_RC} -mysql SHOW VARIABLES output:\n${VAR_RAW} -Timestamp: $(date)" + VAR_ESC=$(printf '%s' "$VAR_RAW" | sed -e 's/&/\&/g' -e 's//\>/g' | sed ':a;N;$!ba;s/\n//g') + BODY=$(cat < +${BODY_ERROR_PREFIX}
+
+Issue: Unable to determine max_connections
+mysql SHOW VARIABLES exit: ${VAR_RC}
+mysql SHOW VARIABLES output: ${VAR_ESC}
+Timestamp: $(date) + +EOF +) echo "$BODY" >> $MONITORING_LOG send_on_status_change "MAXCONN_ERROR" "$BODY" MAX_CONNECTIONS=0 From 5542a46b66db10efe20d50c67889c41c85022c96 Mon Sep 17 00:00:00 2001 From: Henadii Sychevskyi Date: Tue, 11 Nov 2025 13:21:46 +0200 Subject: [PATCH 12/21] Update README.md --- addons/monitoring/README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/addons/monitoring/README.md b/addons/monitoring/README.md index c4e31043..3f5da3d4 100644 --- a/addons/monitoring/README.md +++ b/addons/monitoring/README.md @@ -1,22 +1,17 @@ -# Database Monitoring Add-On (Connections Usage Alerts) - -Detailed description of the add-on that monitors MySQL/MariaDB/Percona connections and sends email alerts. - +# Database Monitoring Add-On ## Purpose The add-on tracks database connection usage and sends an email alert when usage reaches 70% of `max_connections`. Alerts are stateful (sent once per state change) to avoid spam. Emails include detailed metrics. ## Key features - Monitors connection usage (Usage = Threads / max_connections). -- Default threshold: 70% (state `THRESHOLD`). - Stateful notifications (one email per state change): - `OK` — Usage < 70% (back to normal). - `THRESHOLD` — Usage ≥ 70% (threshold exceeded). - `STATUS_ERROR` — failed to get `mysqladmin status`. - `MAXCONN_ERROR` — failed to get `max_connections` (`SHOW VARIABLES`). - Metrics from `mysqladmin status` and `SHOW VARIABLES LIKE 'max_connections'`. -- HTML email: monospace block, bold labels, `
` line breaks, hostname in subject/body, uptime shown as “X days Y hours Z minutes”. -- Configurable schedule (Quartz cron) via add-on settings: 2, 5, 10, 15, 20, 30, 40, 50 minutes. -- Online check (does not stop database services). Runs on all `sqldb` nodes. +- Configurable schedule (Quartz cron) via add-on settings: 5, 10, 15, 20, 30, 40, 50 minutes. +- Runs on all `sqldb` nodes. ## How it works 1. Install: From ec186cbc40cf96d34511037002e559fd103b9c55 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Fri, 21 Nov 2025 15:20:00 +0100 Subject: [PATCH 13/21] JE-66040 --- addons/monitoring/manifest.yml | 90 +++++++++--------- addons/monitoring/scripts/db-monitoring.js | 2 +- addons/monitoring/scripts/db-monitoring.sh | 105 ++++++++++++++++++--- 3 files changed, 140 insertions(+), 57 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 59036aa0..92c4be38 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -65,6 +65,7 @@ buttons: globals: scriptSufix: db-monitoring + scriptName: ${env.name}-${globals.scriptSufix} onAfterClone: install: ${baseUrl}/manifest.yml?_r=${fn.random} @@ -76,7 +77,10 @@ onAfterClone: onInstall: - getReplicaUser - - applyMonitoring + - downloadScripts + - createScript + - addSchedulerToDB + - setSchedulerTimeout onUninstall: - cleanupMonitoring @@ -84,66 +88,66 @@ onUninstall: onBeforeDelete: - cleanupMonitoring +onCustomNodeEvent [name:executeScript]: + script: | + let URL = "${platformUrl}${globals.scriptName}?appid=" + appid + "&token=${globals.random}&envName=${env.name}&uid=${user.uid}&session=" + session; + let Transport = com.hivext.api.core.utils.Transport; + let resp = new Transport().get(URL); + resp = JSON.parse(resp); + if (resp.response && resp.response != 0) { + return new Transport().get("${platformUrl}/1.0/environment/jerror/rest/jerror?appid=" + appid + "&actionname=db-monitoring&callparameters=" + URL + "&email=${user.email}&errorcode=4121&errormessage=" + encodeURIComponent(resp.response.message) + "&priority=high"); + return resp.response; + } + return { result: 0 } + actions: - applyMonitoring: + downloadScripts: + - cmd[sqldb]: |- + curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh + chmod +x /usr/local/sbin/db-monitoring.sh + [ -f /var/log/db-monitoring.log ] || : > /var/log/db-monitoring.log + user: root + createScript: script: | - let envName = "${env.name}"; - let scriptName = envName + "-${globals.scriptSufix}"; - let Transport = com.hivext.api.core.utils.Transport; let Response = com.hivext.api.Response; + let Transport = com.hivext.api.core.utils.Transport; + let scriptBody = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); - var downloadCmd = "curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh && chmod +x /usr/local/sbin/db-monitoring.sh"; - var cmdResp = api.env.control.ExecCmdByGroup(envName, session, "sqldb", toJSON([{command: downloadCmd}]), true, false, "root"); - if (cmdResp.result != 0) return cmdResp; - - var body = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); - var resp = api.dev.scripting.GetScript(appid, session, scriptName); + var resp = api.dev.scripting.GetScript(appid, session, "${globals.scriptName}"); if (resp.result == Response.OK) { - api.dev.scripting.DeleteScript(appid, session, scriptName); + api.dev.scripting.DeleteScript(appid, session, "${globals.scriptName}"); } - resp = api.dev.scripting.CreateScript(appid, session, scriptName, "js", body); + resp = api.dev.scripting.CreateScript(appid, session, "${globals.scriptName}", "js", scriptBody); if (resp.result != 0) return resp; java.lang.Thread.sleep(500); - resp = api.dev.scripting.Build(appid, session, scriptName); + resp = api.dev.scripting.Build(appid, session, "${globals.scriptName}"); if (resp.result != 0) return resp; - var tasks = api.utils.scheduler.GetTasks({appid: appid, session: session}).objects; - for (var i=0; i/dev/null 2>&1 || true +} + function get_last_status(){ [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" 2>/dev/null || echo "" } @@ -61,10 +71,9 @@ EOF function send_on_status_change(){ local new_status="$1" - local body="$2" local last_status="$(get_last_status)" if [ "$new_status" != "$last_status" ]; then - sendEmailNotification "$body" + trigger_sendevent set_status "$new_status" else echo "$(date) ${HOSTNAME_SHORT} Status '$new_status' unchanged, skipping email" >> $MONITORING_LOG @@ -167,18 +176,88 @@ EOF fi } -echo "Monitoring started at $(date)" >> $MONITORING_LOG -check_credentials -collect_metrics -METRICS_BODY=$(build_metrics_body "usage alert") +# cron management: add scheduler and adjust interval +function addScheduler(){ + local cron_file="/etc/cron.d/db-monitoring" + echo "*/10 * * * * root /usr/local/sbin/db-monitoring.sh check >> /var/log/db-monitoring.log 2>&1" > "$cron_file" + chmod 0644 "$cron_file" + chown root:root "$cron_file" 2>/dev/null || true + (systemctl reload crond || service cron reload || service crond reload || true) >/dev/null 2>&1 + echo "$(date) ${HOSTNAME_SHORT} Cron installed at $cron_file" >> $MONITORING_LOG +} -if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then - send_on_status_change "THRESHOLD" "$METRICS_BODY" -else - OK_BODY=$(build_metrics_body "back to normal") - send_on_status_change "OK" "$OK_BODY" -fi +function setSchedulerTimeout(){ + local INTERVAL=10 + for i in "$@"; do + case $i in + --interval=*) + INTERVAL=${i#*=} + shift + shift + ;; + *) + ;; + esac + done + local cron_file="/etc/cron.d/db-monitoring" + [ -f "$cron_file" ] || addScheduler + # update minutes field to */INTERVAL + sed -ri "s|^[#]*[^ ]+ +\* +\* +\* +\* +root .*$|*/${INTERVAL} * * * * root /usr/local/sbin/db-monitoring.sh check >> /var/log/db-monitoring.log 2>\&1|" "$cron_file" + (systemctl reload crond || service cron reload || service crond reload || true) >/dev/null 2>&1 + echo "$(date) ${HOSTNAME_SHORT} Cron interval set to every ${INTERVAL} minutes" >> $MONITORING_LOG +} -echo "Monitoring finished at $(date)" >> $MONITORING_LOG +function check(){ + echo "Monitoring started at $(date)" >> $MONITORING_LOG + check_credentials + collect_metrics + if [ "$USAGE_PCT" -ge "$THRESHOLD" ]; then + send_on_status_change "THRESHOLD" + else + send_on_status_change "OK" + fi + echo "Monitoring finished at $(date)" >> $MONITORING_LOG +} + +function sendEmail(){ + local USER_SESSION="$1" + local USER_EMAIL="$2" + echo "Send email started at $(date)" >> $MONITORING_LOG + check_credentials + collect_metrics + local status="$(get_last_status)" + local title="usage alert" + if [ "$status" = "OK" ]; then + title="back to normal" + fi + local BODY="$(build_metrics_body "$title")" + sendEmailNotification "$BODY" + echo "Send email finished at $(date)" >> $MONITORING_LOG +} + +case "$1" in + addScheduler) + addScheduler + ;; + setSchedulerTimeout) + shift + setSchedulerTimeout "$@" + ;; + check) + check + ;; + sendEmail) + shift + sendEmail "$@" + ;; + *) + # Backward compatibility: if called with two args, send email directly + if [ -n "$USER_SESSION" ] && [ -n "$USER_EMAIL" ]; then + sendEmail "$USER_SESSION" "$USER_EMAIL" + else + echo "Usage: $0 {addScheduler|setSchedulerTimeout --interval=N|check|sendEmail USER_SESSION USER_EMAIL}" | tee -a $MONITORING_LOG + fi + ;; +esac exit 0 \ No newline at end of file From 99690ba9d88396d3ded99c05336ffebcc6492faa Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Fri, 21 Nov 2025 15:59:56 +0100 Subject: [PATCH 14/21] JE-66040 --- addons/monitoring/manifest.yml | 12 ++++++++---- addons/monitoring/scripts/db-monitoring.js | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 92c4be38..848919a5 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -101,12 +101,13 @@ onCustomNodeEvent [name:executeScript]: return { result: 0 } actions: + downloadScripts: - cmd[sqldb]: |- curl -fsSL ${baseUrl}/scripts/db-monitoring.sh -o /usr/local/sbin/db-monitoring.sh chmod +x /usr/local/sbin/db-monitoring.sh - [ -f /var/log/db-monitoring.log ] || : > /var/log/db-monitoring.log user: root + createScript: script: | let Response = com.hivext.api.Response; @@ -126,9 +127,11 @@ actions: let command = "sed -ri 's|^PLATFORM_DOMAIN=.*|PLATFORM_DOMAIN=\"${platformUrl}\"|g' /usr/local/sbin/db-monitoring.sh; " + "sed -ri 's|^USER_SCRIPT_PATH=.*|USER_SCRIPT_PATH=\"${platformUrl}${globals.scriptName}\"|g' /usr/local/sbin/db-monitoring.sh"; return api.env.control.ExecCmdByGroup("${env.name}", session, "sqldb", toJSON([{ command: command }]), true, false, "root"); + addSchedulerToDB: - cmd[sqldb]: bash /usr/local/sbin/db-monitoring.sh addScheduler user: root + setSchedulerTimeout: - cmd[sqldb]: bash /usr/local/sbin/db-monitoring.sh setSchedulerTimeout --interval=${settings.monitorInterval} user: root @@ -141,13 +144,14 @@ actions: api.dev.scripting.DeleteScript(appid, session, "${globals.scriptName}"); } - var rmResp = api.env.control.ExecCmdByGroup(envName, session, "sqldb", toJSON([ + resp = api.env.control.ExecCmdByGroup(envName, session, "sqldb", toJSON([ {command: "rm -f /etc/cron.d/db-monitoring || true"}, {command: "rm -f /usr/local/sbin/db-monitoring.sh || true"}, {command: "systemctl reload crond || service cron reload || true"} ]), true, false, "root"); - if (rmResp.result != 0) return rmResp; + if (resp.result != 0) return resp; + return { result: 0 }; configure: - - setSchedulerTimeout + - setSchedulerTimeout \ No newline at end of file diff --git a/addons/monitoring/scripts/db-monitoring.js b/addons/monitoring/scripts/db-monitoring.js index faef11b5..935d16d0 100644 --- a/addons/monitoring/scripts/db-monitoring.js +++ b/addons/monitoring/scripts/db-monitoring.js @@ -1,4 +1,4 @@ -//@auth +//@reg(envName, token, uid) var ROOT = "root"; var envName = getParam("envName", "${env.envName}"); From 90cffd9b46bdb9fc95bde83d6825674428386c0b Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Fri, 21 Nov 2025 16:10:08 +0100 Subject: [PATCH 15/21] JE-66040 --- addons/monitoring/manifest.yml | 8 +++++--- addons/monitoring/scripts/db-monitoring.js | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 848919a5..a597ed16 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -112,7 +112,9 @@ actions: script: | let Response = com.hivext.api.Response; let Transport = com.hivext.api.core.utils.Transport; + let StrSubstitutor = org.apache.commons.lang3.text.StrSubstitutor; let scriptBody = new Transport().get("${baseUrl}/scripts/db-monitoring.js"); + scriptBody = new StrSubstitutor({token: "${globals.random}"}, "${", "}").replace(scriptBody); var resp = api.dev.scripting.GetScript(appid, session, "${globals.scriptName}"); if (resp.result == Response.OK) { @@ -120,12 +122,12 @@ actions: } resp = api.dev.scripting.CreateScript(appid, session, "${globals.scriptName}", "js", scriptBody); if (resp.result != 0) return resp; - java.lang.Thread.sleep(500); + java.lang.Thread.sleep(1000); resp = api.dev.scripting.Build(appid, session, "${globals.scriptName}"); if (resp.result != 0) return resp; - let command = "sed -ri 's|^PLATFORM_DOMAIN=.*|PLATFORM_DOMAIN=\"${platformUrl}\"|g' /usr/local/sbin/db-monitoring.sh; " + - "sed -ri 's|^USER_SCRIPT_PATH=.*|USER_SCRIPT_PATH=\"${platformUrl}${globals.scriptName}\"|g' /usr/local/sbin/db-monitoring.sh"; + let command = "sed -ri 's|PLATFORM_DOMAIN=.*|PLATFORM_DOMAIN=\"${platformUrl}\"|g' /usr/local/sbin/db-monitoring.sh; " + + "sed -ri 's|USER_SCRIPT_PATH=.*|USER_SCRIPT_PATH=\"${platformUrl}${globals.scriptName}\"|g' /usr/local/sbin/db-monitoring.sh"; return api.env.control.ExecCmdByGroup("${env.name}", session, "sqldb", toJSON([{ command: command }]), true, false, "root"); addSchedulerToDB: diff --git a/addons/monitoring/scripts/db-monitoring.js b/addons/monitoring/scripts/db-monitoring.js index 935d16d0..ce2ff36b 100644 --- a/addons/monitoring/scripts/db-monitoring.js +++ b/addons/monitoring/scripts/db-monitoring.js @@ -2,8 +2,19 @@ var ROOT = "root"; var envName = getParam("envName", "${env.envName}"); +var Response = com.hivext.api.Response; function run() { + var tokenParam = String(getParam("token", "")).replace(/\s/g, ""); + if (!session && tokenParam != "${token}") { + return { + result: Response.PERMISSION_DENIED, + error: "wrong token", + type: "error", + message: "Token [" + tokenParam + "] does not match", + response: { result: Response.PERMISSION_DENIED } + }; + } var info = jelastic.env.control.GetEnvInfo(envName, session); if (info.result != 0) return info; From 1ef0a5b6bcb19c48a551f6cd26c3312c37e72314 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 24 Nov 2025 13:29:49 +0100 Subject: [PATCH 16/21] JE-66040 --- addons/monitoring/scripts/db-monitoring.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 8aeb6470..6ebe360e 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -127,16 +127,15 @@ EOF echo "Monitoring finished at $(date)" >> $MONITORING_LOG exit 1 fi - STATUS="$STATUS_RAW" - UPTIME=$(echo "$STATUS" | grep -o 'Uptime: [0-9]\+' | awk '{print $2}') - THREADS=$(echo "$STATUS" | grep -o 'Threads: [0-9]\+' | awk '{print $2}') - QUESTIONS=$(echo "$STATUS" | grep -o 'Questions: [0-9]\+' | awk '{print $2}') - SLOW=$(echo "$STATUS" | grep -o 'Slow queries: [0-9]\+' | awk '{print $3}') - OPENS=$(echo "$STATUS" | grep -o 'Opens: [0-9]\+' | awk '{print $2}') - FLUSHES=$(echo "$STATUS" | grep -o 'Flush tables: [0-9]\+' | awk '{print $3}') - OPEN_TABLES=$(echo "$STATUS" | grep -o 'Open tables: [0-9]\+' | awk '{print $3}') - QPS=$(echo "$STATUS" | sed -n 's/.*Queries per second avg: \([0-9.]*\).*/\1/p') + UPTIME=$(echo "$STATUS_RAW" | grep -o 'Uptime: [0-9]\+' | awk '{print $2}') + THREADS=$(echo "$STATUS_RAW" | grep -o 'Threads: [0-9]\+' | awk '{print $2}') + QUESTIONS=$(echo "$STATUS_RAW" | grep -o 'Questions: [0-9]\+' | awk '{print $2}') + SLOW=$(echo "$STATUS_RAW" | grep -o 'Slow queries: [0-9]\+' | awk '{print $3}') + OPENS=$(echo "$STATUS_RAW" | grep -o 'Opens: [0-9]\+' | awk '{print $2}') + FLUSHES=$(echo "$STATUS_RAW" | grep -o 'Flush tables: [0-9]\+' | awk '{print $3}') + OPEN_TABLES=$(echo "$STATUS_RAW" | grep -o 'Open tables: [0-9]\+' | awk '{print $3}') + QPS=$(echo "$STATUS_RAW" | sed -n 's/.*Queries per second avg: \([0-9.]*\).*/\1/p') UPTIME_HUMAN="$UPTIME" if [[ "$UPTIME" =~ ^[0-9]+$ ]]; then From c167a71291376fc8d9013058a2d17e12c2f12269 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 24 Nov 2025 13:42:24 +0100 Subject: [PATCH 17/21] JE-66040 --- addons/monitoring/manifest.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index a597ed16..7c2c9d65 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -66,6 +66,7 @@ buttons: globals: scriptSufix: db-monitoring scriptName: ${env.name}-${globals.scriptSufix} + random: ${fn.random} onAfterClone: install: ${baseUrl}/manifest.yml?_r=${fn.random} @@ -93,7 +94,11 @@ onCustomNodeEvent [name:executeScript]: let URL = "${platformUrl}${globals.scriptName}?appid=" + appid + "&token=${globals.random}&envName=${env.name}&uid=${user.uid}&session=" + session; let Transport = com.hivext.api.core.utils.Transport; let resp = new Transport().get(URL); - resp = JSON.parse(resp); + try { + resp = JSON.parse(resp); + } catch (e) { + return { result: 1704, error: "Invalid response from script endpoint", response: resp }; + } if (resp.response && resp.response != 0) { return new Transport().get("${platformUrl}/1.0/environment/jerror/rest/jerror?appid=" + appid + "&actionname=db-monitoring&callparameters=" + URL + "&email=${user.email}&errorcode=4121&errormessage=" + encodeURIComponent(resp.response.message) + "&priority=high"); return resp.response; From fff99912f7e1a67e57b7eaeee4cbac7cf07c5162 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 1 Dec 2025 14:14:27 +0100 Subject: [PATCH 18/21] JE-66040 --- addons/monitoring/manifest.yml | 17 ++++++----------- addons/monitoring/scripts/db-monitoring.js | 20 +++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 7c2c9d65..711b2963 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -91,19 +91,14 @@ onBeforeDelete: onCustomNodeEvent [name:executeScript]: script: | - let URL = "${platformUrl}${globals.scriptName}?appid=" + appid + "&token=${globals.random}&envName=${env.name}&uid=${user.uid}&session=" + session; + let URL = "${platformUrl}${globals.scriptName}?appid=" + appid + "&token=${globals.random}&envName=${env.name}&uid=${user.uid}&session=" + session + "&userEmail=${user.email}"; let Transport = com.hivext.api.core.utils.Transport; - let resp = new Transport().get(URL); - try { + resp = new Transport().get(URL); resp = JSON.parse(resp); - } catch (e) { - return { result: 1704, error: "Invalid response from script endpoint", response: resp }; - } - if (resp.response && resp.response != 0) { - return new Transport().get("${platformUrl}/1.0/environment/jerror/rest/jerror?appid=" + appid + "&actionname=db-monitoring&callparameters=" + URL + "&email=${user.email}&errorcode=4121&errormessage=" + encodeURIComponent(resp.response.message) + "&priority=high"); - return resp.response; - } - return { result: 0 } + if (resp.response && resp.response != 0) { + return new Transport().get("${platformUrl}/1.0/environment/jerror/rest/jerror?appid=" + appid + "&actionname=db-monitoring&callparameters=" + URL + "&email=${user.email}&errorcode=4121&errormessage=" + encodeURIComponent(resp.response.message) + "&priority=high"); + } + return { result: 0 } actions: diff --git a/addons/monitoring/scripts/db-monitoring.js b/addons/monitoring/scripts/db-monitoring.js index ce2ff36b..0a6273c9 100644 --- a/addons/monitoring/scripts/db-monitoring.js +++ b/addons/monitoring/scripts/db-monitoring.js @@ -3,6 +3,7 @@ var ROOT = "root"; var envName = getParam("envName", "${env.envName}"); var Response = com.hivext.api.Response; +var SQLDB = "sqldb"; function run() { var tokenParam = String(getParam("token", "")).replace(/\s/g, ""); @@ -18,20 +19,17 @@ function run() { var info = jelastic.env.control.GetEnvInfo(envName, session); if (info.result != 0) return info; - var nodes = info.nodes || [], node, resp; - var userEmail = user.email; - var userSession = session; - // pass USER_SESSION and USER_EMAIL as positional arguments + var resp; + var userEmail = getParam("userEmail", ""); + var userSession = getParam("session", session); + api.marketplace.console.WriteLog(appid, session, "DB Monitoring: sendEmail started for env " + envName); var command = "/usr/local/sbin/db-monitoring.sh sendEmail '" + userSession + "' '" + userEmail + "'"; - for (var i = 0, n = nodes.length; i < n; i++) { - node = nodes[i]; - if (node.nodeGroup == "sqldb") { - resp = jelastic.env.control.ExecCmdById(envName, session, node.id, toJSON([{ command: command }]), true, ROOT); - if (resp.result != 0) return resp; - } - } + // execute on all SQL DB nodes, analogous to promote-master.js style + resp = api.env.control.ExecCmdByGroup(envName, session, SQLDB, toJSON([{ command: command }]), true, false, ROOT); + if (resp.result != 0) return resp; + api.marketplace.console.WriteLog(appid, session, "DB Monitoring: sendEmail completed"); return { result: 0 }; } From 79b73730cb79d5b32c4f326fa4c86bb9ae4421b1 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 1 Dec 2025 14:27:54 +0100 Subject: [PATCH 19/21] JE-66040 --- addons/monitoring/README.md | 15 +++++++++++---- addons/monitoring/manifest.yml | 11 +++-------- addons/monitoring/scripts/db-monitoring.js | 1 - 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/addons/monitoring/README.md b/addons/monitoring/README.md index 3f5da3d4..ffdb9e21 100644 --- a/addons/monitoring/README.md +++ b/addons/monitoring/README.md @@ -11,18 +11,19 @@ The add-on tracks database connection usage and sends an email alert when usage - `MAXCONN_ERROR` — failed to get `max_connections` (`SHOW VARIABLES`). - Metrics from `mysqladmin status` and `SHOW VARIABLES LIKE 'max_connections'`. - Configurable schedule (Quartz cron) via add-on settings: 5, 10, 15, 20, 30, 40, 50 minutes. -- Runs on all `sqldb` nodes. +- Runs on all `sqldb` nodes (group execution). ## How it works 1. Install: - - Downloads `/usr/local/sbin/db-monitoring.sh` to `sqldb` nodes. - - Creates a script runner `db-monitoring.js` that executes the shell script on all `sqldb` nodes, passing `USER_SESSION` and `USER_EMAIL`. - - Creates a scheduler task with Quartz trigger `cron:0 0/N * ? * * *` (N is the chosen interval). + - Downloads `/usr/local/sbin/db-monitoring.sh` to all `sqldb` nodes. + - Creates a runner script `db-monitoring.js` that invokes the shell script on the `sqldb` group, passing `USER_SESSION` and `USER_EMAIL`. + - Installs a system cron job `/etc/cron.d/db-monitoring` and sets interval via `setSchedulerInterval` (every `N` minutes): `*/N * * * * root /usr/local/sbin/db-monitoring.sh check`. 2. Runtime: - Reads DB credentials from `/.jelenv`: `REPLICA_USER`/`REPLICA_PSWD`. - Collects metrics: `mysqladmin status` and `SHOW VARIABLES LIKE 'max_connections'`. - Calculates usage and determines state (OK/THRESHOLD/ERROR). - Stores the last state in `/var/tmp/db-monitoring.status` and sends email only on state changes. + - On state change, triggers platform event `onCustomNodeEvent [name:executeScript]`, which calls the runner script and sends the email. - Logs to `/var/log/db-monitoring.log`. ## Email content and metrics @@ -41,4 +42,10 @@ Emails are HTML with bold labels and `
` line breaks. Included: ## Logs and artifacts - Monitoring log: `/var/log/db-monitoring.log` (start/finish, email send, errors). - State file: `/var/tmp/db-monitoring.status` (last state to suppress duplicate emails). +- Cron: `/etc/cron.d/db-monitoring` (interval managed by the add-on). + +## Configuration +- Monitoring interval is controlled in the add-on settings (5/10/15/20/30/40/50 minutes). +- The add-on updates cron with `setSchedulerInterval` to `*/N * * * *`. +- Email sending uses platform messaging API and requires valid `session` and `userEmail`, which are passed by the add-on during event handling. diff --git a/addons/monitoring/manifest.yml b/addons/monitoring/manifest.yml index 711b2963..52e0d6ce 100644 --- a/addons/monitoring/manifest.yml +++ b/addons/monitoring/manifest.yml @@ -80,8 +80,7 @@ onInstall: - getReplicaUser - downloadScripts - createScript - - addSchedulerToDB - - setSchedulerTimeout + - setSchedulerInterval onUninstall: - cleanupMonitoring @@ -129,12 +128,8 @@ actions: let command = "sed -ri 's|PLATFORM_DOMAIN=.*|PLATFORM_DOMAIN=\"${platformUrl}\"|g' /usr/local/sbin/db-monitoring.sh; " + "sed -ri 's|USER_SCRIPT_PATH=.*|USER_SCRIPT_PATH=\"${platformUrl}${globals.scriptName}\"|g' /usr/local/sbin/db-monitoring.sh"; return api.env.control.ExecCmdByGroup("${env.name}", session, "sqldb", toJSON([{ command: command }]), true, false, "root"); - - addSchedulerToDB: - - cmd[sqldb]: bash /usr/local/sbin/db-monitoring.sh addScheduler - user: root - setSchedulerTimeout: + setSchedulerInterval: - cmd[sqldb]: bash /usr/local/sbin/db-monitoring.sh setSchedulerTimeout --interval=${settings.monitorInterval} user: root @@ -156,4 +151,4 @@ actions: return { result: 0 }; configure: - - setSchedulerTimeout \ No newline at end of file + - setSchedulerInterval \ No newline at end of file diff --git a/addons/monitoring/scripts/db-monitoring.js b/addons/monitoring/scripts/db-monitoring.js index 0a6273c9..00620678 100644 --- a/addons/monitoring/scripts/db-monitoring.js +++ b/addons/monitoring/scripts/db-monitoring.js @@ -25,7 +25,6 @@ function run() { api.marketplace.console.WriteLog(appid, session, "DB Monitoring: sendEmail started for env " + envName); var command = "/usr/local/sbin/db-monitoring.sh sendEmail '" + userSession + "' '" + userEmail + "'"; - // execute on all SQL DB nodes, analogous to promote-master.js style resp = api.env.control.ExecCmdByGroup(envName, session, SQLDB, toJSON([{ command: command }]), true, false, ROOT); if (resp.result != 0) return resp; From 44076dd028b4f9a010cdc7d44c9af2a1d33beaf8 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 1 Dec 2025 14:28:35 +0100 Subject: [PATCH 20/21] JE-66040 --- addons/monitoring/scripts/db-monitoring.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 6ebe360e..1a5bea45 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -185,7 +185,7 @@ function addScheduler(){ echo "$(date) ${HOSTNAME_SHORT} Cron installed at $cron_file" >> $MONITORING_LOG } -function setSchedulerTimeout(){ +function setSchedulerInterval(){ local INTERVAL=10 for i in "$@"; do case $i in @@ -235,12 +235,9 @@ function sendEmail(){ } case "$1" in - addScheduler) - addScheduler - ;; - setSchedulerTimeout) + setSchedulerInterval) shift - setSchedulerTimeout "$@" + setSchedulerInterval "$@" ;; check) check From 7083e2bc3b2b4d5cbf47b32e50c1d64648e541a6 Mon Sep 17 00:00:00 2001 From: Sychevskyi Henadii Date: Mon, 1 Dec 2025 14:46:47 +0100 Subject: [PATCH 21/21] JE-66040 --- addons/monitoring/scripts/db-monitoring.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/addons/monitoring/scripts/db-monitoring.sh b/addons/monitoring/scripts/db-monitoring.sh index 1a5bea45..4b380bf8 100644 --- a/addons/monitoring/scripts/db-monitoring.sh +++ b/addons/monitoring/scripts/db-monitoring.sh @@ -35,16 +35,16 @@ function trigger_sendevent(){ echo $(date) ${HOSTNAME_SHORT} "Trigger onCustomNodeEvent 'executeScript'" | tee -a $MONITORING_LOG curl -fsSL --max-time 10 --retry 1 --retry-max-time 15 \ --location --request POST "${PLATFORM_DOMAIN}1.0/environment/node/rest/sendevent" \ - --data-urlencode "params={'name': 'executeScript'}" >/dev/null 2>&1 || true + --data-urlencode "params={'name': 'executeScript'}" >/dev/null 2>&1 } function get_last_status(){ - [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" 2>/dev/null || echo "" + [ -f "$STATUS_FILE" ] && cat "$STATUS_FILE" || echo "" } function set_status(){ local status="$1" - echo "$status" > "$STATUS_FILE" 2>/dev/null || true + echo "$status" > "$STATUS_FILE" } # Build reusable metrics body @@ -180,8 +180,8 @@ function addScheduler(){ local cron_file="/etc/cron.d/db-monitoring" echo "*/10 * * * * root /usr/local/sbin/db-monitoring.sh check >> /var/log/db-monitoring.log 2>&1" > "$cron_file" chmod 0644 "$cron_file" - chown root:root "$cron_file" 2>/dev/null || true - (systemctl reload crond || service cron reload || service crond reload || true) >/dev/null 2>&1 + chown root:root "$cron_file" + systemctl reload crond echo "$(date) ${HOSTNAME_SHORT} Cron installed at $cron_file" >> $MONITORING_LOG } @@ -200,9 +200,8 @@ function setSchedulerInterval(){ done local cron_file="/etc/cron.d/db-monitoring" [ -f "$cron_file" ] || addScheduler - # update minutes field to */INTERVAL sed -ri "s|^[#]*[^ ]+ +\* +\* +\* +\* +root .*$|*/${INTERVAL} * * * * root /usr/local/sbin/db-monitoring.sh check >> /var/log/db-monitoring.log 2>\&1|" "$cron_file" - (systemctl reload crond || service cron reload || service crond reload || true) >/dev/null 2>&1 + systemctl reload crond echo "$(date) ${HOSTNAME_SHORT} Cron interval set to every ${INTERVAL} minutes" >> $MONITORING_LOG } @@ -247,12 +246,8 @@ case "$1" in sendEmail "$@" ;; *) - # Backward compatibility: if called with two args, send email directly - if [ -n "$USER_SESSION" ] && [ -n "$USER_EMAIL" ]; then - sendEmail "$USER_SESSION" "$USER_EMAIL" - else - echo "Usage: $0 {addScheduler|setSchedulerTimeout --interval=N|check|sendEmail USER_SESSION USER_EMAIL}" | tee -a $MONITORING_LOG - fi + echo "Usage: $0 {setSchedulerInterval --interval=N|check|sendEmail USER_SESSION USER_EMAIL}" | tee -a $MONITORING_LOG + exit 1 ;; esac