|
| 1 | +#!/usr/bin/python3 |
| 2 | + |
| 3 | +import argparse |
| 4 | +import datetime |
| 5 | +import glob |
| 6 | +import itertools |
| 7 | +import json |
| 8 | +import os |
| 9 | +import psutil |
| 10 | +import re |
| 11 | +import socket |
| 12 | +import subprocess |
| 13 | +import sys |
| 14 | +import pymysql |
| 15 | + |
| 16 | +# https://www.elastic.co/blog/found-crash-elasticsearch#mapping-explosion |
| 17 | + |
| 18 | + |
| 19 | +def tryint(value): |
| 20 | + try: |
| 21 | + return int(value) |
| 22 | + except (ValueError, TypeError): |
| 23 | + return value |
| 24 | + |
| 25 | + |
| 26 | +def get_service_stats(service): |
| 27 | + stats = {'MemoryCurrent': 0} |
| 28 | + output = subprocess.check_output(['/usr/bin/systemctl', 'show', service] + |
| 29 | + ['-p%s' % stat for stat in stats]) |
| 30 | + for line in output.decode().split('\n'): |
| 31 | + if not line: |
| 32 | + continue |
| 33 | + stat, val = line.split('=') |
| 34 | + stats[stat] = int(val) |
| 35 | + |
| 36 | + return stats |
| 37 | + |
| 38 | + |
| 39 | +def get_services_stats(): |
| 40 | + services = [os.path.basename(s) for s in |
| 41 | + glob.glob('/etc/systemd/system/devstack@*.service')] |
| 42 | + return [dict(service=service, **get_service_stats(service)) |
| 43 | + for service in services] |
| 44 | + |
| 45 | + |
| 46 | +def get_process_stats(proc): |
| 47 | + cmdline = proc.cmdline() |
| 48 | + if 'python' in cmdline[0]: |
| 49 | + cmdline = cmdline[1:] |
| 50 | + return {'cmd': cmdline[0], |
| 51 | + 'pid': proc.pid, |
| 52 | + 'args': ' '.join(cmdline[1:]), |
| 53 | + 'rss': proc.memory_info().rss} |
| 54 | + |
| 55 | + |
| 56 | +def get_processes_stats(matches): |
| 57 | + me = os.getpid() |
| 58 | + procs = psutil.process_iter() |
| 59 | + |
| 60 | + def proc_matches(proc): |
| 61 | + return me != proc.pid and any( |
| 62 | + re.search(match, ' '.join(proc.cmdline())) |
| 63 | + for match in matches) |
| 64 | + |
| 65 | + return [ |
| 66 | + get_process_stats(proc) |
| 67 | + for proc in procs |
| 68 | + if proc_matches(proc)] |
| 69 | + |
| 70 | + |
| 71 | +def get_db_stats(host, user, passwd): |
| 72 | + dbs = [] |
| 73 | + db = pymysql.connect(host=host, user=user, password=passwd, |
| 74 | + database='performance_schema', |
| 75 | + cursorclass=pymysql.cursors.DictCursor) |
| 76 | + with db: |
| 77 | + with db.cursor() as cur: |
| 78 | + cur.execute( |
| 79 | + 'SELECT COUNT(*) AS queries,current_schema AS db FROM ' |
| 80 | + 'events_statements_history_long GROUP BY current_schema') |
| 81 | + for row in cur: |
| 82 | + dbs.append({k: tryint(v) for k, v in row.items()}) |
| 83 | + return dbs |
| 84 | + |
| 85 | + |
| 86 | +def get_http_stats_for_log(logfile): |
| 87 | + stats = {} |
| 88 | + for line in open(logfile).readlines(): |
| 89 | + m = re.search('"([A-Z]+) /([^" ]+)( HTTP/1.1)?" ([0-9]{3}) ([0-9]+)', |
| 90 | + line) |
| 91 | + if m: |
| 92 | + method = m.group(1) |
| 93 | + path = m.group(2) |
| 94 | + status = m.group(4) |
| 95 | + size = int(m.group(5)) |
| 96 | + |
| 97 | + try: |
| 98 | + service, rest = path.split('/', 1) |
| 99 | + except ValueError: |
| 100 | + # Root calls like "GET /identity" |
| 101 | + service = path |
| 102 | + rest = '' |
| 103 | + |
| 104 | + stats.setdefault(service, {'largest': 0}) |
| 105 | + stats[service].setdefault(method, 0) |
| 106 | + stats[service][method] += 1 |
| 107 | + stats[service]['largest'] = max(stats[service]['largest'], size) |
| 108 | + |
| 109 | + # Flatten this for ES |
| 110 | + return [{'service': service, 'log': os.path.basename(logfile), |
| 111 | + **vals} |
| 112 | + for service, vals in stats.items()] |
| 113 | + |
| 114 | + |
| 115 | +def get_http_stats(logfiles): |
| 116 | + return list(itertools.chain.from_iterable(get_http_stats_for_log(log) |
| 117 | + for log in logfiles)) |
| 118 | + |
| 119 | + |
| 120 | +def get_report_info(): |
| 121 | + return { |
| 122 | + 'timestamp': datetime.datetime.now().isoformat(), |
| 123 | + 'hostname': socket.gethostname(), |
| 124 | + } |
| 125 | + |
| 126 | + |
| 127 | +if __name__ == '__main__': |
| 128 | + process_defaults = ['privsep', 'mysqld', 'erlang', 'etcd'] |
| 129 | + parser = argparse.ArgumentParser() |
| 130 | + parser.add_argument('--db-user', default='root', |
| 131 | + help=('MySQL user for collecting stats ' |
| 132 | + '(default: "root")')) |
| 133 | + parser.add_argument('--db-pass', default=None, |
| 134 | + help='MySQL password for db-user') |
| 135 | + parser.add_argument('--db-host', default='localhost', |
| 136 | + help='MySQL hostname') |
| 137 | + parser.add_argument('--apache-log', action='append', default=[], |
| 138 | + help='Collect API call stats from this apache log') |
| 139 | + parser.add_argument('--process', action='append', |
| 140 | + default=process_defaults, |
| 141 | + help=('Include process stats for this cmdline regex ' |
| 142 | + '(default is %s)' % ','.join(process_defaults))) |
| 143 | + args = parser.parse_args() |
| 144 | + |
| 145 | + data = { |
| 146 | + 'services': get_services_stats(), |
| 147 | + 'db': args.db_pass and get_db_stats(args.db_host, |
| 148 | + args.db_user, |
| 149 | + args.db_pass) or [], |
| 150 | + 'processes': get_processes_stats(args.process), |
| 151 | + 'api': get_http_stats(args.apache_log), |
| 152 | + 'report': get_report_info(), |
| 153 | + } |
| 154 | + |
| 155 | + print(json.dumps(data, indent=2)) |
0 commit comments