store_module_data

#!/bin/sh
# -*- python -*-
""":"
for cmd in python3 python python2; do
   command -v > /dev/null $cmd && exec $cmd $0 "$@"
done

echo "Error: Could not find a valid python interpreter --> exiting!" >&2
exit 2
":"""
################################################################################

from __future__ import print_function
import os, sys, re, time, argparse, json, operator
from LMODdb import LMODdb

def syshost(name):
    """Safely extract the system hostname (short form)"""
    if not name or not isinstance(name, str):
        return "unknown"
    hostA = name.split('.')
    return hostA[1] if len(hostA) > 1 else hostA[0]


class CmdLineOptions(object):
    def __init__(self):
        pass

    def execute(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("-D", dest='debug', action="store_true", help="Debug Flag")
        parser.add_argument("--delete", dest='delete', action="store_true", default=False, help="delete file after processing")
        parser.add_argument("--store_all", dest='storeAll', action="store_true", default=False, help="Store all modules and not deduplicate")
        parser.add_argument("--confFn", dest='confFn', action="store", default="lmodV2_db.conf", help="DB config file")
        parser.add_argument("--ignoreFn", dest='ignoreFn', action="store", help="List of module patterns to ignore")
        parser.add_argument('dataFnA', metavar='dataFnA', type=str, nargs='+', help='a list of Syslog/Json files to process')
        return parser.parse_args()


class Ignore(object):
    def __init__(self, ignoreFn):
        self.__ignoreA = []
        if ignoreFn and os.path.exists(ignoreFn):
            with open(ignoreFn, 'r') as f:
                for line in f:
                    line = line.strip()
                    if line:
                        self.__ignoreA.append(re.compile(line))
    def ignore(self, module):
        return any(rex.match(module) for rex in self.__ignoreA)


class Reader(object):
    def __init__(self, debug, ignore, ext, lmod):
        self.debug = debug
        self.ignore = ignore
        self.ext = ext
        self.lmod = lmod
        self.modulePatt = re.compile(r'.*ModuleUsage[^ ]* *')

    def handleOneLine(self, line):
        line = line.rstrip()
        if not line:
            return None

        try:
            if self.ext == '.json':
                dataT = json.loads(line)
            else:
                rest = self.modulePatt.sub('', line)
                dataT = dict(re.findall(r'(\S+)=(".*?"|\S+)', rest))

            host = dataT.get('host')
            dataT['syshost'] = syshost(host)
            dataT['date'] = dataT.get('time', time.time())
            dataT.pop('time', None)
            dataT.pop('host', None)

            if dataT['syshost'] == "unknown":
                msg = f"[WARN] Missing host in line: {line}"
                if self.debug:
                    print(msg)
                with open("store.log", "a") as flog:
                    flog.write(msg + "\n")
                return None

            if self.ignore.ignore(dataT.get('module', '')):
                msg = f"[INFO] Ignored module: {dataT.get('module')}"
                if self.debug:
                    print(msg)
                with open("store.log", "a") as flog:
                    flog.write(msg + "\n")
                return None

            if self.debug:
                print(f"[INFO] Parsed line: {dataT}")

            return dataT
        except Exception as e:
            msg = f"[ERROR] Failed to parse line: {line}\n{e}"
            if self.debug:
                print(msg)
            with open("store.log", "a") as flog:
                flog.write(msg + "\n")
            return None


class StoreAll(Reader):
    def __init__(self, debug, ignore, ext, lmod):
        super().__init__(debug, ignore, ext, lmod)
        self.__blkSz  = 1000
        self.__icount = 0
        self.__dataA  = []

    def storeOneLine(self, dataT):
        self.__icount += 1
        self.__dataA.append(dataT.copy())
        if self.__icount >= self.__blkSz:
            self.lmod.data_to_db(self.debug, self.__dataA)
            self.__dataA.clear()
            self.__icount = 0

    def save(self):
        if self.__dataA:
            self.lmod.data_to_db(self.debug, self.__dataA)


class Dedup(Reader):
    def __init__(self, debug, ignore, ext, lmod):
        self.__usageT = {}
        super().__init__(debug, ignore, ext, lmod)

    def storeOneLine(self, dataT):
        key = dataT['user'] + ' ' + dataT['syshost'] + ' ' + dataT['module'] + ' ' + dataT['path']
        if key not in self.__usageT:
            self.__usageT[key] = {
                'user'    : dataT['user'],
                'syshost' : dataT['syshost'],
                'module'  : dataT['module'],
                'path'    : dataT['path'],
                'date'    : dataT['date'],
            }

    def save(self):
        dataA = list(self.__usageT.values())
        dataA.sort(key=operator.itemgetter('date'))
        if dataA:
            self.lmod.data_to_db(self.debug, dataA)


def main():
    args   = CmdLineOptions().execute()
    lmod   = LMODdb(args.confFn)
    ignore = Ignore(args.ignoreFn)

    for dataFn in args.dataFnA:
        baseNm, extension = os.path.splitext(dataFn)
        reader = StoreAll(args.debug, ignore, extension, lmod) if args.storeAll else Dedup(args.debug, ignore, extension, lmod)

        if not os.path.exists(dataFn):
            continue

        with open(dataFn, "r") as file:
            for line in file:
                dataT = reader.handleOneLine(line)
                if not dataT:
                    continue
                reader.storeOneLine(dataT)

        reader.save()

        if args.delete:
            os.remove(dataFn)


if __name__ == '__main__':
    main()