From d55eb546770c1ba6895e38536398884404613040 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Fri, 1 Sep 2017 12:29:32 -0400 Subject: [PATCH 01/20] wholesale rewrite - use Ruamel's YAML library, do some parameter checking, handle STDIN as input. --- src/main/python/yamlreader/__init__.py | 4 +- src/main/python/yamlreader/yamlreader.py | 183 +++++++++++++---------- 2 files changed, 109 insertions(+), 78 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index cb5def8..978afbd 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -2,6 +2,6 @@ from __future__ import print_function, absolute_import, unicode_literals, division -from .yamlreader import data_merge, yaml_load, YamlReaderError +from .yamlreader import data_merge, get_files, YamlReaderError -__all__ = ['data_merge', 'yaml_load', 'YamlReaderError'] +__all__ = ['data_merge', 'YamlReaderError'] diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 413d2a0..588a329 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -1,35 +1,36 @@ from __future__ import print_function, absolute_import, unicode_literals, division -__version__ = '3.0.3' +__version__ = '3.1.0' -from yaml import MarkedYAMLError, safe_load, safe_dump +import ruamel.yaml as yaml +from ruamel.yaml import MarkedYAMLError, safe_load, safe_dump import glob import os +import sys import logging import six - -class NoDefault(object): - def __str__(self): - return "No default data" - - -NO_DEFAULT = NoDefault() +logger = logging.getLogger(__name__) class YamlReaderError(Exception): - pass + def __init__(self, msg=''): + super().__init__(msg) + logger.error(msg, sys.exc_info()) + + def __str__(str): + return self.msg def data_merge(a, b): """merges b into a and return merged result based on http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge - and extended to also merge arrays and to replace the content of keys with the same name + and extended to also merge arrays (append) and dict keys replaced if having the same name. NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen""" key = None - # ## debug output - # sys.stderr.write("DEBUG: %s to %s\n" %(b,a)) + + logger.debug("data_merge(): %s to %s\n" %(b,a)) try: if a is None or isinstance(a, (six.string_types, float, six.integer_types)): # border case for first run or if a is a primitive @@ -51,7 +52,7 @@ def data_merge(a, b): else: a[key] = b[key] else: - raise YamlReaderError('Cannot merge non-dict "%s" into dict "%s"' % (b, a)) + raise YamlReaderError('UNSUPPORTED merge non-dict "%s" into dict "%s"' % (b, a)) else: raise YamlReaderError('NOT IMPLEMENTED "%s" into "%s"' % (b, a)) except TypeError as e: @@ -59,87 +60,117 @@ def data_merge(a, b): return a -def yaml_load(source, defaultdata=NO_DEFAULT): - """merge YAML data from files found in source - - Always returns a dict. The YAML files are expected to contain some kind of - key:value structures, possibly deeply nested. When merging, lists are - appended and dict keys are replaced. The YAML files are read with the - yaml.safe_load function. - - source can be a file, a dir, a list/tuple of files or a string containing - a glob expression (with ?*[]). - - For a directory, all *.yaml files will be read in alphabetical order. +def get_files(source, suffix='yaml'): + """ + source can be a file, a directory, a list/tuple of files or + a string containing a glob expression (with ?*[]). - defaultdata can be used to initialize the data. + For a directory, filenames of *.yaml will be read. """ - logger = logging.getLogger(__name__) - logger.debug("initialized with source=%s, defaultdata=%s", source, defaultdata) - if defaultdata is NO_DEFAULT: - data = None - else: - data = defaultdata files = [] - if type(source) is not str and len(source) == 1: - # when called from __main source is always a list, even if it contains only one item. - # turn into a string if it contains only one item to support our different call modes - source = source[0] + if type(source) is list or type(source) is tuple: - # got a list, assume to be files - files = source - elif os.path.isdir(source): - # got a dir, read all *.yaml files - files = sorted(glob.glob(os.path.join(source, "*.yaml"))) + # when called from __main() as get_files(args, ...), 'source' is always a list of size >=0. + if len(source) == 1: + # turn into a string to evaluate further + source = source[0] + else: + for item in source: + # iterate to expand list of potential dirs and files + files.extend(get_files(item, suffix)) + return files + + if type(source) is not str or len(source) == 0 or source == '-': + return [] + + if os.path.isdir(source): + files = glob.glob(os.path.join(source, '*.' + suffix)) elif os.path.isfile(source): - # got a single file, turn it into list to use the same code + # turn single file into list files = [source] else: # try to use the source as a glob - files = sorted(glob.glob(source)) - if files: - logger.debug("Reading %s\n", ", ".join(files)) - for yaml_file in files: - try: - with open(yaml_file) as f: - new_data = safe_load(f) - logger.debug("YAML LOAD: %s", new_data) - except MarkedYAMLError as e: - logger.error("YAML Error: %s", e) - raise YamlReaderError("YAML Error: %s" % str(e)) - if new_data is not None: - data = data_merge(data, new_data) - else: - if defaultdata is NO_DEFAULT: - logger.error("No YAML data found in %s and no default data given", source) - raise YamlReaderError("No YAML data found in %s" % source) + files = glob.glob(source) - return data + return files def __main(): import optparse - parser = optparse.OptionParser(usage="%prog [options] source...", - description="Merge YAML data from given files, dir or file glob", + parser = optparse.OptionParser(usage="%prog [options] source ...", + description="Merge YAML data from given files, directory or glob", version="%" + "prog %s" % __version__, - prog="yamlreader") - parser.add_option("--debug", dest="debug", action="store_true", default=False, + prog='yamlreader') + + parser.add_option('-d', '--debug', dest='debug', action='store_true', default=False, help="Enable debug logging [%default]") - options, args = parser.parse_args() - if options.debug: - logger = logging.getLogger() - loghandler = logging.StreamHandler() - loghandler.setFormatter(logging.Formatter('yamlreader: %(levelname)s: %(message)s')) - logger.addHandler(loghandler) - logger.setLevel(logging.DEBUG) + parser.add_option('-t', '--indent', dest='indent', action='store', type='int', default=2, + help="indent width [%default ]") + parser.add_option('--loader', dest='loader', action='store', default='safe', + help="loader class [ %default ]") + parser.add_option('--dumper', dest='dumper', action='store', default='safe', + help="dumper class [ %default ]") + # CloudFormation can't handle anchors + parser.add_option('-x', '--no-anchor', dest='no_anchor', action='store_true', default=False, + help="unroll anchors and aliases [ %default ]") + parser.add_option('--sort-files', dest='sort_files', action='store_true', default=False, + help="sort input filenames [ %default ]") + parser.add_option('-r', '--reverse', dest='reverse', action='store_true', default=False, + help="sort direction [ %default ]") + parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=False, + help="sort keys in dump [ %default ]") + parser.add_option('-l', '--logfile', dest='logfile', action='store', default=None) + parser.add_option('--suffix', dest='suffix', action='store', default='yaml') - if not args: - parser.error("Need at least one argument") try: - print(safe_dump(yaml_load(args, defaultdata={}), - indent=4, default_flow_style=False, canonical=False)) + (options, args) = parser.parse_args() except Exception as e: parser.error(e) + log_handler = logging.StreamHandler(options.logfile) + log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)s: %(message)s')) + logger.addHandler(log_handler) + if options.debug: + logger.setLevel(logging.DEBUG) + + # see http://yaml.readthedocs.io/en/latest/detail.html for examples + indent = {'mapping': options.indent, 'sequence': options.indent * 2, 'offset': options.indent} + data = None + files = get_files(args, options.suffix) + + myaml = yaml.YAML(typ='safe') + myaml.preserve_quotes=True + myaml.default_flow_style=False + myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) + myaml.representer.ignore_aliases = lambda *args: True + # NOTICE! sort_keys *ONLY* works with matt's version + myaml.representer.sort_keys = options.sort_keys + + if len(files) == 0: + # Hack! force at least 1 pass thru FOR loop and '' stands in for + files = [''] + + for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: + logger.debug("Reading file %s\n", yaml_file) + try: + #new_data = yaml.load(open(yaml_file) if len(yaml_file) else sys.stdin, Loader=Loader, preserve_quotes=True) + new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) + logger.debug("YAML Load: %s", new_data) + except MarkedYAMLError as e: + # logger.exception("YAML Error: %s", e) + raise YamlReaderError("YAML Error: %s" % str(e)) + + if new_data is not None: + data = data_merge(data, new_data) + + if not len(data): + logger.warn("No YAML data found in %s", source) + else: + try: + myaml.dump(data, sys.stdout) + except Exception as e: + logger.exception(e, sys.exc_info()) + + if __name__ == "__main__": __main() From 130ec9302cefd9ba96bd424ae0920b96cc2c8b6b Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Tue, 3 Oct 2017 18:58:39 -0400 Subject: [PATCH 02/20] remove unnecessary library references --- src/main/python/yamlreader/yamlreader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 588a329..6ebfe53 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -1,9 +1,10 @@ +#!env python3 + from __future__ import print_function, absolute_import, unicode_literals, division __version__ = '3.1.0' import ruamel.yaml as yaml -from ruamel.yaml import MarkedYAMLError, safe_load, safe_dump import glob import os import sys From aa56d1fb26221b1500f9d8ed5b8a49c773ff942b Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Tue, 3 Oct 2017 19:20:16 -0400 Subject: [PATCH 03/20] rewrite custom Exception handler and simplify key merge function --- src/main/python/yamlreader/yamlreader.py | 107 +++++++++++++++-------- 1 file changed, 71 insertions(+), 36 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 6ebfe53..31290e0 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -15,74 +15,109 @@ class YamlReaderError(Exception): - def __init__(self, msg=''): + """write YAML processing errors to logger""" + #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT + def __init__(self, msg, *args, **kwargs): + level = logging.ERROR + + if args: + level = args[0].upper() if isinstance(args[0], str) else args[0] + #TODO case statement to generate/modify strings so it's not buried in multiple + # places in code. eg. 'filenotfound' is easy case. msg == filename(s) + # try: super().__init__(msg) - logger.error(msg, sys.exc_info()) + logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, + exc_info=(logger.getEffectiveLevel() == logging.DEBUG), kwargs) - def __str__(str): - return self.msg + if (level == logging.FATAL): + sys.exit(1) + + #TODO break out and differentiate as needed. some raise, others (all?) pass + pass def data_merge(a, b): """merges b into a and return merged result + based on http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge and extended to also merge arrays (append) and dict keys replaced if having the same name. - NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen""" + NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen + """ key = None - logger.debug("data_merge(): %s to %s\n" %(b,a)) + #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) try: + # border case for first run or if a is a primitive if a is None or isinstance(a, (six.string_types, float, six.integer_types)): - # border case for first run or if a is a primitive a = b elif isinstance(a, list): - # lists can be only appended - if isinstance(b, list): - # merge lists - a.extend(b) - else: - # append to list - a.append(b) + a.extend(b) if isinstance(b, list) else a.append(b) + a = list(set(a)) elif isinstance(a, dict): # dicts must be merged if isinstance(b, dict): for key in b: - if key in a: - a[key] = data_merge(a[key], b[key]) - else: - a[key] = b[key] + a[key] = data_merge(a[key], b[key]) if key in a else b[key] else: - raise YamlReaderError('UNSUPPORTED merge non-dict "%s" into dict "%s"' % (b, a)) + raise TypeError else: - raise YamlReaderError('NOT IMPLEMENTED "%s" into "%s"' % (b, a)) - except TypeError as e: - raise YamlReaderError('TypeError "%s" in key "%s" when merging "%s" into "%s"' % (e, key, b, a)) + raise TypeError + +#----- original + # if a is None or isinstance(a, (six.string_types, float, six.integer_types)): + # # border case for first run or if a is a primitive + # a = b + # elif isinstance(a, list): + # # lists can be only appended + # if isinstance(b, list): + # # merge lists + # a.extend(b) + # else: + # # append to list + # a.append(b) + # elif isinstance(a, dict): + # # dicts must be merged + # if isinstance(b, dict): + # for key in b: + # if key in a: + # a[key] = data_merge(a[key], b[key]) + # else: + # a[key] = b[key] + # else: + # raise YamlReaderError('Illegal - %s into %s\n "%s" -> "%s"' % + # (type(b), type(a), b, a), logging.WARNING) + # else: + # raise YamlReaderError('TODO - %s into %s\n "%s" -> "%s"' % + # (type(b), type(a), b, a), logging.WARNING) + except (TypeError, LookupError) as e: + raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % + (e, type(b), type(a), b, a), logging.WARNING) + # or str(e) also with e.__name__ ? + return a def get_files(source, suffix='yaml'): - """ + """Examine path elements for files to processing + source can be a file, a directory, a list/tuple of files or a string containing a glob expression (with ?*[]). - For a directory, filenames of *.yaml will be read. + For a directory, filenames of $suffix will be read. """ + files = [] - if type(source) is list or type(source) is tuple: - # when called from __main() as get_files(args, ...), 'source' is always a list of size >=0. - if len(source) == 1: - # turn into a string to evaluate further - source = source[0] - else: - for item in source: - # iterate to expand list of potential dirs and files - files.extend(get_files(item, suffix)) - return files + if source is None or len(source) == 0 or source == '-': + return [''] - if type(source) is not str or len(source) == 0 or source == '-': - return [] + #if type(source) is list or type(source) is tuple: + if isinstance(source, list) or isinstance(source, tuple): + for item in source: + # iterate to expand list of potential dirs and files + files.extend(get_files(item, suffix)) + return files if os.path.isdir(source): files = glob.glob(os.path.join(source, '*.' + suffix)) From 39e9a7d595abca6239030f04f08a33f9a5770ae6 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Tue, 3 Oct 2017 19:36:18 -0400 Subject: [PATCH 04/20] add command-line options for logging --- src/main/python/yamlreader/yamlreader.py | 129 ++++++++++++++++------- 1 file changed, 89 insertions(+), 40 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 31290e0..facdedb 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -128,85 +128,134 @@ def get_files(source, suffix='yaml'): # try to use the source as a glob files = glob.glob(source) + if len(files) == 0: + YamlReaderError('FileNotFoundError for %r' % (source), logging.WARNING) + return files def __main(): import optparse parser = optparse.OptionParser(usage="%prog [options] source ...", - description="Merge YAML data from given files, directory or glob", + description='Merge YAML data from files, directory or glob', version="%" + "prog %s" % __version__, prog='yamlreader') parser.add_option('-d', '--debug', dest='debug', action='store_true', default=False, - help="Enable debug logging [%default]") - parser.add_option('-t', '--indent', dest='indent', action='store', type='int', default=2, - help="indent width [%default ]") - parser.add_option('--loader', dest='loader', action='store', default='safe', - help="loader class [ %default ]") - parser.add_option('--dumper', dest='dumper', action='store', default='safe', - help="dumper class [ %default ]") - # CloudFormation can't handle anchors + help="Enable debug logging (true *%default*)") + parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, + help="write progress (true *%default*)") + + parser.add_option('-l', '--logfile', dest='logfile', action='store', default=None) + parser.add_option('--log', dest='loglevel', action='store', default='INFO', + help="(DEBUG *%default WARNING ERROR CRITICAL)") + + # CloudFormation can't handle anchors or aliases (sep '17) parser.add_option('-x', '--no-anchor', dest='no_anchor', action='store_true', default=False, - help="unroll anchors and aliases [ %default ]") + help="unroll anchors/aliases (true *%default)") + parser.add_option('-u', '--duplicate-keys', dest='duplicate_keys', action='store_true', default=False, + help="allow duplicate keys (true *%default)") + + parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=False, + help="sort keys in dump (true *%default)") parser.add_option('--sort-files', dest='sort_files', action='store_true', default=False, - help="sort input filenames [ %default ]") + help="sort input filenames (true *%default)") parser.add_option('-r', '--reverse', dest='reverse', action='store_true', default=False, - help="sort direction [ %default ]") - parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=False, - help="sort keys in dump [ %default ]") - parser.add_option('-l', '--logfile', dest='logfile', action='store', default=None) + help="sort direction (true *%default)") + parser.add_option('--suffix', dest='suffix', action='store', default='yaml') + parser.add_option('-t', '--indent', dest='indent', action='store', type=int, default=2, + help="indent width (%default)") + parser.add_option('--loader', dest='loader', action='store', default='safe', + help="loader class (base *%default roundtrip unsafe)") + try: (options, args) = parser.parse_args() except Exception as e: parser.error(e) - log_handler = logging.StreamHandler(options.logfile) - log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)s: %(message)s')) + # mangle options.loader for API consuption + options.loader= options.loader.lower() + if options.loader == 'roundtrip': + options.loader = 'rt' + + if isinstance(options.loglevel, str): + options.loglevel = options.loglevel.upper() + + + if options.logfile: + log_handler = logging.FileHandler(options.logfile, mode='w') + else: + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)9s %(message)s')) logger.addHandler(log_handler) + logger.propagate = False if options.debug: logger.setLevel(logging.DEBUG) + else: + # in 3.2+ we can do + #logger.setLevel(options.loglevel) + logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) + + # remove Traceback output in Verbose, squelch stacktrace all other times + # if options.verbose: + # sys.excepthook = lambda exctype,exc,traceback : print("{}: {}".format(exctype.__name__, exc)) + # else: + # sys.excepthook = lambda *args: None # see http://yaml.readthedocs.io/en/latest/detail.html for examples - indent = {'mapping': options.indent, 'sequence': options.indent * 2, 'offset': options.indent} - data = None + data = new_data = None + files = get_files(args, options.suffix) + if len(files) == 0: + #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(args)), 'CRITICAL') + raise YamlReaderError('No files found! %s' % str(args), logging.FATAL) + # FATAL ==> not reached - myaml = yaml.YAML(typ='safe') + indent = { + 'mapping' : options.indent, + 'sequence': options.indent * 2, + 'offset' : options.indent + } + + myaml = yaml.YAML(typ=options.loader) myaml.preserve_quotes=True myaml.default_flow_style=False + myaml.allow_duplicate_keys = options.duplicate_keys myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) myaml.representer.ignore_aliases = lambda *args: True - # NOTICE! sort_keys *ONLY* works with matt's version - myaml.representer.sort_keys = options.sort_keys - if len(files) == 0: - # Hack! force at least 1 pass thru FOR loop and '' stands in for - files = [''] + # NOTICE! sort_keys is a noop unless using Matt's version of + # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) + if hasattr(myaml.representer, 'sort_keys'): + myaml.representer.sort_keys = options.sort_keys for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: - logger.debug("Reading file %s\n", yaml_file) + if options.verbose: + logger.info('Reading file "%s"', yaml_file) try: - #new_data = yaml.load(open(yaml_file) if len(yaml_file) else sys.stdin, Loader=Loader, preserve_quotes=True) new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) - logger.debug("YAML Load: %s", new_data) - except MarkedYAMLError as e: - # logger.exception("YAML Error: %s", e) - raise YamlReaderError("YAML Error: %s" % str(e)) + logger.debug('Payload: %r\n', new_data) + except (yaml.MarkedYAMLError) as e: + logger.warning('YAML.load() -- %s' % str(e)) + #raise YamlReaderError('YAML.load() -- %s' % str(e)) - if new_data is not None: + if new_data: # is not None: data = data_merge(data, new_data) + else: + logger.warning('No YAML data found in "%s"', yaml_file) - if not len(data): - logger.warn("No YAML data found in %s", source) - else: - try: - myaml.dump(data, sys.stdout) - except Exception as e: - logger.exception(e, sys.exc_info()) + if data is None or len(data) == 0: + logger.critical('No YAML data found anywhere!') + return 1 + + try: + myaml.dump(data, sys.stdout) + except yaml.MarkedYAMLError as e: + raise YamlReaderError('YAML.dump() -- %s' % str(e)) + #YamlReaderError("YAML.dump(): %s" % str(e)) if __name__ == "__main__": - __main() + sys.exit(__main()) From 06bf80cf063fb55763b61f05de06dc14e4b15f89 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Tue, 3 Oct 2017 19:47:05 -0400 Subject: [PATCH 05/20] remove original value merge logic --- src/main/python/yamlreader/yamlreader.py | 26 ------------------------ 1 file changed, 26 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index facdedb..944ed68 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -64,32 +64,6 @@ def data_merge(a, b): else: raise TypeError -#----- original - # if a is None or isinstance(a, (six.string_types, float, six.integer_types)): - # # border case for first run or if a is a primitive - # a = b - # elif isinstance(a, list): - # # lists can be only appended - # if isinstance(b, list): - # # merge lists - # a.extend(b) - # else: - # # append to list - # a.append(b) - # elif isinstance(a, dict): - # # dicts must be merged - # if isinstance(b, dict): - # for key in b: - # if key in a: - # a[key] = data_merge(a[key], b[key]) - # else: - # a[key] = b[key] - # else: - # raise YamlReaderError('Illegal - %s into %s\n "%s" -> "%s"' % - # (type(b), type(a), b, a), logging.WARNING) - # else: - # raise YamlReaderError('TODO - %s into %s\n "%s" -> "%s"' % - # (type(b), type(a), b, a), logging.WARNING) except (TypeError, LookupError) as e: raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % (e, type(b), type(a), b, a), logging.WARNING) From 14c5f6b1f8a083a13386449b2353a461332fd5ca Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Wed, 4 Oct 2017 12:00:35 -0400 Subject: [PATCH 06/20] add JSON output option --- src/main/python/yamlreader/yamlreader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 944ed68..23d371b 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -5,6 +5,7 @@ __version__ = '3.1.0' import ruamel.yaml as yaml +import json import glob import os import sys @@ -124,6 +125,8 @@ def __main(): parser.add_option('--log', dest='loglevel', action='store', default='INFO', help="(DEBUG *%default WARNING ERROR CRITICAL)") + parser.add_option('-j', '--json', dest='json', action='store_true', default=False, + help="output to JSON (true *%default)") # CloudFormation can't handle anchors or aliases (sep '17) parser.add_option('-x', '--no-anchor', dest='no_anchor', action='store_true', default=False, help="unroll anchors/aliases (true *%default)") @@ -225,7 +228,10 @@ def __main(): return 1 try: - myaml.dump(data, sys.stdout) + if options.json: + json.dump(data, sys.stdout) + else: + myaml.dump(data, sys.stdout) except yaml.MarkedYAMLError as e: raise YamlReaderError('YAML.dump() -- %s' % str(e)) #YamlReaderError("YAML.dump(): %s" % str(e)) From f8f1c3e7f692c6b75f8a7f33953e7d1ed00ff553 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Wed, 4 Oct 2017 14:29:17 -0400 Subject: [PATCH 07/20] add formatting support to json.dump and trap what should be theoretical exceptions --- src/main/python/yamlreader/yamlreader.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 23d371b..39ac0ed 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -28,7 +28,7 @@ def __init__(self, msg, *args, **kwargs): # try: super().__init__(msg) logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, - exc_info=(logger.getEffectiveLevel() == logging.DEBUG), kwargs) + exc_info=(logger.getEffectiveLevel() == logging.DEBUG), **kwargs) if (level == logging.FATAL): sys.exit(1) @@ -229,12 +229,15 @@ def __main(): try: if options.json: - json.dump(data, sys.stdout) + json.dump(data, sys.stdout, indent=indent['mapping']) else: myaml.dump(data, sys.stdout) except yaml.MarkedYAMLError as e: raise YamlReaderError('YAML.dump() -- %s' % str(e)) #YamlReaderError("YAML.dump(): %s" % str(e)) + except (ValueError, OverflowError, TypeError): + # JSON dump might trigger + pass if __name__ == "__main__": From 1cde1b2130db3929c9bd2058b808fc7f8fc9e8df Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Wed, 11 Oct 2017 19:27:11 -0400 Subject: [PATCH 08/20] define value overwriting instead of merge --- src/main/python/yamlreader/yamlreader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 39ac0ed..2f8e8e8 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -37,7 +37,7 @@ def __init__(self, msg, *args, **kwargs): pass -def data_merge(a, b): +def data_merge(a, b, merge=True): """merges b into a and return merged result based on http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge @@ -50,7 +50,8 @@ def data_merge(a, b): #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) try: # border case for first run or if a is a primitive - if a is None or isinstance(a, (six.string_types, float, six.integer_types)): + if a is None or merge == False or \ + isinstance(a, (six.string_types, float, six.integer_types)): a = b elif isinstance(a, list): a.extend(b) if isinstance(b, list) else a.append(b) @@ -127,6 +128,10 @@ def __main(): parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help="output to JSON (true *%default)") + parser.add_option('-m', '--merge', dest='merge', action='store_true', default=True, + help="merge a key's values (*%default false)") + parser.add_option('--overwrite', dest='merge', action='store_false', + help="overwrite a key's values (true *false)") # CloudFormation can't handle anchors or aliases (sep '17) parser.add_option('-x', '--no-anchor', dest='no_anchor', action='store_true', default=False, help="unroll anchors/aliases (true *%default)") @@ -219,7 +224,7 @@ def __main(): #raise YamlReaderError('YAML.load() -- %s' % str(e)) if new_data: # is not None: - data = data_merge(data, new_data) + data = data_merge(data, new_data, options.merge) else: logger.warning('No YAML data found in "%s"', yaml_file) From d61f0c30a07e3ce8db77b74c40fcb9928b4a33ae Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Wed, 11 Oct 2017 20:04:22 -0400 Subject: [PATCH 09/20] reformat Optparse help --- src/main/python/yamlreader/yamlreader.py | 107 ++++++++++++++--------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 2f8e8e8..5453665 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -6,11 +6,9 @@ import ruamel.yaml as yaml import json -import glob import os import sys import logging -import six logger = logging.getLogger(__name__) @@ -45,6 +43,8 @@ def data_merge(a, b, merge=True): NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen """ + + import six key = None #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) @@ -83,6 +83,7 @@ def get_files(source, suffix='yaml'): For a directory, filenames of $suffix will be read. """ + import glob files = [] if source is None or len(source) == 0 or source == '-': @@ -111,45 +112,73 @@ def get_files(source, suffix='yaml'): def __main(): - import optparse + import optparse, textwrap + parser = optparse.OptionParser(usage="%prog [options] source ...", - description='Merge YAML data from files, directory or glob', - version="%" + "prog %s" % __version__, - prog='yamlreader') - - parser.add_option('-d', '--debug', dest='debug', action='store_true', default=False, - help="Enable debug logging (true *%default*)") - parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=False, - help="write progress (true *%default*)") - - parser.add_option('-l', '--logfile', dest='logfile', action='store', default=None) - parser.add_option('--log', dest='loglevel', action='store', default='INFO', - help="(DEBUG *%default WARNING ERROR CRITICAL)") - - parser.add_option('-j', '--json', dest='json', action='store_true', default=False, - help="output to JSON (true *%default)") - parser.add_option('-m', '--merge', dest='merge', action='store_true', default=True, - help="merge a key's values (*%default false)") - parser.add_option('--overwrite', dest='merge', action='store_false', - help="overwrite a key's values (true *false)") + description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', + version="%" + "prog %s" % __version__, prog='yamlreader') + + parser.add_option('-d', '--debug', dest='debug', + action='store_true', default=False, + help="Enable debug logging (true *%default*)") + + parser.add_option('-v', '--verbose', dest='verbose', + action='store_true', default=False, + help="write progress (true *%default*)") + + parser.add_option('-q', '--quiet', dest='verbose', + action='store_false', + help="minimize output (*True false)") + + parser.add_option('-l', '--logfile', dest='logfile', + action='store', default=None) + + parser.add_option('--log', dest='loglevel', + action='store', default='INFO', + help="(DEBUG *%default WARNING ERROR CRITICAL)") + + parser.add_option('-j', '--json', dest='json', + action='store_true', default=False, + help="output to JSON (true *%default)") + + parser.add_option('-m', '--merge', dest='merge', + action='store_true', default=True, + help="merge a key's values (*%default false)") + + parser.add_option('--overwrite', dest='merge', + action='store_false', + help="overwrite a key's values (true *false)") # CloudFormation can't handle anchors or aliases (sep '17) - parser.add_option('-x', '--no-anchor', dest='no_anchor', action='store_true', default=False, - help="unroll anchors/aliases (true *%default)") - parser.add_option('-u', '--duplicate-keys', dest='duplicate_keys', action='store_true', default=False, - help="allow duplicate keys (true *%default)") - - parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=False, - help="sort keys in dump (true *%default)") - parser.add_option('--sort-files', dest='sort_files', action='store_true', default=False, - help="sort input filenames (true *%default)") - parser.add_option('-r', '--reverse', dest='reverse', action='store_true', default=False, - help="sort direction (true *%default)") - - parser.add_option('--suffix', dest='suffix', action='store', default='yaml') - parser.add_option('-t', '--indent', dest='indent', action='store', type=int, default=2, - help="indent width (%default)") - parser.add_option('--loader', dest='loader', action='store', default='safe', - help="loader class (base *%default roundtrip unsafe)") + parser.add_option('-x', '--no-anchor', dest='no_anchor', + action='store_true', default=False, + help="unroll anchors/aliases (true *%default)") + + parser.add_option('-u', '--duplicate-keys', dest='duplicate_keys', + action='store_true', default=False, + help="allow duplicate keys (true *%default)") + + parser.add_option('-r', '--reverse', dest='reverse', + action='store_true', default=False, + help="sort direction (true *%default)") + + parser.add_option('-k', '--sort-keys', dest='sort_keys', + action='store_true', default=False, + help="sort keys in dump (true *%default)") + + parser.add_option('--sort-files', dest='sort_files', + action='store_true', default=False, + help="sort input filenames (true *%default)") + + parser.add_option('--suffix', dest='suffix', + action='store', default='yaml') + + parser.add_option('-t', '--indent', dest='indent', + action='store', type=int, default=2, + help="indent width (%default)") + + parser.add_option('--loader', dest='loader', + action='store', default='safe', + help="loader class (base *%default roundtrip unsafe)") try: From 88062e4f80f8d439d45d6c1e6663df6fb958faf8 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Wed, 11 Oct 2017 20:25:19 -0400 Subject: [PATCH 10/20] localize module imports --- src/main/python/yamlreader/yamlreader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index 5453665..b65bb0d 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -6,7 +6,6 @@ import ruamel.yaml as yaml import json -import os import sys import logging @@ -83,7 +82,7 @@ def get_files(source, suffix='yaml'): For a directory, filenames of $suffix will be read. """ - import glob + import os, glob files = [] if source is None or len(source) == 0 or source == '-': @@ -135,7 +134,7 @@ def __main(): parser.add_option('--log', dest='loglevel', action='store', default='INFO', - help="(DEBUG *%default WARNING ERROR CRITICAL)") + help="(debug *%default warning error critical)") parser.add_option('-j', '--json', dest='json', action='store_true', default=False, @@ -221,7 +220,7 @@ def __main(): files = get_files(args, options.suffix) if len(files) == 0: #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(args)), 'CRITICAL') - raise YamlReaderError('No files found! %s' % str(args), logging.FATAL) + raise YamlReaderError('No source files found! %s' % str(args), logging.FATAL) # FATAL ==> not reached indent = { From 5a91482c324a26fe92e68cb52ad79e326776ed22 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Thu, 12 Oct 2017 03:50:46 -0400 Subject: [PATCH 11/20] refactor all defaults to one location. move imports closer to use. replace __main with main() for 3rd party invocation --- src/main/python/yamlreader/__init__.py | 4 +- src/main/python/yamlreader/yamlreader.py | 136 ++++++++++++++--------- 2 files changed, 85 insertions(+), 55 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 978afbd..3c24478 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -2,6 +2,6 @@ from __future__ import print_function, absolute_import, unicode_literals, division -from .yamlreader import data_merge, get_files, YamlReaderError +from .yamlreader import main, parse_cmdline, data_merge, get_files, YamlReaderError -__all__ = ['data_merge', 'YamlReaderError'] +__all__ = ['main', 'data_merge', 'YamlReaderError'] diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py index b65bb0d..b1c91f6 100644 --- a/src/main/python/yamlreader/yamlreader.py +++ b/src/main/python/yamlreader/yamlreader.py @@ -1,14 +1,19 @@ -#!env python3 - +# $Id$ from __future__ import print_function, absolute_import, unicode_literals, division __version__ = '3.1.0' - -import ruamel.yaml as yaml -import json +#TODO rename file as __init__ +#import ruamel.yaml as yaml import sys import logging + +__defaults = dict( + debug=False, verbose=False, logfile='', loglevel=logging.ERROR, + json=False, merge=True, no_anchor=True, dup_keys=False, reverse=False, + sort_keys=False, sort_files=False, suffix='yaml', indent=2, loader='safe' + ) + logger = logging.getLogger(__name__) @@ -22,6 +27,7 @@ def __init__(self, msg, *args, **kwargs): level = args[0].upper() if isinstance(args[0], str) else args[0] #TODO case statement to generate/modify strings so it's not buried in multiple # places in code. eg. 'filenotfound' is easy case. msg == filename(s) + # TODO invoke via 'raise YamlReaderError(msg, level) from FileNotFoundError'? # try: super().__init__(msg) logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, @@ -33,7 +39,7 @@ def __init__(self, msg, *args, **kwargs): #TODO break out and differentiate as needed. some raise, others (all?) pass pass - +#TODO rename to just merge def data_merge(a, b, merge=True): """merges b into a and return merged result @@ -46,21 +52,23 @@ def data_merge(a, b, merge=True): import six key = None - #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) + #logger.trace('Attempting merge of "%s" into "%s"\n' % (b, a)) try: # border case for first run or if a is a primitive - if a is None or merge == False or \ - isinstance(a, (six.string_types, float, six.integer_types)): + if a is None or isinstance(a, (six.string_types, float, six.integer_types)): a = b elif isinstance(a, list): a.extend(b) if isinstance(b, list) else a.append(b) a = list(set(a)) elif isinstance(a, dict): - # dicts must be merged - if isinstance(b, dict): + if not merge: + a.update(b) + elif isinstance(b, dict): for key in b: a[key] = data_merge(a[key], b[key]) if key in a else b[key] else: + # XXX technically a Tuple or List of at least 2 wide + # could be used with [0] as key, [1] as value raise TypeError else: raise TypeError @@ -105,108 +113,128 @@ def get_files(source, suffix='yaml'): files = glob.glob(source) if len(files) == 0: - YamlReaderError('FileNotFoundError for %r' % (source), logging.WARNING) + YamlReaderError('FileNotFoundError for "%r"' % source, logging.WARNING) return files -def __main(): - import optparse, textwrap +def parse_cmdline(): + import optparse +# global options, files parser = optparse.OptionParser(usage="%prog [options] source ...", description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', version="%" + "prog %s" % __version__, prog='yamlreader') + parser.disable_interspersed_args() parser.add_option('-d', '--debug', dest='debug', - action='store_true', default=False, + action='store_true', default=__defaults['debug'], help="Enable debug logging (true *%default*)") parser.add_option('-v', '--verbose', dest='verbose', - action='store_true', default=False, - help="write progress (true *%default*)") + action='store_true', default=__defaults['verbose'], + help="show progress (true *%default*)") parser.add_option('-q', '--quiet', dest='verbose', action='store_false', help="minimize output (*True false)") parser.add_option('-l', '--logfile', dest='logfile', - action='store', default=None) + action='store', default=__defaults['logfile']) - parser.add_option('--log', dest='loglevel', - action='store', default='INFO', + parser.add_option('--level', dest='loglevel', + action='store', default=__defaults['loglevel'], help="(debug *%default warning error critical)") parser.add_option('-j', '--json', dest='json', - action='store_true', default=False, + action='store_true', default=__defaults['json'], help="output to JSON (true *%default)") parser.add_option('-m', '--merge', dest='merge', - action='store_true', default=True, + action='store_true', default=__defaults['merge'], help="merge a key's values (*%default false)") parser.add_option('--overwrite', dest='merge', action='store_false', - help="overwrite a key's values (true *false)") - # CloudFormation can't handle anchors or aliases (sep '17) + help="overwrite a key's values (true *False)") + + # CloudFormation can't handle anchors or aliases (sep '17) parser.add_option('-x', '--no-anchor', dest='no_anchor', - action='store_true', default=False, + action='store_true', default=__defaults['no_anchor'], help="unroll anchors/aliases (true *%default)") - parser.add_option('-u', '--duplicate-keys', dest='duplicate_keys', - action='store_true', default=False, + parser.add_option('-u', '--duplicate-keys', dest='dup_keys', + action='store_true', default=__defaults['dup_keys'], help="allow duplicate keys (true *%default)") parser.add_option('-r', '--reverse', dest='reverse', - action='store_true', default=False, + action='store_true', default=__defaults['reverse'], help="sort direction (true *%default)") parser.add_option('-k', '--sort-keys', dest='sort_keys', - action='store_true', default=False, + action='store_true', default=__defaults['sort_keys'], help="sort keys in dump (true *%default)") parser.add_option('--sort-files', dest='sort_files', - action='store_true', default=False, + action='store_true', default=__defaults['sort_files'], help="sort input filenames (true *%default)") parser.add_option('--suffix', dest='suffix', - action='store', default='yaml') + action='store', default=__defaults['suffix']) parser.add_option('-t', '--indent', dest='indent', - action='store', type=int, default=2, + action='store', type=int, default=__defaults['indent'], help="indent width (%default)") parser.add_option('--loader', dest='loader', - action='store', default='safe', + action='store', default=__defaults['loader'], help="loader class (base *%default roundtrip unsafe)") - try: - (options, args) = parser.parse_args() +# (options, files) = parser.parse_args() + return parser.parse_args() except Exception as e: parser.error(e) - # mangle options.loader for API consuption - options.loader= options.loader.lower() + +def main(options, *argv): + from optparse import Values + import ruamel.yaml as yaml + import json + + if isinstance(options, Values): + for k, v in __defaults.items(): + options.ensure_value(k, v) + elif options is None: + options = Values(__defaults) + elif isinstance(options, dict): + options = Values(__defaults.update(options)) + else: + raise YamlReaderError('TypeError - "options" (%s)' % type(options)) + return 1 + + # adjust 'loader' because Ruamel's cryptic short name if options.loader == 'roundtrip': options.loader = 'rt' + # adjust 'loglevel' since typing ALLCAPS is annoying if isinstance(options.loglevel, str): options.loglevel = options.loglevel.upper() - if options.logfile: log_handler = logging.FileHandler(options.logfile, mode='w') else: log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)9s %(message)s')) logger.addHandler(log_handler) logger.propagate = False + if options.debug: logger.setLevel(logging.DEBUG) else: - # in 3.2+ we can do - #logger.setLevel(options.loglevel) - logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) + logger.setLevel(options.loglevel) + #logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) # remove Traceback output in Verbose, squelch stacktrace all other times # if options.verbose: @@ -217,11 +245,12 @@ def __main(): # see http://yaml.readthedocs.io/en/latest/detail.html for examples data = new_data = None - files = get_files(args, options.suffix) + #TODO break this into a function. process_file()? + files = get_files(argv, options.suffix) if len(files) == 0: - #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(args)), 'CRITICAL') - raise YamlReaderError('No source files found! %s' % str(args), logging.FATAL) - # FATAL ==> not reached + #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(argv)), 'CRITICAL') + raise YamlReaderError('No source files found! %s' % str(argv), logging.ERROR) + return 2 indent = { 'mapping' : options.indent, @@ -232,11 +261,11 @@ def __main(): myaml = yaml.YAML(typ=options.loader) myaml.preserve_quotes=True myaml.default_flow_style=False - myaml.allow_duplicate_keys = options.duplicate_keys + myaml.allow_duplicate_keys = options.dup_keys myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) myaml.representer.ignore_aliases = lambda *args: True - # NOTICE! sort_keys is a noop unless using Matt's version of + # NOTICE! sort_keys is a NOOP unless using Matt's version of # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) if hasattr(myaml.representer, 'sort_keys'): myaml.representer.sort_keys = options.sort_keys @@ -249,7 +278,7 @@ def __main(): logger.debug('Payload: %r\n', new_data) except (yaml.MarkedYAMLError) as e: logger.warning('YAML.load() -- %s' % str(e)) - #raise YamlReaderError('YAML.load() -- %s' % str(e)) + #raise YamlReaderError('during load() of "%s"' % yaml_file) from e if new_data: # is not None: data = data_merge(data, new_data, options.merge) @@ -266,12 +295,13 @@ def __main(): else: myaml.dump(data, sys.stdout) except yaml.MarkedYAMLError as e: - raise YamlReaderError('YAML.dump() -- %s' % str(e)) + raise YamlReaderError('dump() -- %s' % str(e)) #YamlReaderError("YAML.dump(): %s" % str(e)) except (ValueError, OverflowError, TypeError): - # JSON dump might trigger + # JSON dump might trigger these pass - -if __name__ == "__main__": - sys.exit(__main()) + +if __name__ == '__main__': + (opts, args) = parse_cmdline() + sys.exit(main(opts, args)) From 163ef3ee03afd5fd6741bb5009c4d6a810b35784 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Thu, 12 Oct 2017 04:34:02 -0400 Subject: [PATCH 12/20] collapse into __init__. small tweaks to message formatting --- src/main/python/yamlreader/__init__.py | 303 +++++++++++++++++++++- src/main/python/yamlreader/yamlreader.py | 307 ----------------------- 2 files changed, 300 insertions(+), 310 deletions(-) delete mode 100644 src/main/python/yamlreader/yamlreader.py diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 3c24478..abc709d 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -1,7 +1,304 @@ # -*- coding: utf-8 -*- - from __future__ import print_function, absolute_import, unicode_literals, division -from .yamlreader import main, parse_cmdline, data_merge, get_files, YamlReaderError +__all__ = ['main', 'data_merge', 'YamlReaderError'] +__version__ = '3.1.0' + +import sys +import logging + +__defaults = dict( + debug=False, verbose=False, logfile='', loglevel=logging.ERROR, + json=False, merge=True, no_anchor=True, dup_keys=False, reverse=False, + sort_keys=False, sort_files=False, suffix='yaml', indent=2, loader='safe' + ) + +logger = logging.getLogger(__name__) + + +class YamlReaderError(Exception): + """write YAML processing errors to logger""" + #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT + def __init__(self, msg, *args, **kwargs): + level = logging.ERROR + + if args: + level = args[0].upper() if isinstance(args[0], str) else args[0] + #TODO case statement to generate/modify strings so it's not buried in multiple + # places in code. eg. 'filenotfound' is easy case. msg == filename(s) + # TODO invoke via 'raise YamlReaderError(msg, level) from FileNotFoundError'? + # try: + super().__init__(msg) + logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, + exc_info=(logger.getEffectiveLevel() == logging.DEBUG), **kwargs) + + if (level == logging.FATAL): + sys.exit(1) + + #TODO break out and differentiate as needed. some raise, others (all?) pass + pass + + +def data_merge(a, b, merge=True): + """merges b into a and return merged result + + based on http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge + and extended to also merge arrays (append) and dict keys replaced if having the same name. + + NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen + """ + + import six + key = None + + #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) + try: + # border case for first run or if a is a primitive + if a is None or isinstance(a, (six.string_types, float, six.integer_types)): + a = b + elif isinstance(a, list): + a.extend(b) if isinstance(b, list) else a.append(b) + a = list(set(a)) + elif isinstance(a, dict): + if not merge: + a.update(b) + elif isinstance(b, dict): + for key in b: + a[key] = data_merge(a[key], b[key]) if key in a else b[key] + else: + # XXX technically a Tuple or List of at least 2 wide + # could be used with [0] as key, [1] as value + raise TypeError + else: + raise TypeError + + except (TypeError, LookupError) as e: + raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % + (e, type(b), type(a), b, a), logging.WARNING) + # or str(e) also with e.__name__ ? + + return a + + +def get_files(source, suffix='yaml'): + """Examine path elements for files to processing + + source can be a file, a directory, a list/tuple of files or + a string containing a glob expression (with ?*[]). + + For a directory, filenames of $suffix will be read. + """ + + import os, glob + files = [] + + if source is None or len(source) == 0 or source == '-': + return [''] + + #if type(source) is list or type(source) is tuple: + if isinstance(source, list) or isinstance(source, tuple): + for item in source: + # iterate to expand list of potential dirs and files + files.extend(get_files(item, suffix)) + return files + + if os.path.isdir(source): + files = glob.glob(os.path.join(source, '*.' + suffix)) + elif os.path.isfile(source): + # turn single file into list + files = [source] + else: + # try to use the source as a glob + files = glob.glob(source) + + if len(files) == 0: + YamlReaderError('FileNotFoundError for %r' % source, logging.WARNING) + + return files + + +def parse_cmdline(): + import optparse + + parser = optparse.OptionParser(usage="%prog [options] source ...", + description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', + version="%" + "prog %s" % __version__, prog='yamlreader') + parser.disable_interspersed_args() + + parser.add_option('-d', '--debug', dest='debug', + action='store_true', default=__defaults['debug'], + help="Enable debug logging (true *%default*)") + + parser.add_option('-v', '--verbose', dest='verbose', + action='store_true', default=__defaults['verbose'], + help="show progress (true *%default*)") + + parser.add_option('-q', '--quiet', dest='verbose', + action='store_false', + help="minimize output (*True false)") + + parser.add_option('-l', '--logfile', dest='logfile', + action='store', default=__defaults['logfile']) + + parser.add_option('--level', dest='loglevel', + action='store', default=__defaults['loglevel'], + help="(debug *%default warning error critical)") + + parser.add_option('-j', '--json', dest='json', + action='store_true', default=__defaults['json'], + help="output to JSON (true *%default)") + + parser.add_option('-m', '--merge', dest='merge', + action='store_true', default=__defaults['merge'], + help="merge a key's values (*%default false)") + + parser.add_option('--overwrite', dest='merge', + action='store_false', + help="overwrite a key's values (true *False)") + + # CloudFormation can't handle anchors or aliases (sep '17) + parser.add_option('-x', '--no-anchor', dest='no_anchor', + action='store_true', default=__defaults['no_anchor'], + help="unroll anchors/aliases (true *%default)") + + parser.add_option('-u', '--duplicate-keys', dest='dup_keys', + action='store_true', default=__defaults['dup_keys'], + help="allow duplicate keys (true *%default)") + + parser.add_option('-r', '--reverse', dest='reverse', + action='store_true', default=__defaults['reverse'], + help="sort direction (true *%default)") + + parser.add_option('-k', '--sort-keys', dest='sort_keys', + action='store_true', default=__defaults['sort_keys'], + help="sort keys in dump (true *%default)") + + parser.add_option('--sort-files', dest='sort_files', + action='store_true', default=__defaults['sort_files'], + help="sort input filenames (true *%default)") + + parser.add_option('--suffix', dest='suffix', + action='store', default=__defaults['suffix']) + + parser.add_option('-t', '--indent', dest='indent', + action='store', type=int, default=__defaults['indent'], + help="indent width (%default)") + + parser.add_option('--loader', dest='loader', + action='store', default=__defaults['loader'], + help="loader class (base *%default roundtrip unsafe)") + + try: + return parser.parse_args() + except Exception as e: + parser.error(e) + + +def main(options, *argv): + from optparse import Values + import ruamel.yaml as yaml + import json + + if isinstance(options, Values): + for k, v in __defaults.items(): + options.ensure_value(k, v) + elif options is None: + options = Values(__defaults) + elif isinstance(options, dict): + options = Values(__defaults.update(options)) + else: + raise YamlReaderError('TypeError - "options" (%s)' % type(options)) + return 1 + + # adjust 'loader' because Ruamel's cryptic short name + if options.loader == 'roundtrip': + options.loader = 'rt' + + # adjust 'loglevel' since typing ALLCAPS is annoying + if isinstance(options.loglevel, str): + options.loglevel = options.loglevel.upper() + + if options.logfile: + log_handler = logging.FileHandler(options.logfile, mode='w') + else: + log_handler = logging.StreamHandler() + + log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)8s %(message)s')) + logger.addHandler(log_handler) + logger.propagate = False + + if options.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(options.loglevel) + #logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) + + # remove Traceback output in Verbose, squelch stacktrace all other times + # if options.verbose: + # sys.excepthook = lambda exctype,exc,traceback : print("{}: {}".format(exctype.__name__, exc)) + # else: + # sys.excepthook = lambda *args: None + + # see http://yaml.readthedocs.io/en/latest/detail.html for examples + data = new_data = None + + #TODO break this into a function. process_file()? + files = get_files(argv, options.suffix) + if len(files) == 0: + #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(argv)), 'CRITICAL') + raise YamlReaderError('No source files found! %s' % argv) + return 1 + + indent = { + 'mapping' : options.indent, + 'sequence': options.indent * 2, + 'offset' : options.indent + } + + myaml = yaml.YAML(typ=options.loader) + myaml.preserve_quotes=True + myaml.default_flow_style=False + myaml.allow_duplicate_keys = options.dup_keys + myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) + myaml.representer.ignore_aliases = lambda *args: True + + # NOTICE! sort_keys is a NOOP unless using Matt's version of + # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) + if hasattr(myaml.representer, 'sort_keys'): + myaml.representer.sort_keys = options.sort_keys + + for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: + if options.verbose: + logger.info('Reading file "%s"', yaml_file) + try: + new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) + logger.debug('Payload: %r\n', new_data) + except (yaml.MarkedYAMLError) as e: + logger.warning('YAML.load() -- %s' % str(e)) + #raise YamlReaderError('during load() of "%s"' % yaml_file) from e + + if new_data: # is not None: + data = data_merge(data, new_data, options.merge) + else: + logger.warning('No YAML data found in "%s"', yaml_file) + + if data is None or len(data) == 0: + logger.critical('No YAML data found anywhere!') + return 1 + + try: + if options.json: + json.dump(data, sys.stdout, indent=indent['mapping']) + else: + myaml.dump(data, sys.stdout) + except yaml.MarkedYAMLError as e: + raise YamlReaderError('dump() -- %s' % str(e)) + #YamlReaderError("YAML.dump(): %s" % str(e)) + except (ValueError, OverflowError, TypeError): + # JSON dump might trigger these + pass + -__all__ = ['main', 'data_merge', 'YamlReaderError'] +if __name__ == '__main__': + (opts, args) = parse_cmdline() + sys.exit(main(opts, args)) diff --git a/src/main/python/yamlreader/yamlreader.py b/src/main/python/yamlreader/yamlreader.py deleted file mode 100644 index b1c91f6..0000000 --- a/src/main/python/yamlreader/yamlreader.py +++ /dev/null @@ -1,307 +0,0 @@ -# $Id$ -from __future__ import print_function, absolute_import, unicode_literals, division - -__version__ = '3.1.0' -#TODO rename file as __init__ -#import ruamel.yaml as yaml -import sys -import logging - - -__defaults = dict( - debug=False, verbose=False, logfile='', loglevel=logging.ERROR, - json=False, merge=True, no_anchor=True, dup_keys=False, reverse=False, - sort_keys=False, sort_files=False, suffix='yaml', indent=2, loader='safe' - ) - -logger = logging.getLogger(__name__) - - -class YamlReaderError(Exception): - """write YAML processing errors to logger""" - #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT - def __init__(self, msg, *args, **kwargs): - level = logging.ERROR - - if args: - level = args[0].upper() if isinstance(args[0], str) else args[0] - #TODO case statement to generate/modify strings so it's not buried in multiple - # places in code. eg. 'filenotfound' is easy case. msg == filename(s) - # TODO invoke via 'raise YamlReaderError(msg, level) from FileNotFoundError'? - # try: - super().__init__(msg) - logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, - exc_info=(logger.getEffectiveLevel() == logging.DEBUG), **kwargs) - - if (level == logging.FATAL): - sys.exit(1) - - #TODO break out and differentiate as needed. some raise, others (all?) pass - pass - -#TODO rename to just merge -def data_merge(a, b, merge=True): - """merges b into a and return merged result - - based on http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge - and extended to also merge arrays (append) and dict keys replaced if having the same name. - - NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen - """ - - import six - key = None - - #logger.trace('Attempting merge of "%s" into "%s"\n' % (b, a)) - try: - # border case for first run or if a is a primitive - if a is None or isinstance(a, (six.string_types, float, six.integer_types)): - a = b - elif isinstance(a, list): - a.extend(b) if isinstance(b, list) else a.append(b) - a = list(set(a)) - elif isinstance(a, dict): - if not merge: - a.update(b) - elif isinstance(b, dict): - for key in b: - a[key] = data_merge(a[key], b[key]) if key in a else b[key] - else: - # XXX technically a Tuple or List of at least 2 wide - # could be used with [0] as key, [1] as value - raise TypeError - else: - raise TypeError - - except (TypeError, LookupError) as e: - raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % - (e, type(b), type(a), b, a), logging.WARNING) - # or str(e) also with e.__name__ ? - - return a - - -def get_files(source, suffix='yaml'): - """Examine path elements for files to processing - - source can be a file, a directory, a list/tuple of files or - a string containing a glob expression (with ?*[]). - - For a directory, filenames of $suffix will be read. - """ - - import os, glob - files = [] - - if source is None or len(source) == 0 or source == '-': - return [''] - - #if type(source) is list or type(source) is tuple: - if isinstance(source, list) or isinstance(source, tuple): - for item in source: - # iterate to expand list of potential dirs and files - files.extend(get_files(item, suffix)) - return files - - if os.path.isdir(source): - files = glob.glob(os.path.join(source, '*.' + suffix)) - elif os.path.isfile(source): - # turn single file into list - files = [source] - else: - # try to use the source as a glob - files = glob.glob(source) - - if len(files) == 0: - YamlReaderError('FileNotFoundError for "%r"' % source, logging.WARNING) - - return files - - -def parse_cmdline(): - import optparse -# global options, files - - parser = optparse.OptionParser(usage="%prog [options] source ...", - description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', - version="%" + "prog %s" % __version__, prog='yamlreader') - parser.disable_interspersed_args() - - parser.add_option('-d', '--debug', dest='debug', - action='store_true', default=__defaults['debug'], - help="Enable debug logging (true *%default*)") - - parser.add_option('-v', '--verbose', dest='verbose', - action='store_true', default=__defaults['verbose'], - help="show progress (true *%default*)") - - parser.add_option('-q', '--quiet', dest='verbose', - action='store_false', - help="minimize output (*True false)") - - parser.add_option('-l', '--logfile', dest='logfile', - action='store', default=__defaults['logfile']) - - parser.add_option('--level', dest='loglevel', - action='store', default=__defaults['loglevel'], - help="(debug *%default warning error critical)") - - parser.add_option('-j', '--json', dest='json', - action='store_true', default=__defaults['json'], - help="output to JSON (true *%default)") - - parser.add_option('-m', '--merge', dest='merge', - action='store_true', default=__defaults['merge'], - help="merge a key's values (*%default false)") - - parser.add_option('--overwrite', dest='merge', - action='store_false', - help="overwrite a key's values (true *False)") - - # CloudFormation can't handle anchors or aliases (sep '17) - parser.add_option('-x', '--no-anchor', dest='no_anchor', - action='store_true', default=__defaults['no_anchor'], - help="unroll anchors/aliases (true *%default)") - - parser.add_option('-u', '--duplicate-keys', dest='dup_keys', - action='store_true', default=__defaults['dup_keys'], - help="allow duplicate keys (true *%default)") - - parser.add_option('-r', '--reverse', dest='reverse', - action='store_true', default=__defaults['reverse'], - help="sort direction (true *%default)") - - parser.add_option('-k', '--sort-keys', dest='sort_keys', - action='store_true', default=__defaults['sort_keys'], - help="sort keys in dump (true *%default)") - - parser.add_option('--sort-files', dest='sort_files', - action='store_true', default=__defaults['sort_files'], - help="sort input filenames (true *%default)") - - parser.add_option('--suffix', dest='suffix', - action='store', default=__defaults['suffix']) - - parser.add_option('-t', '--indent', dest='indent', - action='store', type=int, default=__defaults['indent'], - help="indent width (%default)") - - parser.add_option('--loader', dest='loader', - action='store', default=__defaults['loader'], - help="loader class (base *%default roundtrip unsafe)") - - try: -# (options, files) = parser.parse_args() - return parser.parse_args() - except Exception as e: - parser.error(e) - - -def main(options, *argv): - from optparse import Values - import ruamel.yaml as yaml - import json - - if isinstance(options, Values): - for k, v in __defaults.items(): - options.ensure_value(k, v) - elif options is None: - options = Values(__defaults) - elif isinstance(options, dict): - options = Values(__defaults.update(options)) - else: - raise YamlReaderError('TypeError - "options" (%s)' % type(options)) - return 1 - - # adjust 'loader' because Ruamel's cryptic short name - if options.loader == 'roundtrip': - options.loader = 'rt' - - # adjust 'loglevel' since typing ALLCAPS is annoying - if isinstance(options.loglevel, str): - options.loglevel = options.loglevel.upper() - - if options.logfile: - log_handler = logging.FileHandler(options.logfile, mode='w') - else: - log_handler = logging.StreamHandler() - - log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)9s %(message)s')) - logger.addHandler(log_handler) - logger.propagate = False - - if options.debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(options.loglevel) - #logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) - - # remove Traceback output in Verbose, squelch stacktrace all other times - # if options.verbose: - # sys.excepthook = lambda exctype,exc,traceback : print("{}: {}".format(exctype.__name__, exc)) - # else: - # sys.excepthook = lambda *args: None - - # see http://yaml.readthedocs.io/en/latest/detail.html for examples - data = new_data = None - - #TODO break this into a function. process_file()? - files = get_files(argv, options.suffix) - if len(files) == 0: - #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(argv)), 'CRITICAL') - raise YamlReaderError('No source files found! %s' % str(argv), logging.ERROR) - return 2 - - indent = { - 'mapping' : options.indent, - 'sequence': options.indent * 2, - 'offset' : options.indent - } - - myaml = yaml.YAML(typ=options.loader) - myaml.preserve_quotes=True - myaml.default_flow_style=False - myaml.allow_duplicate_keys = options.dup_keys - myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) - myaml.representer.ignore_aliases = lambda *args: True - - # NOTICE! sort_keys is a NOOP unless using Matt's version of - # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) - if hasattr(myaml.representer, 'sort_keys'): - myaml.representer.sort_keys = options.sort_keys - - for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: - if options.verbose: - logger.info('Reading file "%s"', yaml_file) - try: - new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) - logger.debug('Payload: %r\n', new_data) - except (yaml.MarkedYAMLError) as e: - logger.warning('YAML.load() -- %s' % str(e)) - #raise YamlReaderError('during load() of "%s"' % yaml_file) from e - - if new_data: # is not None: - data = data_merge(data, new_data, options.merge) - else: - logger.warning('No YAML data found in "%s"', yaml_file) - - if data is None or len(data) == 0: - logger.critical('No YAML data found anywhere!') - return 1 - - try: - if options.json: - json.dump(data, sys.stdout, indent=indent['mapping']) - else: - myaml.dump(data, sys.stdout) - except yaml.MarkedYAMLError as e: - raise YamlReaderError('dump() -- %s' % str(e)) - #YamlReaderError("YAML.dump(): %s" % str(e)) - except (ValueError, OverflowError, TypeError): - # JSON dump might trigger these - pass - - -if __name__ == '__main__': - (opts, args) = parse_cmdline() - sys.exit(main(opts, args)) From 88f1108b1be56e8f9c7ba0ad3bec4a5b40e8cce5 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Thu, 12 Oct 2017 05:26:39 -0400 Subject: [PATCH 13/20] facilitate direct execution of module as script --- bin/yamlreader | 12 ++++++++++++ build.py | 6 +++--- src/main/python/yamlreader/__main__.py | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 bin/yamlreader create mode 100644 src/main/python/yamlreader/__main__.py diff --git a/bin/yamlreader b/bin/yamlreader new file mode 100644 index 0000000..1888a67 --- /dev/null +++ b/bin/yamlreader @@ -0,0 +1,12 @@ +#!/usr/bin/python3 + +# -*- coding: utf-8 -*- +import re +import sys + +sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) +from yamlreader import parse_cmdline, main + +(opts, args) = parse_cmdline() +sys.exit(main(opts, args)) + diff --git a/build.py b/build.py index 81ed8ce..bcebdf7 100644 --- a/build.py +++ b/build.py @@ -13,7 +13,7 @@ summary = 'Merge YAML data from given files, dir or file glob' authors = [Author('Schlomo Schapiro', "schlomo.schapiro@immobilienscout24.de")] url = 'https://github.com/ImmobilienScout24/yamlreader' -version = '3.0.4' +version = '3.1.0' description = open("README.rst").read() license = 'Apache License 2.0' @@ -22,10 +22,10 @@ @init def set_properties(project): - project.depends_on("PyYAML") + project.depends_on("ruamel.yaml") project.depends_on("six") - project.set_property('distutils_console_scripts', ['yamlreader=yamlreader.yamlreader:__main']) + project.set_property('distutils_console_scripts', ['yamlreader=yamlreader:__main__']) project.set_property("distutils_classifiers", [ "Programming Language :: Python", diff --git a/src/main/python/yamlreader/__main__.py b/src/main/python/yamlreader/__main__.py new file mode 100644 index 0000000..14584c6 --- /dev/null +++ b/src/main/python/yamlreader/__main__.py @@ -0,0 +1,5 @@ +import sys +from __init__ import parse_cmdline, main + +(opts, args) = parse_cmdline() +sys.exit(main(opts, args)) From 5999269a127e43c5088b4afd644be87a6ee43c9c Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Sat, 14 Oct 2017 17:03:03 -0400 Subject: [PATCH 14/20] intermediate save on major logging and backwards compatability refactor --- src/main/python/yamlreader/__init__.py | 465 +++++++++++++++++-------- src/main/python/yamlreader/__main__.py | 7 +- src/main/python/yamlreader/logging.py | 132 +++++++ 3 files changed, 449 insertions(+), 155 deletions(-) create mode 100644 src/main/python/yamlreader/logging.py diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index abc709d..49b5d37 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -1,42 +1,92 @@ # -*- coding: utf-8 -*- from __future__ import print_function, absolute_import, unicode_literals, division -__all__ = ['main', 'data_merge', 'YamlReaderError'] -__version__ = '3.1.0' +__all__ = ['yaml_load', 'data_merge', 'YamlReaderError'] +__version__ = '4.0.0' -import sys +import os, sys +#FIXME optparse deprecated in favor of argparse!! +import optparse import logging +from .yrlogging import * + + +# see http://yaml.readthedocs.io/en/latest/overview.html +import ruamel.yaml as yaml __defaults = dict( - debug=False, verbose=False, logfile='', loglevel=logging.ERROR, - json=False, merge=True, no_anchor=True, dup_keys=False, reverse=False, - sort_keys=False, sort_files=False, suffix='yaml', indent=2, loader='safe' + debug = False, + verbose = False, + quiet = False, + ignore_error = True, + logfile = None, + log_format = '%(levelname)-8s "%(message)s"', + log_level = logging.INFO, + console_format = None, + console_level = logging.ERROR, + file_format = None, + file_level = logging.INFO, + merge = True, + no_anchor = True, + dup_keys = True, + sort_keys = False, + sort_files = True, # v3.0 backward-compat + reverse = False, + json = False, + suffix = 'yaml', + indent = 2, + loader = 'safe' ) - +options = optparse.Values(__defaults) +options.console_format = '%s: ' % __name__ + options.log_format + +yaml_loaders = ['safe', 'base', 'rt', 'unsafe'] +#XXX use **dict to convert to kwargs +__yaml_defaults = dict( + preserve_quotes = True, + default_flow_style = False, + # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences + indent = {} + ) +#XXX +print("my name is %s" % __name__) logger = logging.getLogger(__name__) +logger.propagate = False + +myaml = None class YamlReaderError(Exception): """write YAML processing errors to logger""" + #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT - def __init__(self, msg, *args, **kwargs): - level = logging.ERROR + def __init__(self, msg, rc=os.EX_SOFTWARE, level=logging.ERROR): #*args, **kwargs): + + # send_to_logger = False + + # for handle in logger.get(handlers): + # if isinstance(handle, logging.FileHandler): + # send_to_logger = True + # break + # if isinstance(level, str): + # level = getLevel(level) - if args: - level = args[0].upper() if isinstance(args[0], str) else args[0] #TODO case statement to generate/modify strings so it's not buried in multiple # places in code. eg. 'filenotfound' is easy case. msg == filename(s) # TODO invoke via 'raise YamlReaderError(msg, level) from FileNotFoundError'? - # try: - super().__init__(msg) - logger.log(level, '%s::%s', sys._getframe().f_back.f_code.co_name, msg, - exc_info=(logger.getEffectiveLevel() == logging.DEBUG), **kwargs) - - if (level == logging.FATAL): - sys.exit(1) + super().__init__(msg) + frame = sys._getframe().f_back.f_code.co_name #TODO break out and differentiate as needed. some raise, others (all?) pass - pass + + if level > logging.CRITICAL or options.ignore_error == False: + # restore default exception formatting + sys.excepthook = sys.__excepthook__ + logger.log(level, '%s::%s', frame, msg, exc_info=True) + # mimic signals.h SIGTERM, or use os.EX_* + sys.exit(128+rc) + + logger.log(level, '%s::%s', frame, msg, exc_info=(options.verbose or options.debug)) def data_merge(a, b, merge=True): @@ -74,21 +124,19 @@ def data_merge(a, b, merge=True): except (TypeError, LookupError) as e: raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % - (e, type(b), type(a), b, a), logging.WARNING) + (e, type(b), type(a), b, a), logging.WARNING) from e # or str(e) also with e.__name__ ? return a def get_files(source, suffix='yaml'): - """Examine path elements for files to processing - - source can be a file, a directory, a list/tuple of files or - a string containing a glob expression (with ?*[]). + """Examine pathspec elements for files to process - For a directory, filenames of $suffix will be read. + 'source' can be a filename, directory, a list/tuple of same, + or a glob expression with wildcard notion (?*[]). + If a directory, filenames ending in $suffix will be chosen. """ - import os, glob files = [] @@ -112,193 +160,306 @@ def get_files(source, suffix='yaml'): files = glob.glob(source) if len(files) == 0: - YamlReaderError('FileNotFoundError for %r' % source, logging.WARNING) + # TODO what is suitable error level? + # if options.ignore_error , use ERROR, otherwise WARNING + level = logging.WARNING if options.ignore_error else logging.ERROR + YamlReaderError('FileNotFoundError for %r' % source, + rc=os.EX_OSFILE, + level=level) return files def parse_cmdline(): - import optparse + """Process command-line options""" - parser = optparse.OptionParser(usage="%prog [options] source ...", + usage = "%prog [options] source ..." + parser = optparse.OptionParser(usage, description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', version="%" + "prog %s" % __version__, prog='yamlreader') + parser.disable_interspersed_args() parser.add_option('-d', '--debug', dest='debug', action='store_true', default=__defaults['debug'], - help="Enable debug logging (true *%default*)") + help="enable debugging %default") parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=__defaults['verbose'], - help="show progress (true *%default*)") + help="extra messages %default") - parser.add_option('-q', '--quiet', dest='verbose', - action='store_false', - help="minimize output (*True false)") + parser.add_option('-q', '--quiet', dest='quiet', + action='store_true', default=__defaults['quiet'], + help="minimize output %default") + + parser.add_option('-c', '--continue', dest='ignore_error', + action='store_true', default=__defaults['ignore_error'], + help="errors as not fatal %default") parser.add_option('-l', '--logfile', dest='logfile', action='store', default=__defaults['logfile']) - parser.add_option('--level', dest='loglevel', - action='store', default=__defaults['loglevel'], - help="(debug *%default warning error critical)") + #TODO log_format = '%(levelname)8s "%(message)s"', - parser.add_option('-j', '--json', dest='json', - action='store_true', default=__defaults['json'], - help="output to JSON (true *%default)") + parser.add_option('--log-level', dest='log_level', + action='store', default=__defaults['log_level'], + help=' '.join(logging._nameToLevel.keys()), + choices=list(logging._nameToLevel.keys())) - parser.add_option('-m', '--merge', dest='merge', - action='store_true', default=__defaults['merge'], - help="merge a key's values (*%default false)") + parser.add_option('--console-level', dest='console_level', + action='store', default=__defaults['console_level'], + help=" %default ") - parser.add_option('--overwrite', dest='merge', - action='store_false', - help="overwrite a key's values (true *False)") + parser.add_option('--file-level', dest='file_level', + action='store', default=__defaults['file_level']) - # CloudFormation can't handle anchors or aliases (sep '17) - parser.add_option('-x', '--no-anchor', dest='no_anchor', - action='store_true', default=__defaults['no_anchor'], - help="unroll anchors/aliases (true *%default)") + parser.add_option('-M', '--overwrite', dest='merge', + action='store_false', default=not __defaults['merge'], + help="overwrite keys %default") - parser.add_option('-u', '--duplicate-keys', dest='dup_keys', - action='store_true', default=__defaults['dup_keys'], - help="allow duplicate keys (true *%default)") + # CloudFormation can't handle anchors or aliases in final output + parser.add_option('-X', '--no-anchor', dest='no_anchor', + action='store_true', default=__defaults['no_anchor'], + help="unroll anchors %default") - parser.add_option('-r', '--reverse', dest='reverse', - action='store_true', default=__defaults['reverse'], - help="sort direction (true *%default)") + parser.add_option('-u', '--unique-keys', dest='dup_keys', + action='store_false', default=__defaults['dup_keys'], + help="skip duplicates %default") parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=__defaults['sort_keys'], - help="sort keys in dump (true *%default)") + help="sort keys %default") + + parser.add_option('-S', '--no-sort-files', dest='sort_files', + action='store_false', default=__defaults['sort_files'], + help="sort filenames %default") + + parser.add_option('-r', '--reverse', dest='reverse', + action='store_true', default=__defaults['reverse'], + help="sort direction %default") - parser.add_option('--sort-files', dest='sort_files', - action='store_true', default=__defaults['sort_files'], - help="sort input filenames (true *%default)") + parser.add_option('-j', '--json', dest='json', + action='store_true', default=__defaults['json'], + help="output as JSON %default") parser.add_option('--suffix', dest='suffix', - action='store', default=__defaults['suffix']) + action='store', default=__defaults['suffix'], + help="filename suffix '%default'") + #TODO - defaults for Yaml constructor. + # move loader and indent into __yaml_defaults and prepend name with 'yaml' parser.add_option('-t', '--indent', dest='indent', action='store', type=int, default=__defaults['indent'], - help="indent width (%default)") + help=" %default") parser.add_option('--loader', dest='loader', action='store', default=__defaults['loader'], - help="loader class (base *%default roundtrip unsafe)") - + help="%s %s" % (' '.join(yaml_loaders), __defaults['loader']), + choices=yaml_loaders) +# | %default try: return parser.parse_args() - except Exception as e: - parser.error(e) + #FIXME figure out what to trap + except Exception as ex: + parser.error(ex) -def main(options, *argv): - from optparse import Values - import ruamel.yaml as yaml - import json +def _newYaml(): + #TODO use kwargs or module defaults? + global myaml + + try: + if not isinstance(myaml, yaml.YAML): + myaml = yaml.YAML(typ=options.loader) + + # useful defaults for AWS CloudFormation + myaml.preserve_quotes=True + myaml.default_flow_style=False + myaml.allow_duplicate_keys = options.dup_keys + myaml.representer.ignore_aliases = lambda *args: True + + # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences + myaml.indent = dict( + mapping = options.indent, + sequence = options.indent * 2, + offset = options.indent + ) + #FIXME what can YAML() throw? need to catch Math error, possibly Type and ValueError + except Exception as ex: + raise YamlReaderError('XXX') from ex + + +def yaml_load(source, defaultdata=None): + """merge YAML data from files found in source + + Always returns a dict. The files are read with the 'safe' loader + though the other 3 options are possible. + + 'source' can be a file, a dir, a list/tuple of files or a string containing + a glob expression (with ?*[]). + For a directory, all *.yaml files will be read in alphabetical order. + """ + global myaml - if isinstance(options, Values): - for k, v in __defaults.items(): - options.ensure_value(k, v) - elif options is None: - options = Values(__defaults) - elif isinstance(options, dict): - options = Values(__defaults.update(options)) - else: - raise YamlReaderError('TypeError - "options" (%s)' % type(options)) - return 1 + logger.debug("yaml_load() initialized with source='%s', defaultdata='%s'", source, defaultdata) + _newYaml() + + # NOTICE - sort_keys is a NOOP unless Matt's version of + # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) + if hasattr(myaml.representer, 'sort_keys'): + myaml.representer.sort_keys = options.sort_keys - # adjust 'loader' because Ruamel's cryptic short name - if options.loader == 'roundtrip': - options.loader = 'rt' + files = get_files(source, options.suffix) + if len(files) == 0: + raise YamlReaderError('FileNotFoundError for %s' % source) + return None - # adjust 'loglevel' since typing ALLCAPS is annoying - if isinstance(options.loglevel, str): - options.loglevel = options.loglevel.upper() + data = defaultdata + for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: + if options.verbose: + logger.debug("processing '%s'...", yaml_file) - if options.logfile: - log_handler = logging.FileHandler(options.logfile, mode='w') - else: - log_handler = logging.StreamHandler() + try: + new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) + logger.debug('payload: %r\n', new_data) + except yaml.MarkedYAMLError as ex: + YamlReaderError('during YAML.load() of %s' % yaml_file, + rc=os.EX_DATAERR, level=getLevel('NOTICE')) + except: + #FIXME what to do? + pass + + if new_data: + data = data_merge(data, new_data, options.merge) + elif options.verbose: + #XXX rc=os.EX_NOINPUT, actually os.EX_DATAERR + logger.info("no YAML data in %s", yaml_file) - log_handler.setFormatter(logging.Formatter('yamlreader: %(levelname)8s %(message)s')) - logger.addHandler(log_handler) - logger.propagate = False + return data - if options.debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(options.loglevel) - #logger.setLevel(getattr(logging, options.loglevel, logging.INFO)) + +def __main(opts, *argv): + import json + global options - # remove Traceback output in Verbose, squelch stacktrace all other times - # if options.verbose: - # sys.excepthook = lambda exctype,exc,traceback : print("{}: {}".format(exctype.__name__, exc)) - # else: - # sys.excepthook = lambda *args: None + #TODO split varification into separate helper? getting too long. + try: + # merge __defaults + user-supplied into 'options' + if isinstance(opts, optparse.Values): + kv = vars(opts) + elif isinstance(opts, dict): + kv = opts + elif opts is None: + kv = {} + else: + # too early for YamlReaderError + raise TypeError("%s not supported for parameter 'opts'" % type(opts)) - # see http://yaml.readthedocs.io/en/latest/detail.html for examples - data = new_data = None + print(kv) #XXX - #TODO break this into a function. process_file()? - files = get_files(argv, options.suffix) - if len(files) == 0: - #raise YamlReaderError('%s: "%s"' % (FileNotFoundError.__name__, str(argv)), 'CRITICAL') - raise YamlReaderError('No source files found! %s' % argv) - return 1 - - indent = { - 'mapping' : options.indent, - 'sequence': options.indent * 2, - 'offset' : options.indent - } - - myaml = yaml.YAML(typ=options.loader) - myaml.preserve_quotes=True - myaml.default_flow_style=False - myaml.allow_duplicate_keys = options.dup_keys - myaml.indent(mapping=indent['mapping'], sequence=indent['sequence'], offset=indent['offset']) - myaml.representer.ignore_aliases = lambda *args: True - - # NOTICE! sort_keys is a NOOP unless using Matt's version of - # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) - if hasattr(myaml.representer, 'sort_keys'): - myaml.representer.sort_keys = options.sort_keys + for k, v in kv.items(): + options.ensure_value(k, v) - for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: - if options.verbose: - logger.info('Reading file "%s"', yaml_file) + if not (options.log_level and options.log_format): + raise ValueError('options.log_* can not be blank') + #TODO check other fields which can't be blank + + except Exception as ex: + logger.critical("%s while merging 'opts' into 'options'.\n %r\n %r", + ex.__name__, opts, options) + return os.EX_CONFIG + + + # adjust 'loader' because Ruamel's cryptic short name + if options.loader == 'roundtrip': + options.loader = 'rt' + + # normalize logging 'levels' and upcase for downstream lookups + for attr in (s + '_level' for s in ['log', 'console', 'file']): try: - new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) - logger.debug('Payload: %r\n', new_data) - except (yaml.MarkedYAMLError) as e: - logger.warning('YAML.load() -- %s' % str(e)) - #raise YamlReaderError('during load() of "%s"' % yaml_file) from e + setattr(options, attr, str.upper(getattr(options,attr))) + except TypeError: + pass - if new_data: # is not None: - data = data_merge(data, new_data, options.merge) + if options.debug: + options.loglevel = logging.DEBUG # getLevel('DEBUG') + # reset to trigger Handler-specific override + options.console_level = options.file_level = None + options.verbose = True + logger.setlevel(logging.DEBUG) + + # override/set Handler-specific levels from parent + if not options.console_level: + options.console_level = options.log_level + if not options.file_level: + options.file_level = options.log_level + + if not options.console_format: + options.console_format = options.log_format + if not options.file_format: + options.file_level = options.log_format + + if not options.quiet: + try: + console_handler = logging.StreamHandler() + console_handler.setFormatter(logging.Formatter(options.console_format)) + console_handler.setLevel(options.console_level) + except: #FIXME what to trap? + msg='' + raise #TODO logger.error() + if not options.ignore_error: + return os.EX_CONFIG + else: + logger.addHandler(console_handler) + + if options.logfile: + try: + file_handler = logging.FileHandler(options.logfile, mode='w') + file_handler.setFormatter(logging.Formatter(options.file_format)) + file_handler.setLevel(options.file_level) + except (FileNotFoundError, OSError): #TODO what else? rc=os.EX_OSFILE + options.logfile = None + msg='' + raise #TODO logger.error() + if options.ignore_error: + return os.EX_CONFIG else: - logger.warning('No YAML data found in "%s"', yaml_file) + logger.addHandler(file_handler) + + # squelch stacktrace if quiet. This affects all handlers, however which + # wasn't the intent - just keep the console clear and not duplicate + # exception strings. TODO + if options.quiet: + sys.excepthook = lambda *args: None + elif options.debug: + pass + else: + sys.excepthook = lambda exctype, exc, traceback : print("{}: {}".format(exctype.__name__, exc)) + + # Finally ready to do useful work! + data = yaml_load(argv) if data is None or len(data) == 0: - logger.critical('No YAML data found anywhere!') - return 1 + # a NOOP is not an error, but no point going further + if option.verbose: + logger.info('No YAML data found at all!') + return os.EX_NOINPUT try: if options.json: - json.dump(data, sys.stdout, indent=indent['mapping']) + json.dump(data, sys.stdout, options.indent) else: myaml.dump(data, sys.stdout) - except yaml.MarkedYAMLError as e: - raise YamlReaderError('dump() -- %s' % str(e)) - #YamlReaderError("YAML.dump(): %s" % str(e)) - except (ValueError, OverflowError, TypeError): + #TODO combine logging, no need to raise since no external caller. + #except yaml.MarkedYAMLError as ex: + #except (ValueError, OverflowError, TypeError): json.dump()? + except Exception as ex: + logger.warning('%s while dump()' % ex.__name__) # JSON dump might trigger these - pass - + return os.EX_DATAERR + + return os.EX_OK -if __name__ == '__main__': - (opts, args) = parse_cmdline() - sys.exit(main(opts, args)) +# if __name__ == '__main__': + # (opts, args) = parse_cmdline() + # sys.exit(__main(opts, args)) diff --git a/src/main/python/yamlreader/__main__.py b/src/main/python/yamlreader/__main__.py index 14584c6..4182df5 100644 --- a/src/main/python/yamlreader/__main__.py +++ b/src/main/python/yamlreader/__main__.py @@ -1,5 +1,6 @@ import sys -from __init__ import parse_cmdline, main +from . import parse_cmdline, __main -(opts, args) = parse_cmdline() -sys.exit(main(opts, args)) +if __name__ == '__main__': + (opts, args) = parse_cmdline() + sys.exit(__main(opts, args)) diff --git a/src/main/python/yamlreader/logging.py b/src/main/python/yamlreader/logging.py new file mode 100644 index 0000000..50d9a0b --- /dev/null +++ b/src/main/python/yamlreader/logging.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function, absolute_import, unicode_literals, division + +__all__ = ['getLevelName', 'getLevel'] #, 'getLevelOrName', '_checkLevel'] + +import logging + +# private re-implementations till Python Core fixes Lib/logging +# XXX bug numbers here + +# define missing syslog(3) levels and also handy helpers +logging.addLevelName(logging.NOTSET, 'ALL') +logging.addLevelName(logging.DEBUG - 5, 'TRACE') +logging.addLevelName(logging.INFO + 5, 'NOTICE') +# fix Lib/logging improperly conflating CRITICAL and FATAL +logging.addLevelName(logging.CRITICAL + 1, 'FATAL') +logging.addLevelName(logging.CRITICAL + 10, 'ALERT') +logging.addLevelName(logging.CRITICAL + 20, 'EMERG') +logging.addLevelName(logging.CRITICAL + 99, 'ABORT') + + +def getLevelName(level, format='%s', no_match=None): +# strict={'case': False, 'type': False, 'map': False}, +# fixup=False + """Return the textual representation of 'level'. + + Whether predefined (eg. CRITICAL -> "CRITICAL") or user-defined via + addLevelName(), the string associated with 'level' is chosen. + Otherwise, 'level' (no_match == NONE) or 'no_match' is returned + subject to formatting per 'format'. + + In the spirit of "be liberal in what you accept", any value of 'level' + that survives int() will be accepted (FUTURE: subject to 'strict'). + + Issue #29220 introduced the BAD IDEA that passing an empty string + (an obvious TypeError) would return same. This was requested in order + to squash the fall-thru behavior of returning "Level %s", when the + multi-word response was itself the actual ERROR since it broke all + field-based log processing! The astute reader will note that an empty + string causes the same pathology... + + DEPRECATION WARNING: + This function WRONGLY returned the mapped Integer if a String form + was provided. This violates the clearly stated purpose and forces + the caller into defensive Type checks or suffer future TypeErrors. + + NOTE: + Does no bounds or validity checks. Use _checkLevel(). + + FUTURE: + In strict mode, enforce parameter dataType, case, or membership. + """ + + try: + # check Name->Level in case called incorrectly (backward compat) + if level in _nameToLevel: + return format % level + + # retval = _checkLevel(level, flags, fix=T/F) + # if isinstance(retval, bool) then handle pass/fail, else update level with fixed value + + result = _levelToName.get(int(level)) + if result is not None: + return format % result + + except TypeError: + if raiseExceptions: + raise("parameter 'level' must reduce to an Integer") + except ValueError: + pass + + return format % level if no_match is None else format % no_match + + +def getLevel(levelName, no_match=logging.NOTSET): +# strict={'case': False, 'type': False, 'map': False}, +# fixup=False + """Return the numeric representation of levelName. + + see getLevelName() for background + """ + try: + result = _nameToLevel.get(levelName) + if result is not None: + return result + + return int(levelName) + + except ValueError: + if raiseExceptions: + raise("parameter 'levelName' must be a defined String") + + return no_match + + +def getLevelOrName(level): + pass + + +def _checkLevel(level): + pass +# # strict={'case': False, 'type': False, 'map': False}, + + # """Check parameter against defined values. + # ???Return NOTSET if invalid. + + # Since all logging.$level() functions choose to emit based on + # numeric comparison, a default of ERROR would be more friendly. + # """ + # rv = NOTSET + # try: + # if level in _nameToLevel: + # rv = _nameToLevel[level] + # elif level in _levelToName: + # rv = level + # else: + # #FIXME - test harness injects '+1', so tolerating + # # arbitrary integers is expected behavior. Why? + # # raise ValueError + # rv = int(level) + # except (TypeError, ValueError, KeyError) as err: + # if raiseExceptions: + # # test harness (../test/test_logging) expects 'TypeError' + # raise TypeError("Level not an integer or a valid string: %r" % level) from err + # except Exception: + # pass + + # return rv + + + + From a14e4ca7eeac59fccc08eb1e5c504f8ae6c32b6e Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Sat, 14 Oct 2017 17:07:10 -0400 Subject: [PATCH 15/20] sync with __init__ to allow calling package directly --- bin/yamlreader | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/yamlreader b/bin/yamlreader index 1888a67..44641bf 100644 --- a/bin/yamlreader +++ b/bin/yamlreader @@ -1,12 +1,13 @@ #!/usr/bin/python3 -# -*- coding: utf-8 -*- import re import sys +import os sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) -from yamlreader import parse_cmdline, main +from yamlreader import parse_cmdline, __main -(opts, args) = parse_cmdline() -sys.exit(main(opts, args)) +if __name__ == '__main__': + (opts, args) = parse_cmdline() + sys.exit(__main(opts, args)) From d03ded85e06396b7e67639005792e4cb8f1c6fea Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Sat, 14 Oct 2017 17:07:59 -0400 Subject: [PATCH 16/20] trim bad whitespace --- src/main/python/yamlreader/__init__.py | 29 ++++++++++++------------- src/main/python/yamlreader/logging.py | 30 +++++++++++++------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 49b5d37..a982401 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -48,8 +48,7 @@ # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences indent = {} ) -#XXX -print("my name is %s" % __name__) + logger = logging.getLogger(__name__) logger.propagate = False @@ -61,7 +60,7 @@ class YamlReaderError(Exception): #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT def __init__(self, msg, rc=os.EX_SOFTWARE, level=logging.ERROR): #*args, **kwargs): - + # send_to_logger = False # for handle in logger.get(handlers): @@ -85,7 +84,7 @@ def __init__(self, msg, rc=os.EX_SOFTWARE, level=logging.ERROR): #*args, **kwarg logger.log(level, '%s::%s', frame, msg, exc_info=True) # mimic signals.h SIGTERM, or use os.EX_* sys.exit(128+rc) - + logger.log(level, '%s::%s', frame, msg, exc_info=(options.verbose or options.debug)) @@ -123,7 +122,7 @@ def data_merge(a, b, merge=True): raise TypeError except (TypeError, LookupError) as e: - raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % + raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % (e, type(b), type(a), b, a), logging.WARNING) from e # or str(e) also with e.__name__ ? @@ -174,7 +173,7 @@ def parse_cmdline(): """Process command-line options""" usage = "%prog [options] source ..." - parser = optparse.OptionParser(usage, + parser = optparse.OptionParser(usage, description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', version="%" + "prog %s" % __version__, prog='yamlreader') @@ -267,7 +266,7 @@ def parse_cmdline(): def _newYaml(): #TODO use kwargs or module defaults? global myaml - + try: if not isinstance(myaml, yaml.YAML): myaml = yaml.YAML(typ=options.loader) @@ -291,10 +290,10 @@ def _newYaml(): def yaml_load(source, defaultdata=None): """merge YAML data from files found in source - + Always returns a dict. The files are read with the 'safe' loader though the other 3 options are possible. - + 'source' can be a file, a dir, a list/tuple of files or a string containing a glob expression (with ?*[]). For a directory, all *.yaml files will be read in alphabetical order. @@ -303,7 +302,7 @@ def yaml_load(source, defaultdata=None): logger.debug("yaml_load() initialized with source='%s', defaultdata='%s'", source, defaultdata) _newYaml() - + # NOTICE - sort_keys is a NOOP unless Matt's version of # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) if hasattr(myaml.representer, 'sort_keys'): @@ -323,7 +322,7 @@ def yaml_load(source, defaultdata=None): new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) logger.debug('payload: %r\n', new_data) except yaml.MarkedYAMLError as ex: - YamlReaderError('during YAML.load() of %s' % yaml_file, + YamlReaderError('during YAML.load() of %s' % yaml_file, rc=os.EX_DATAERR, level=getLevel('NOTICE')) except: #FIXME what to do? @@ -337,7 +336,7 @@ def yaml_load(source, defaultdata=None): return data - + def __main(opts, *argv): import json global options @@ -365,7 +364,7 @@ def __main(opts, *argv): #TODO check other fields which can't be blank except Exception as ex: - logger.critical("%s while merging 'opts' into 'options'.\n %r\n %r", + logger.critical("%s while merging 'opts' into 'options'.\n %r\n %r", ex.__name__, opts, options) return os.EX_CONFIG @@ -387,7 +386,7 @@ def __main(opts, *argv): options.console_level = options.file_level = None options.verbose = True logger.setlevel(logging.DEBUG) - + # override/set Handler-specific levels from parent if not options.console_level: options.console_level = options.log_level @@ -457,7 +456,7 @@ def __main(opts, *argv): logger.warning('%s while dump()' % ex.__name__) # JSON dump might trigger these return os.EX_DATAERR - + return os.EX_OK # if __name__ == '__main__': diff --git a/src/main/python/yamlreader/logging.py b/src/main/python/yamlreader/logging.py index 50d9a0b..6460134 100644 --- a/src/main/python/yamlreader/logging.py +++ b/src/main/python/yamlreader/logging.py @@ -24,9 +24,9 @@ def getLevelName(level, format='%s', no_match=None): # fixup=False """Return the textual representation of 'level'. - Whether predefined (eg. CRITICAL -> "CRITICAL") or user-defined via - addLevelName(), the string associated with 'level' is chosen. - Otherwise, 'level' (no_match == NONE) or 'no_match' is returned + Whether predefined (eg. CRITICAL -> "CRITICAL") or user-defined via + addLevelName(), the string associated with 'level' is chosen. + Otherwise, 'level' (no_match == NONE) or 'no_match' is returned subject to formatting per 'format'. In the spirit of "be liberal in what you accept", any value of 'level' @@ -34,8 +34,8 @@ def getLevelName(level, format='%s', no_match=None): Issue #29220 introduced the BAD IDEA that passing an empty string (an obvious TypeError) would return same. This was requested in order - to squash the fall-thru behavior of returning "Level %s", when the - multi-word response was itself the actual ERROR since it broke all + to squash the fall-thru behavior of returning "Level %s", when the + multi-word response was itself the actual ERROR since it broke all field-based log processing! The astute reader will note that an empty string causes the same pathology... @@ -43,14 +43,14 @@ def getLevelName(level, format='%s', no_match=None): This function WRONGLY returned the mapped Integer if a String form was provided. This violates the clearly stated purpose and forces the caller into defensive Type checks or suffer future TypeErrors. - + NOTE: Does no bounds or validity checks. Use _checkLevel(). - + FUTURE: In strict mode, enforce parameter dataType, case, or membership. """ - + try: # check Name->Level in case called incorrectly (backward compat) if level in _nameToLevel: @@ -68,7 +68,7 @@ def getLevelName(level, format='%s', no_match=None): raise("parameter 'level' must reduce to an Integer") except ValueError: pass - + return format % level if no_match is None else format % no_match @@ -76,7 +76,7 @@ def getLevel(levelName, no_match=logging.NOTSET): # strict={'case': False, 'type': False, 'map': False}, # fixup=False """Return the numeric representation of levelName. - + see getLevelName() for background """ try: @@ -89,7 +89,7 @@ def getLevel(levelName, no_match=logging.NOTSET): except ValueError: if raiseExceptions: raise("parameter 'levelName' must be a defined String") - + return no_match @@ -101,7 +101,7 @@ def _checkLevel(level): pass # # strict={'case': False, 'type': False, 'map': False}, - # """Check parameter against defined values. + # """Check parameter against defined values. # ???Return NOTSET if invalid. # Since all logging.$level() functions choose to emit based on @@ -114,7 +114,7 @@ def _checkLevel(level): # elif level in _levelToName: # rv = level # else: - # #FIXME - test harness injects '+1', so tolerating + # #FIXME - test harness injects '+1', so tolerating # # arbitrary integers is expected behavior. Why? # # raise ValueError # rv = int(level) @@ -125,8 +125,8 @@ def _checkLevel(level): # except Exception: # pass - # return rv + # return rv + - From 238674ef21f8925a29686c9e3206ddc09dbdd1b0 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Fri, 8 Dec 2017 21:51:33 -0800 Subject: [PATCH 17/20] dangling whitespace, options handling --- src/main/python/yamlreader/__init__.py | 89 ++++++++++++++------------ 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index a982401..598a730 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -37,11 +37,9 @@ indent = 2, loader = 'safe' ) -options = optparse.Values(__defaults) -options.console_format = '%s: ' % __name__ + options.log_format -yaml_loaders = ['safe', 'base', 'rt', 'unsafe'] -#XXX use **dict to convert to kwargs +yaml_loaders = ['safe', 'base', 'roundtrip', 'unsafe'] +#TODO use **dict to convert to kwargs __yaml_defaults = dict( preserve_quotes = True, default_flow_style = False, @@ -49,13 +47,16 @@ indent = {} ) + +options = optparse.Values(__defaults) +options.console_format = '%s: ' % __name__ + options.log_format logger = logging.getLogger(__name__) logger.propagate = False - myaml = None class YamlReaderError(Exception): +#TODO rename to ...Exception for more obviousness? plus the level being dealt with isn't necessary just ERROR. """write YAML processing errors to logger""" #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT @@ -121,10 +122,9 @@ def data_merge(a, b, merge=True): else: raise TypeError - except (TypeError, LookupError) as e: - raise YamlReaderError('caught %r merging %r into %r\n "%s" -> "%s"' % - (e, type(b), type(a), b, a), logging.WARNING) from e - # or str(e) also with e.__name__ ? + except (TypeError, LookupError) as ex: + raise YamlReaderError('caught %s (%s) merging %r into %r\n "%s" -> "%s"' % + (type(ex).__name__, ex, type(b), type(a), b, a), logging.WARNING) from e return a @@ -173,6 +173,7 @@ def parse_cmdline(): """Process command-line options""" usage = "%prog [options] source ..." + #FIXME replace with argparse class parser = optparse.OptionParser(usage, description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', version="%" + "prog %s" % __version__, prog='yamlreader') @@ -262,14 +263,17 @@ def parse_cmdline(): except Exception as ex: parser.error(ex) +def _configure(): + pass def _newYaml(): #TODO use kwargs or module defaults? global myaml + if isinstance(myaml, yaml.YAML): + return try: - if not isinstance(myaml, yaml.YAML): - myaml = yaml.YAML(typ=options.loader) + myaml = yaml.YAML(typ=options.loader) # useful defaults for AWS CloudFormation myaml.preserve_quotes=True @@ -280,7 +284,7 @@ def _newYaml(): # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences myaml.indent = dict( mapping = options.indent, - sequence = options.indent * 2, + sequence = options.indent if options.indent >= 4 else options.indent * 2, offset = options.indent ) #FIXME what can YAML() throw? need to catch Math error, possibly Type and ValueError @@ -299,6 +303,8 @@ def yaml_load(source, defaultdata=None): For a directory, all *.yaml files will be read in alphabetical order. """ global myaml + data = defaultdata + new_data = None logger.debug("yaml_load() initialized with source='%s', defaultdata='%s'", source, defaultdata) _newYaml() @@ -313,7 +319,6 @@ def yaml_load(source, defaultdata=None): raise YamlReaderError('FileNotFoundError for %s' % source) return None - data = defaultdata for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: if options.verbose: logger.debug("processing '%s'...", yaml_file) @@ -341,33 +346,36 @@ def __main(opts, *argv): import json global options - #TODO split varification into separate helper? getting too long. try: - # merge __defaults + user-supplied into 'options' if isinstance(opts, optparse.Values): - kv = vars(opts) + new = vars(opts) elif isinstance(opts, dict): - kv = opts + new = opts elif opts is None: - kv = {} + new = {} else: # too early for YamlReaderError raise TypeError("%s not supported for parameter 'opts'" % type(opts)) - print(kv) #XXX - - for k, v in kv.items(): - options.ensure_value(k, v) - - if not (options.log_level and options.log_format): - raise ValueError('options.log_* can not be blank') - #TODO check other fields which can't be blank + vars(options).update(new) except Exception as ex: - logger.critical("%s while merging 'opts' into 'options'.\n %r\n %r", - ex.__name__, opts, options) + logger.critical("caught %s (%s) while merging options", + type(ex).__name__, ex) return os.EX_CONFIG +#FIXME +# some/much of this actually needs to get moved to yaml_load() sub routine since that's the +# public method call from external. Retain all stdIO and logfile stuff though. load_data() +# only throws exceptions with formatted strings but NO io. Caller is responsible for catching +# and emitting as their want. + + if not (options.log_level and options.log_format): + raise ValueError('options.log_* can not be blank') + + + #TODO split varification into separate helper? getting too long. + #TODO check other fields which can't be left blank and have no defaults # adjust 'loader' because Ruamel's cryptic short name if options.loader == 'roundtrip': @@ -376,8 +384,10 @@ def __main(opts, *argv): # normalize logging 'levels' and upcase for downstream lookups for attr in (s + '_level' for s in ['log', 'console', 'file']): try: - setattr(options, attr, str.upper(getattr(options,attr))) - except TypeError: + level = getattr(options, attr) + if isinstance(level, str): + setattr(options, attr, str.upper(level)) + except (AttributeError, TypeError) as ex: pass if options.debug: @@ -385,9 +395,9 @@ def __main(opts, *argv): # reset to trigger Handler-specific override options.console_level = options.file_level = None options.verbose = True - logger.setlevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) - # override/set Handler-specific levels from parent + # override/set Handler-specific levels from 'parent' if not options.console_level: options.console_level = options.log_level if not options.file_level: @@ -410,7 +420,6 @@ def __main(opts, *argv): return os.EX_CONFIG else: logger.addHandler(console_handler) - if options.logfile: try: file_handler = logging.FileHandler(options.logfile, mode='w') @@ -428,6 +437,7 @@ def __main(opts, *argv): # squelch stacktrace if quiet. This affects all handlers, however which # wasn't the intent - just keep the console clear and not duplicate # exception strings. TODO + if options.quiet: sys.excepthook = lambda *args: None elif options.debug: @@ -435,30 +445,29 @@ def __main(opts, *argv): else: sys.excepthook = lambda exctype, exc, traceback : print("{}: {}".format(exctype.__name__, exc)) + #TODO handle case of 'batching' files, say process them individually. and emit as multi-doc yaml. + # is there a method for JSON? I don't think so. # Finally ready to do useful work! data = yaml_load(argv) if data is None or len(data) == 0: # a NOOP is not an error, but no point going further - if option.verbose: + if options.verbose: logger.info('No YAML data found at all!') return os.EX_NOINPUT try: if options.json: - json.dump(data, sys.stdout, options.indent) + # JSON is hard to read at less than 4 spaces + json.dump(data, sys.stdout, indent=options.indent * 2 if options.indent < 4 else options.indent) else: myaml.dump(data, sys.stdout) #TODO combine logging, no need to raise since no external caller. #except yaml.MarkedYAMLError as ex: #except (ValueError, OverflowError, TypeError): json.dump()? except Exception as ex: - logger.warning('%s while dump()' % ex.__name__) + logger.error('caught %s (%s) while dump()' % type(ex).__name__, ex) # JSON dump might trigger these return os.EX_DATAERR return os.EX_OK - -# if __name__ == '__main__': - # (opts, args) = parse_cmdline() - # sys.exit(__main(opts, args)) From 0f38a32cce5ee37f7dfafe8379a5f48d3439524d Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Thu, 14 Dec 2017 08:23:00 -0500 Subject: [PATCH 18/20] checkpoint - big cleanup of logging of status and exceptions --- src/main/python/yamlreader/__init__.py | 528 ++++++++++++++----------- src/main/python/yamlreader/logging.py | 99 ++--- 2 files changed, 353 insertions(+), 274 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 598a730..10b5a1d 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -8,68 +8,81 @@ #FIXME optparse deprecated in favor of argparse!! import optparse import logging -from .yrlogging import * - +# define missing syslog(3) levels and also handy helpers +logging.addLevelName(logging.DEBUG - 5, 'TRACE') +logging.addLevelName(logging.INFO + 5, 'NOTICE') +# fix Lib/logging improperly conflating CRITICAL and FATAL +logging.addLevelName(logging.CRITICAL + 1, 'FATAL') +logging.addLevelName(logging.CRITICAL + 10, 'ALERT') +logging.addLevelName(logging.CRITICAL + 20, 'EMERG') +logging.addLevelName(99, 'ABORT') # see http://yaml.readthedocs.io/en/latest/overview.html import ruamel.yaml as yaml +from .yrlogging import getLevel #, getLevelName + +#FIXME everything that isn't 'main' +#FIXME putthis mostly back into main.py, +# the class methods can log. myaml being a class global is fine too. +# options can stay, console_format needs to move to main(). +__yaml_loaders = ['safe', 'roundtrip', 'unsafe'] __defaults = dict( debug = False, verbose = False, quiet = False, - ignore_error = True, + + console_format = '%s: ' % __name__ + '%(levelname)-8s "%(message)s"', + console_level = logging.WARNING, + logfile_format = '%(levelname)-8s "%(message)s"', + logfile_level = logging.INFO, logfile = None, - log_format = '%(levelname)-8s "%(message)s"', - log_level = logging.INFO, - console_format = None, - console_level = logging.ERROR, - file_format = None, - file_level = logging.INFO, + merge = True, - no_anchor = True, - dup_keys = True, + anchors = False, + allow_duplicate_keys = True, sort_keys = False, sort_files = True, # v3.0 backward-compat reverse = False, json = False, - suffix = 'yaml', + suffix = '', + recurse = False, indent = 2, loader = 'safe' ) -yaml_loaders = ['safe', 'base', 'roundtrip', 'unsafe'] -#TODO use **dict to convert to kwargs -__yaml_defaults = dict( - preserve_quotes = True, - default_flow_style = False, - # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences - indent = {} - ) +options = optparse.Values(__defaults) +#options.console_format = '%s: ' % __name__ + options.log_format +#TODO there's a way to detect teh 'debug' or 'verbose' flags that the parser was called with. +# sys.flags.debug, optimize, verbose and quiet. https://docs.python.org/3.3/library/sys.html -options = optparse.Values(__defaults) -options.console_format = '%s: ' % __name__ + options.log_format logger = logging.getLogger(__name__) logger.propagate = False + +#TODO refactor the options checking for logging to separate method +# and key off of logger.hasHandlers() = False inside yaml_load() and __main() to initialize +logger.setLevel(logging.INFO) + + +#TODO rename to '_yaml' or 'readerYaml' or figure out a way to pass it around WITHOUT it being a global myaml = None class YamlReaderError(Exception): +# the *ONLY* exception being raised out of this class should be this. #TODO rename to ...Exception for more obviousness? plus the level being dealt with isn't necessary just ERROR. """write YAML processing errors to logger""" - #TODO if I was called as a raise, then do super() otherwise stomp on that output since it ends up on STDOUT - def __init__(self, msg, rc=os.EX_SOFTWARE, level=logging.ERROR): #*args, **kwargs): - - # send_to_logger = False + def __init__(self, msg, level=logging.ERROR, rc=os.EX_SOFTWARE): #*args, **kwargs): # for handle in logger.get(handlers): # if isinstance(handle, logging.FileHandler): # send_to_logger = True # break - # if isinstance(level, str): - # level = getLevel(level) + #TODO check/rationalize log level. + if isinstance(level, str): + level = getLevel(level) #TODO case statement to generate/modify strings so it's not buried in multiple # places in code. eg. 'filenotfound' is easy case. msg == filename(s) @@ -77,16 +90,20 @@ def __init__(self, msg, rc=os.EX_SOFTWARE, level=logging.ERROR): #*args, **kwarg super().__init__(msg) frame = sys._getframe().f_back.f_code.co_name - #TODO break out and differentiate as needed. some raise, others (all?) pass - if level > logging.CRITICAL or options.ignore_error == False: - # restore default exception formatting - sys.excepthook = sys.__excepthook__ - logger.log(level, '%s::%s', frame, msg, exc_info=True) - # mimic signals.h SIGTERM, or use os.EX_* - sys.exit(128+rc) + if not options.quiet: + # don't use logger.exception() because handling it ourself + logger.log(level, '%s::%s', frame, msg, exc_info=(options.verbose or options.debug)) - logger.log(level, '%s::%s', frame, msg, exc_info=(options.verbose or options.debug)) + if level > logging.CRITICAL and not options.ignore_error: + #if options.quiet: + # raises SystemExit and invokes any 'finally' clauses + # if untrapped, the Python interpreter exits; no stack traceback is printed. + # mimic signals.h SIGTERM, or use os.EX_* + sys.exit(128+rc) + # else: + # restore default exception formatting? + # sys.excepthook = sys.__excepthook__ def data_merge(a, b, merge=True): @@ -97,11 +114,9 @@ def data_merge(a, b, merge=True): NOTE: tuples and arbitrary objects are not handled as it is totally ambiguous what should happen """ - import six - key = None + logger.debug('Attempt data_merge() of\n\t"%s"\n into\n\t"%s"\n' % (b, a)) - #logger.debug('Attempting merge of "%s" into "%s"\n' % (b, a)) try: # border case for first run or if a is a primitive if a is None or isinstance(a, (six.string_types, float, six.integer_types)): @@ -113,23 +128,29 @@ def data_merge(a, b, merge=True): if not merge: a.update(b) elif isinstance(b, dict): - for key in b: - a[key] = data_merge(a[key], b[key]) if key in a else b[key] + for key in b.keys(): + if key in a: + a[key] = data_merge(a[key], b[key]) + else: + a[key] = b[key] else: - # XXX technically a Tuple or List of at least 2 wide + # TODO technically a Tuple or List of at least 2 wide # could be used with [0] as key, [1] as value raise TypeError else: raise TypeError except (TypeError, LookupError) as ex: - raise YamlReaderError('caught %s (%s) merging %r into %r\n "%s" -> "%s"' % - (type(ex).__name__, ex, type(b), type(a), b, a), logging.WARNING) from e + logger.warning('caught %s:(%s) while data_merge()' % (type(ex).__name__, ex)) + logger.debug('data_merge(): "%s" -> "%s"' % (b, a)) + #FIXME do we really want to emit this here? use raise or just log? + # raise YamlReaderError('caught %s (%s) merging %r into %r\n "%s" -> "%s"' % + # (type(ex).__name__, ex, type(b), type(a), b, a), from ex return a -def get_files(source, suffix='yaml'): +def get_files(source, suffix='', recurse=False): """Examine pathspec elements for files to process 'source' can be a filename, directory, a list/tuple of same, @@ -140,9 +161,9 @@ def get_files(source, suffix='yaml'): files = [] if source is None or len(source) == 0 or source == '-': - return [''] + files = [''] + return files - #if type(source) is list or type(source) is tuple: if isinstance(source, list) or isinstance(source, tuple): for item in source: # iterate to expand list of potential dirs and files @@ -150,29 +171,118 @@ def get_files(source, suffix='yaml'): return files if os.path.isdir(source): - files = glob.glob(os.path.join(source, '*.' + suffix)) + files = glob.glob(os.path.join(source, '*' + suffix)) elif os.path.isfile(source): # turn single file into list files = [source] else: - # try to use the source as a glob - files = glob.glob(source) + # TODO??? change to iglob() and accept only '*suffix'? + files = glob.glob(source, recursive=recurse) + + return files + + +def _newYaml(preserve_quotes=True, default_flow_style=False, indent=None): + #FIXME should be 'settings' which map to Yaml internals directly + # ala for key in settings.keys() myaml.key = settings[key] + # handle special case that is 'indent' + global myaml + + try: + myaml = yaml.YAML(typ=options.loader) + + # useful defaults for AWS CloudFormation + myaml.preserve_quotes=preserve_quotes + myaml.default_flow_style=default_flow_style + myaml.allow_duplicate_keys = options.allow_duplicate_keys + if not options.anchors: + myaml.representer.ignore_aliases = lambda *args: True + + # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences + #TODO update indents based on options.indent if parameter is None? Do we assume {} means no indent? + if isinstance(indent, dict): + # TODO seq >= offset+2 + myaml.indent(mapping=indent['mapping'], offset=indent['offset'], sequence = indent['sequence']) + elif isinstance(int, indent): + myaml.indent(indent) + #else: TODO throw something? + + #FIXME what can YAML() throw? need to catch Math error, possibly Type and ValueError + # ??AttributeError when calling constructors + except KeyError: + # ignore + pass + except Exception as ex: + #TODO + raise YamlReaderError('XXX') from ex + + +def yaml_load(source, data=None, + preserve_quotes=True, default_flow_style=False, + indent=dict(mapping=options.indent, sequence=options.indent, offset=0)): + #TODO pass in a pre-instantiated YAML class object so any 3rd party (API compat) + """merge YAML data from files found in source + + Always returns a dict. The files are read with the 'safe' loader + though the other 3 options are possible. + + 'source' can be a file, a dir, a list/tuple of files or a string containing + a glob expression (with ?*[]). + For a directory, all *.yaml files will be read in alphabetical order. + """ + global myaml + logger.log(getLevel('TRACE'), "yaml_load() called with\n\tsource='%s'\n\tdata='%s'", source, data) + #TODO bring _newYaml back here, it's not THAT long. + # assume already configured + if not isinstance(myaml, yaml.YAML): + _newYaml(preserve_quotes, default_flow_style, indent) + + # NOTICE - sort_keys is a NOOP unless Matt's version of + # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) + if hasattr(myaml.representer, 'sort_keys'): + myaml.representer.sort_keys = options.sort_keys + + files = get_files(source, options.suffix, options.recurse) if len(files) == 0: - # TODO what is suitable error level? - # if options.ignore_error , use ERROR, otherwise WARNING - level = logging.WARNING if options.ignore_error else logging.ERROR - YamlReaderError('FileNotFoundError for %r' % source, - rc=os.EX_OSFILE, - level=level) + raise YamlReaderError("FileNotFoundError for %s" % source, rc=os.EX_OSFILE) + # from FileNotFoundError("%s" % source) + # from YamlReaderError("FileNotFoundError for %s" % source, rc=os.EX_OSFILE) - return files + new_data = None + for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: + logger.info("processing '%s' ...", yaml_file) + + try: + new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) + except (yaml.error.YAMLError, yaml.error.YAMLStreamError) as ex: + raise YamlReaderError("%s during YAML.load()" % ex, rc=os.EX_DATAERR) + + except (yaml.error.YAMLWarning, yaml.error.YAMLFutureWarning) as ex: + if options.verbose: + logger.warning("%s during YAML.load()", type(ex).__name__) + logger.log(getLevel('NOTICE'), "%s", ex) + # Ruamel throws this despite allow_duplicate_keys? + except Exception as ex: + #FIXME stuff from open(), data_merge() + # just silently squelch everything but MarkedYAML? + logger.warning("unhandled %s during YAML.load() of '%s'" % (type(ex).__name__, yaml_file)) + if not options.ignore_error: + raise + + if new_data: + logger.debug('payload: %r\n', new_data) + data = data_merge(data, new_data, options.merge) + else: + logger.log(getLevel('NOTICE'), "no payload found in '%s'", yaml_file) + return data + def parse_cmdline(): """Process command-line options""" - usage = "%prog [options] source ..." + #FIXME replace with argparse class parser = optparse.OptionParser(usage, description='Merge YAML/JSON elements from Files, Directories, or Glob pattern', @@ -182,80 +292,100 @@ def parse_cmdline(): parser.add_option('-d', '--debug', dest='debug', action='store_true', default=__defaults['debug'], - help="enable debugging %default") + help='%-35s %s' % ('enable debugging', "%default")) parser.add_option('-v', '--verbose', dest='verbose', action='store_true', default=__defaults['verbose'], - help="extra messages %default") + help='%-35s %s' % ('extra messages', "%default")) parser.add_option('-q', '--quiet', dest='quiet', action='store_true', default=__defaults['quiet'], - help="minimize output %default") + help='%-35s %s' % ('minimize output', "%default")) + # only useful if invoked via __main__ parser.add_option('-c', '--continue', dest='ignore_error', - action='store_true', default=__defaults['ignore_error'], - help="errors as not fatal %default") + action='store_true', default=False, + help='%-35s %s' % ('even if >CRITICAL', "%default")) parser.add_option('-l', '--logfile', dest='logfile', action='store', default=__defaults['logfile']) #TODO log_format = '%(levelname)8s "%(message)s"', - parser.add_option('--log-level', dest='log_level', - action='store', default=__defaults['log_level'], - help=' '.join(logging._nameToLevel.keys()), - choices=list(logging._nameToLevel.keys())) + #FIXME, using _levelToName is cheating + levels=list(logging._nameToLevel.keys()) + levelstr, i = ('', 1) + for lev in sorted(logging._levelToName.keys()): + levelstr += '%s:%d ' % (logging._levelToName[lev], lev) + # if (i % 4) == 0: + # levelstr += '\n' # TODO escape sequences gets lost on output + # i+=1 + # fake entry just for Help + parser.add_option('--xxx-level', + action='store_const', const=logging.INFO, + help='%s' % (levelstr)) parser.add_option('--console-level', dest='console_level', action='store', default=__defaults['console_level'], - help=" %default ") + help='%-35s %s' % ('', + logging._levelToName.get(__defaults['console_level'])), + choices=levels) - parser.add_option('--file-level', dest='file_level', - action='store', default=__defaults['file_level']) - - parser.add_option('-M', '--overwrite', dest='merge', - action='store_false', default=not __defaults['merge'], - help="overwrite keys %default") + parser.add_option('--file-level', dest='logfile_level', + action='store', default=__defaults['logfile_level'], + help='%-35s %s' % ('', + logging._levelToName.get(__defaults['logfile_level'])), + choices=levels) # CloudFormation can't handle anchors or aliases in final output - parser.add_option('-X', '--no-anchor', dest='no_anchor', - action='store_true', default=__defaults['no_anchor'], - help="unroll anchors %default") + parser.add_option('-x', dest='anchors', + action='store_true', default=__defaults['anchors'], + help='%-35s %s' % ('preserve anchors/aliases', "%default")) - parser.add_option('-u', '--unique-keys', dest='dup_keys', - action='store_false', default=__defaults['dup_keys'], - help="skip duplicates %default") + parser.add_option('-M', '--overwrite', dest='merge', + action='store_false', default=__defaults['merge'], + help='%-35s %s' % ('overwrite keys (last win)', "%default")) + +#FIXME is the sense correct? does it effect Merge in practical terms? aka if dup = false, it just skips the merge? +# as opposed to merge=false means overwrite. +# this nullifies merge in either state. + parser.add_option('-U', '--unique-keys', dest='allow_duplicate_keys', + action='store_false', default=not __defaults['allow_duplicate_keys'], + help='%-35s %s' % ('skip duplicate keys (first win)', "%default")) parser.add_option('-k', '--sort-keys', dest='sort_keys', action='store_true', default=__defaults['sort_keys'], - help="sort keys %default") + help='%-35s %s' % ('sort keys', "%default")) parser.add_option('-S', '--no-sort-files', dest='sort_files', action='store_false', default=__defaults['sort_files'], - help="sort filenames %default") + help='%-35s %s' % ('sort filenames', "%default")) parser.add_option('-r', '--reverse', dest='reverse', action='store_true', default=__defaults['reverse'], - help="sort direction %default") + help='%-35s %s' % ('sort direction', "%default")) parser.add_option('-j', '--json', dest='json', action='store_true', default=__defaults['json'], - help="output as JSON %default") + help='%-35s %s' % ('output as JSON', "%default")) parser.add_option('--suffix', dest='suffix', action='store', default=__defaults['suffix'], - help="filename suffix '%default'") + help='%-35s %s' % ("if dir|glob() apply filter '*suffix'", "%default")) + + parser.add_option('--recurse', dest='recurse', + action='store_true', default=__defaults['recurse'], + help='%-35s %s' % ("expand '**' in filespec", "%default")) #TODO - defaults for Yaml constructor. - # move loader and indent into __yaml_defaults and prepend name with 'yaml' parser.add_option('-t', '--indent', dest='indent', action='store', type=int, default=__defaults['indent'], - help=" %default") + help='%-35s %s' % ('', "%default")) parser.add_option('--loader', dest='loader', action='store', default=__defaults['loader'], - help="%s %s" % (' '.join(yaml_loaders), __defaults['loader']), - choices=yaml_loaders) + help='%-35s %s' % (' '.join(__yaml_loaders), __defaults['loader']), + choices=__yaml_loaders) # | %default try: return parser.parse_args() @@ -263,88 +393,13 @@ def parse_cmdline(): except Exception as ex: parser.error(ex) + def _configure(): pass -def _newYaml(): - #TODO use kwargs or module defaults? - global myaml - - if isinstance(myaml, yaml.YAML): - return - try: - myaml = yaml.YAML(typ=options.loader) - - # useful defaults for AWS CloudFormation - myaml.preserve_quotes=True - myaml.default_flow_style=False - myaml.allow_duplicate_keys = options.dup_keys - myaml.representer.ignore_aliases = lambda *args: True - - # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences - myaml.indent = dict( - mapping = options.indent, - sequence = options.indent if options.indent >= 4 else options.indent * 2, - offset = options.indent - ) - #FIXME what can YAML() throw? need to catch Math error, possibly Type and ValueError - except Exception as ex: - raise YamlReaderError('XXX') from ex - - -def yaml_load(source, defaultdata=None): - """merge YAML data from files found in source - - Always returns a dict. The files are read with the 'safe' loader - though the other 3 options are possible. - - 'source' can be a file, a dir, a list/tuple of files or a string containing - a glob expression (with ?*[]). - For a directory, all *.yaml files will be read in alphabetical order. - """ - global myaml - data = defaultdata - new_data = None - - logger.debug("yaml_load() initialized with source='%s', defaultdata='%s'", source, defaultdata) - _newYaml() - - # NOTICE - sort_keys is a NOOP unless Matt's version of - # Ruamel's YAML library (https://bitbucket.org/tb3088/yaml) - if hasattr(myaml.representer, 'sort_keys'): - myaml.representer.sort_keys = options.sort_keys - - files = get_files(source, options.suffix) - if len(files) == 0: - raise YamlReaderError('FileNotFoundError for %s' % source) - return None - - for yaml_file in sorted(files, reverse=options.reverse) if options.sort_files else files: - if options.verbose: - logger.debug("processing '%s'...", yaml_file) - - try: - new_data = myaml.load(open(yaml_file) if len(yaml_file) else sys.stdin) - logger.debug('payload: %r\n', new_data) - except yaml.MarkedYAMLError as ex: - YamlReaderError('during YAML.load() of %s' % yaml_file, - rc=os.EX_DATAERR, level=getLevel('NOTICE')) - except: - #FIXME what to do? - pass - - if new_data: - data = data_merge(data, new_data, options.merge) - elif options.verbose: - #XXX rc=os.EX_NOINPUT, actually os.EX_DATAERR - logger.info("no YAML data in %s", yaml_file) - - return data - - def __main(opts, *argv): - import json global options + global logger try: if isinstance(opts, optparse.Values): @@ -355,119 +410,132 @@ def __main(opts, *argv): new = {} else: # too early for YamlReaderError - raise TypeError("%s not supported for parameter 'opts'" % type(opts)) + logging.lastResort("Type '%s' not supported for 'opts'" % type(opts)) + return os.EX_CONFIG vars(options).update(new) except Exception as ex: - logger.critical("caught %s (%s) while merging options", - type(ex).__name__, ex) + logging.lastResort("caught %s (%s) while merging options" % (type(ex).__name__, ex)) return os.EX_CONFIG -#FIXME -# some/much of this actually needs to get moved to yaml_load() sub routine since that's the -# public method call from external. Retain all stdIO and logfile stuff though. load_data() -# only throws exceptions with formatted strings but NO io. Caller is responsible for catching -# and emitting as their want. - - if not (options.log_level and options.log_format): - raise ValueError('options.log_* can not be blank') +#FIXME this section belong in _configure which sets up loghandlers +# and is called from yaml_load() if it detects it's not setup. +# eg. if options isnot of type or myaml is not of proper type - #TODO split varification into separate helper? getting too long. - #TODO check other fields which can't be left blank and have no defaults - - # adjust 'loader' because Ruamel's cryptic short name - if options.loader == 'roundtrip': - options.loader = 'rt' +#XXX FIXME this whole block needs to be called from _configure as part of main class. # normalize logging 'levels' and upcase for downstream lookups - for attr in (s + '_level' for s in ['log', 'console', 'file']): + # gratuitous since optparse() is enforcing values? + for attr in (s + '_level' for s in ['console', 'logfile']): try: level = getattr(options, attr) if isinstance(level, str): - setattr(options, attr, str.upper(level)) - except (AttributeError, TypeError) as ex: - pass + # FIXME more cheating + setattr(options, attr, logger._nameToLevel[str.upper(level)]) + if logging._levelToName[getattr(options, attr)] is None: + # failsafe + setattr(options, attr, logging.INFO) + except (AttributeError, TypeError, ValueError) as ex: + logging.lastResort("'%s'(%s) during logging failsafe" % (type(ex).__name__, ex)) + if not options.ignore_error: + return os.EX_CONFIG + + if options.verbose: + options.console_level=logging.INFO if options.debug: - options.loglevel = logging.DEBUG # getLevel('DEBUG') - # reset to trigger Handler-specific override - options.console_level = options.file_level = None - options.verbose = True logger.setLevel(logging.DEBUG) - - # override/set Handler-specific levels from 'parent' - if not options.console_level: - options.console_level = options.log_level - if not options.file_level: - options.file_level = options.log_level - - if not options.console_format: - options.console_format = options.log_format - if not options.file_format: - options.file_level = options.log_format + options.console_level = logging.DEBUG + options.logfile_level = logging.DEBUG #TODO getLevel('TRACE') + options.verbose = True + options.quiet = False if not options.quiet: try: console_handler = logging.StreamHandler() console_handler.setFormatter(logging.Formatter(options.console_format)) console_handler.setLevel(options.console_level) - except: #FIXME what to trap? - msg='' - raise #TODO logger.error() + logger.addHandler(console_handler) + except Exception as ex: #FIXME what to trap? + logging.lastResort("%s (%s) configuring console_handler" % (type(ex).__name__, ex)) if not options.ignore_error: return os.EX_CONFIG - else: - logger.addHandler(console_handler) + if options.logfile: try: file_handler = logging.FileHandler(options.logfile, mode='w') - file_handler.setFormatter(logging.Formatter(options.file_format)) - file_handler.setLevel(options.file_level) - except (FileNotFoundError, OSError): #TODO what else? rc=os.EX_OSFILE - options.logfile = None - msg='' - raise #TODO logger.error() - if options.ignore_error: - return os.EX_CONFIG - else: + file_handler.setFormatter(logging.Formatter(options.logfile_format)) + file_handler.setLevel(options.logfile_level) logger.addHandler(file_handler) + except (FileNotFoundError, OSError) as ex: #TODO what else? + options.logfile = None + logging.lastResort("%s (%s) configuring file_handler" % (type(ex).__name__, ex, options.logfile)) + if not options.ignore_error: + return os.EX_OSFILE - # squelch stacktrace if quiet. This affects all handlers, however which - # wasn't the intent - just keep the console clear and not duplicate - # exception strings. TODO - + # for 'quiet' one could removeHandler(console) but if there are NO + # handlers Logging helpfully provides one (stderr, level=WARNING) if options.quiet: + if not logger.hasHandlers(): + logger.addHandler(logging.NullHandler()) + # gratuitous? since sys.exit() squelches stacktrace sys.excepthook = lambda *args: None elif options.debug: pass else: - sys.excepthook = lambda exctype, exc, traceback : print("{}: {}".format(exctype.__name__, exc)) + sys.excepthook = lambda exctype, exc, traceback : print("uncaught! {}: {}".format(exctype.__name__, exc)) + + # FINALLY logging is ready! + #TODO split varification into separate configure() helper? getting too long. + #TODO check other fields which can't be left blank and have no defaults + #TODO handle case of 'batching' files, say process them individually. and emit as multi-doc yaml. # is there a method for JSON? I don't think so. - # Finally ready to do useful work! - data = yaml_load(argv) - if data is None or len(data) == 0: - # a NOOP is not an error, but no point going further - if options.verbose: - logger.info('No YAML data found at all!') - return os.EX_NOINPUT + # HACK 'safe' loader mangles anchors by generating new IDs. + if options.anchors: # and options.original_anchors: + options.loader = 'rt' + # adjust 'loader' because Ruamel's cryptic short name + if options.loader == 'roundtrip': + options.loader = 'rt' + + # Finally ready to do useful work! + #FIXME deal with YamlReaderError getting thrown. from say no input files. try: + data = yaml_load(argv) + + if data is None or len(data) == 0: + # empty is not an error, but no point going further + if options.verbose: + logger.warning('No YAML data anywhere!') + return os.EX_NOINPUT + +# try: if options.json: + import json # JSON is hard to read at less than 4 spaces json.dump(data, sys.stdout, indent=options.indent * 2 if options.indent < 4 else options.indent) else: myaml.dump(data, sys.stdout) + + # except SystemExit: + # raise + except YamlReaderError as ex: + pass + # eg. argv was invalid + # TODO take action? no way to know what teh os.EX_ value is... + #TODO combine logging, no need to raise since no external caller. #except yaml.MarkedYAMLError as ex: #except (ValueError, OverflowError, TypeError): json.dump()? except Exception as ex: - logger.error('caught %s (%s) while dump()' % type(ex).__name__, ex) # JSON dump might trigger these + logger.error('caught %s (%s) while main()' % (type(ex).__name__, ex)) return os.EX_DATAERR - return os.EX_OK + else: + return os.EX_OK diff --git a/src/main/python/yamlreader/logging.py b/src/main/python/yamlreader/logging.py index 6460134..de9b3f8 100644 --- a/src/main/python/yamlreader/logging.py +++ b/src/main/python/yamlreader/logging.py @@ -8,17 +8,6 @@ # private re-implementations till Python Core fixes Lib/logging # XXX bug numbers here -# define missing syslog(3) levels and also handy helpers -logging.addLevelName(logging.NOTSET, 'ALL') -logging.addLevelName(logging.DEBUG - 5, 'TRACE') -logging.addLevelName(logging.INFO + 5, 'NOTICE') -# fix Lib/logging improperly conflating CRITICAL and FATAL -logging.addLevelName(logging.CRITICAL + 1, 'FATAL') -logging.addLevelName(logging.CRITICAL + 10, 'ALERT') -logging.addLevelName(logging.CRITICAL + 20, 'EMERG') -logging.addLevelName(logging.CRITICAL + 99, 'ABORT') - - def getLevelName(level, format='%s', no_match=None): # strict={'case': False, 'type': False, 'map': False}, # fixup=False @@ -53,13 +42,13 @@ def getLevelName(level, format='%s', no_match=None): try: # check Name->Level in case called incorrectly (backward compat) - if level in _nameToLevel: + if level in logging._nameToLevel: return format % level # retval = _checkLevel(level, flags, fix=T/F) # if isinstance(retval, bool) then handle pass/fail, else update level with fixed value - result = _levelToName.get(int(level)) + result = logging._levelToName.get(int(level)) if result is not None: return format % result @@ -80,7 +69,7 @@ def getLevel(levelName, no_match=logging.NOTSET): see getLevelName() for background """ try: - result = _nameToLevel.get(levelName) + result = logging._nameToLevel.get(levelName) if result is not None: return result @@ -97,36 +86,58 @@ def getLevelOrName(level): pass -def _checkLevel(level): +def _checkLevel(level, case=False, type=False, map=False): + #TODO define check as dictionary pass -# # strict={'case': False, 'type': False, 'map': False}, - - # """Check parameter against defined values. - # ???Return NOTSET if invalid. - - # Since all logging.$level() functions choose to emit based on - # numeric comparison, a default of ERROR would be more friendly. + # """Check parameter against defined values + # + # Returns corresponding or original Integer, or NOTSET if no-match. + # Will raise TypeError or ValueError as applicable. + # + # NOTE: Since all logging.$level() functions choose to emit based on + # numeric comparison, a default of ERROR would be more logical. # """ - # rv = NOTSET - # try: - # if level in _nameToLevel: - # rv = _nameToLevel[level] - # elif level in _levelToName: - # rv = level - # else: - # #FIXME - test harness injects '+1', so tolerating - # # arbitrary integers is expected behavior. Why? - # # raise ValueError - # rv = int(level) - # except (TypeError, ValueError, KeyError) as err: - # if raiseExceptions: - # # test harness (../test/test_logging) expects 'TypeError' - # raise TypeError("Level not an integer or a valid string: %r" % level) from err - # except Exception: - # pass - - # return rv - - - + try: + if isinstance(level, str): + if not case: + level = str.upper(level) + rv = _nameToLevel.get(level) + # if rv is None: + # XXX what now? + if isinstance(level, int) or not type: + # flip negative values + level = int(level) + if level in _levelToName(level): + rv = level + else: + # tolerate any Integer value + rv = NOTSET if map else level + if rv is None: + level = str(level) + if rv is None: + if level in _levelToName or (not type and int(level) in _levelToName): + rv = NOTSET if level < NOTSET else level + # rv = level + if rv is None and map: + raise ValueError + else: + # return parameter even though invalid + rv = level + # sor level < NOTSET or level > ???: + # #raise ValueError + # if isinstance(level, int): + # XXX check >NOTSET + # else: + # raise TypeError + #FIXME - test harness injects '+1', so tolerating + # arbitrary integers is expected behavior. Why? + # raise ValueError + rv = int(level) + except (TypeError, ValueError, KeyError) as err: + if raiseExceptions: + # test harness (../test/test_logging) expects 'TypeError' ONLY + raise TypeError("Level not an integer or a valid string: %r" % level) from err + except Exception: + pass + return NOTSET - 1 if rv is None else rv From 24448e15731fc8cfb2470ce2a65cd9b45e659a31 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Mon, 26 Feb 2018 16:25:51 -0500 Subject: [PATCH 19/20] whitespace and EOL management --- .gitattributes | 4 ++++ src/main/python/yamlreader/__init__.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8676941 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto +*.sh text eol=lf +*.pl text eol=lf +*.py* text eol=lf diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 10b5a1d..9c503b2 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -21,8 +21,8 @@ import ruamel.yaml as yaml from .yrlogging import getLevel #, getLevelName -#FIXME everything that isn't 'main' -#FIXME putthis mostly back into main.py, +#FIXME everything that isn't 'main' +#FIXME putthis mostly back into main.py, # the class methods can log. myaml being a class global is fine too. # options can stay, console_format needs to move to main(). @@ -201,7 +201,7 @@ def _newYaml(preserve_quotes=True, default_flow_style=False, indent=None): # see http://yaml.readthedocs.io/en/latest/detail.html#indentation-of-block-sequences #TODO update indents based on options.indent if parameter is None? Do we assume {} means no indent? if isinstance(indent, dict): - # TODO seq >= offset+2 + # TODO seq >= offset+2 myaml.indent(mapping=indent['mapping'], offset=indent['offset'], sequence = indent['sequence']) elif isinstance(int, indent): myaml.indent(indent) @@ -210,7 +210,7 @@ def _newYaml(preserve_quotes=True, default_flow_style=False, indent=None): #FIXME what can YAML() throw? need to catch Math error, possibly Type and ValueError # ??AttributeError when calling constructors except KeyError: - # ignore + # ignore pass except Exception as ex: #TODO @@ -277,7 +277,7 @@ def yaml_load(source, data=None, logger.log(getLevel('NOTICE'), "no payload found in '%s'", yaml_file) return data - + def parse_cmdline(): """Process command-line options""" @@ -327,7 +327,7 @@ def parse_cmdline(): parser.add_option('--console-level', dest='console_level', action='store', default=__defaults['console_level'], - help='%-35s %s' % ('', + help='%-35s %s' % ('', logging._levelToName.get(__defaults['console_level'])), choices=levels) @@ -448,7 +448,7 @@ def __main(opts, *argv): if options.debug: logger.setLevel(logging.DEBUG) options.console_level = logging.DEBUG - options.logfile_level = logging.DEBUG #TODO getLevel('TRACE') + options.logfile_level = logging.DEBUG #TODO getLevel('TRACE') options.verbose = True options.quiet = False @@ -462,7 +462,7 @@ def __main(opts, *argv): logging.lastResort("%s (%s) configuring console_handler" % (type(ex).__name__, ex)) if not options.ignore_error: return os.EX_CONFIG - + if options.logfile: try: file_handler = logging.FileHandler(options.logfile, mode='w') @@ -491,7 +491,7 @@ def __main(opts, *argv): #TODO split varification into separate configure() helper? getting too long. #TODO check other fields which can't be left blank and have no defaults - + #TODO handle case of 'batching' files, say process them individually. and emit as multi-doc yaml. # is there a method for JSON? I don't think so. From 7cef3b5d9a398822af3e9bae92577c3366ef595d Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Mon, 19 Mar 2018 17:29:24 -0400 Subject: [PATCH 20/20] unhack class loader, and replace STDLIB/logging functions with private rewrite --- src/main/python/yamlreader/__init__.py | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/python/yamlreader/__init__.py b/src/main/python/yamlreader/__init__.py index 9c503b2..3ff8a3a 100644 --- a/src/main/python/yamlreader/__init__.py +++ b/src/main/python/yamlreader/__init__.py @@ -7,7 +7,17 @@ import os, sys #FIXME optparse deprecated in favor of argparse!! import optparse + +# see http://yaml.readthedocs.io/en/latest/overview.html +import ruamel.yaml as yaml + +#from .logging import getLevel #, getLevelName +import yamlreader.logging as yrl import logging +#override core functions +logging.getLevel = yrl.getLevel +logging.getLevelName = yrl.getLevelName + # define missing syslog(3) levels and also handy helpers logging.addLevelName(logging.DEBUG - 5, 'TRACE') logging.addLevelName(logging.INFO + 5, 'NOTICE') @@ -17,9 +27,6 @@ logging.addLevelName(logging.CRITICAL + 20, 'EMERG') logging.addLevelName(99, 'ABORT') -# see http://yaml.readthedocs.io/en/latest/overview.html -import ruamel.yaml as yaml -from .yrlogging import getLevel #, getLevelName #FIXME everything that isn't 'main' #FIXME putthis mostly back into main.py, @@ -82,7 +89,7 @@ def __init__(self, msg, level=logging.ERROR, rc=os.EX_SOFTWARE): #*args, **kwarg # break #TODO check/rationalize log level. if isinstance(level, str): - level = getLevel(level) + level = logging.getLevel(level) #TODO case statement to generate/modify strings so it's not buried in multiple # places in code. eg. 'filenotfound' is easy case. msg == filename(s) @@ -218,7 +225,7 @@ def _newYaml(preserve_quotes=True, default_flow_style=False, indent=None): def yaml_load(source, data=None, - preserve_quotes=True, default_flow_style=False, + preserve_quotes=True, default_flow_style=False, indent=dict(mapping=options.indent, sequence=options.indent, offset=0)): #TODO pass in a pre-instantiated YAML class object so any 3rd party (API compat) """merge YAML data from files found in source @@ -232,7 +239,7 @@ def yaml_load(source, data=None, """ global myaml - logger.log(getLevel('TRACE'), "yaml_load() called with\n\tsource='%s'\n\tdata='%s'", source, data) + logger.log(logging.getLevel('TRACE'), "yaml_load() called with\n\tsource='%s'\n\tdata='%s'", source, data) #TODO bring _newYaml back here, it's not THAT long. # assume already configured if not isinstance(myaml, yaml.YAML): @@ -261,7 +268,7 @@ def yaml_load(source, data=None, except (yaml.error.YAMLWarning, yaml.error.YAMLFutureWarning) as ex: if options.verbose: logger.warning("%s during YAML.load()", type(ex).__name__) - logger.log(getLevel('NOTICE'), "%s", ex) + logger.log(logging.getLevel('NOTICE'), "%s", ex) # Ruamel throws this despite allow_duplicate_keys? except Exception as ex: #FIXME stuff from open(), data_merge() @@ -274,7 +281,7 @@ def yaml_load(source, data=None, logger.debug('payload: %r\n', new_data) data = data_merge(data, new_data, options.merge) else: - logger.log(getLevel('NOTICE'), "no payload found in '%s'", yaml_file) + logger.log(logging.getLevel('NOTICE'), "no payload found in '%s'", yaml_file) return data @@ -346,8 +353,8 @@ def parse_cmdline(): action='store_false', default=__defaults['merge'], help='%-35s %s' % ('overwrite keys (last win)', "%default")) -#FIXME is the sense correct? does it effect Merge in practical terms? aka if dup = false, it just skips the merge? -# as opposed to merge=false means overwrite. +#FIXME is the sense correct? does it effect Merge in practical terms? aka if dup = false, +# it just skips the merge as opposed to merge=false means overwrite. # this nullifies merge in either state. parser.add_option('-U', '--unique-keys', dest='allow_duplicate_keys', action='store_false', default=not __defaults['allow_duplicate_keys'],