diff --git a/setup.py b/setup.py index 9a28e31f..25ac2769 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,7 @@ def launch_http_server(directory): "mock", "PyYAML", "pytest", + "toml", ] diff --git a/tests/test_configargparse.py b/tests/test_configargparse.py index 968b2b02..913d274a 100644 --- a/tests/test_configargparse.py +++ b/tests/test_configargparse.py @@ -1815,11 +1815,379 @@ def testYAMLConfigFileParser_w_ArgumentParser_parsed_values(self): # print argparse unittest source code def print_source_code(source_code, line_numbers, context_lines=10): - for n in line_numbers: - logging.debug("##### Code around line %s #####" % n) - lines_to_print = set(range(n - context_lines, n + context_lines)) - for n2, line in enumerate(source_code.split("\n"), 1): - if n2 in lines_to_print: - logging.debug("%s %5d: %s" % ("**" if n2 == n else " ", n2, line)) - - # print_source_code(test_argparse_source_code, [4540, 4565]) + for n in line_numbers: + logging.debug("##### Code around line %s #####" % n) + lines_to_print = set(range(n - context_lines, n + context_lines)) + for n2, line in enumerate(source_code.split("\n"), 1): + if n2 in lines_to_print: + logging.debug("%s %5d: %s" % ( + "**" if n2 == n else " ", n2, line)) + #print_source_code(test_argparse_source_code, [4540, 4565]) + + +# Tests for the newly added parsers in 1.5.5 +from configargparse import parse_toml_section_name, unquote_str, IniConfigParser, TomlConfigParser, CompositeConfigParser + +class TestNewlyAddedParsersInVersion_1_5_5(TestCase): + + def test_unquote_str(self) -> None: + + assert unquote_str('string') == 'string' + assert unquote_str('"string') == '"string' + assert unquote_str('string"') == 'string"' + assert unquote_str('"string"') == 'string' + assert unquote_str('\'string\'') == 'string' + assert unquote_str('"""string"""') == 'string' + assert unquote_str('\'\'\'string\'\'\'') == 'string' + assert unquote_str('"""\nstring"""') == '\nstring' + assert unquote_str('\'\'\'string\n\'\'\'') == 'string\n' + assert unquote_str('"""\nstring \n"""') == '\nstring \n' + assert unquote_str('\'\'\'\n string\n\'\'\'') == '\n string\n' + + assert unquote_str('\'\'\'string') == '\'\'\'string' + assert unquote_str('string\'\'\'') == 'string\'\'\'' + assert unquote_str('"""string') == '"""string' + assert unquote_str('string"""') == 'string"""' + assert unquote_str('"""str"""ing"""') == '"""str"""ing"""' + assert unquote_str('str\'ing') == 'str\'ing' + assert unquote_str('""""value""""') == '""""value""""' + + def test_parse_toml_section_keys(self) -> None: + assert parse_toml_section_name('tool.pydoctor') == ('tool', 'pydoctor') + assert parse_toml_section_name(' tool.pydoctor ') == ('tool', 'pydoctor') + assert parse_toml_section_name(' "tool".pydoctor ') == ('tool', 'pydoctor') + assert parse_toml_section_name(' tool."pydoctor" ') == ('tool', 'pydoctor') + + + def test_IniConfigParser(self): + # Not supported by configparser (currently raises error) + # {'line': 'key value', 'expected': ('key', 'value', None)}, + # {'line': 'key value', 'expected': ('key', 'value', None)}, + # {'line': ' key value ', 'expected': ('key', 'value', None)} + # {'line': 'key ', 'expected': ('key', 'true', None)}, + # {'line': 'key', 'expected': ('key', 'true', None)}, + # {'line': 'key ', 'expected': ('key', 'true', None)}, + # {'line': ' key ', 'expected': ('key', 'true', None)}, + + p = IniConfigParser(['soft'], False) + + for test in get_IniConfigParser_cases(): + try: + parsed_obj = p.parse(StringIO('[soft]\n'+test['line'])) + except Exception as e: + raise AssertionError("Line %r, error: %s" % (test['line'], str(e))) from e + else: + parsed_obj = dict(parsed_obj) + expected = {test['expected'][0]: test['expected'][1]} + assert parsed_obj==expected, "Line %r" % (test['line']) + + def test_IniConfigParser_multiline_text_to_list(self): + + p = IniConfigParser(['soft'], True) + + for test in get_IniConfigParser_multiline_text_to_list_cases(): + try: + parsed_obj = p.parse(StringIO('[soft]\n'+test['line'])) + except Exception as e: + raise AssertionError("Line %r, error: %s" % (test['line'], str(e))) from e + else: + parsed_obj = dict(parsed_obj) + expected = {test['expected'][0]: test['expected'][1]} + assert parsed_obj==expected, "Line %r" % (test['line']) + + def test_TomlConfigParser(self): + + p = TomlConfigParser(['soft']) + + for test in get_TomlConfigParser_cases(): + try: + parsed_obj = p.parse(StringIO('[soft]\n'+test['line'])) + except Exception as e: + raise AssertionError("Line %r, error: %s" % (test['line'], str(e))) from e + else: + parsed_obj = dict(parsed_obj) + expected = {test['expected'][0]: test['expected'][1]} + assert parsed_obj==expected, "Line %r" % (test['line']) + + def test_CompositeConfigParser(self): + p = CompositeConfigParser([TomlConfigParser(['soft']), + IniConfigParser(['soft'], True)]) + + for test in (*get_TomlConfigParser_cases(), + *get_IniConfigParser_multiline_text_to_list_cases()): + try: + parsed_obj = p.parse(StringIO('[soft]\n'+test['line'])) + except Exception as e: + raise AssertionError("Line %r, error: %s" % (test['line'], str(e))) from e + else: + parsed_obj = dict(parsed_obj) + expected = {test['expected'][0]: test['expected'][1]} + assert parsed_obj==expected, "Line %r" % (test['line']) + + +INI_SIMPLE_STRINGS = [ + {'line': 'key = value # not_a_comment # not_a_comment', 'expected': ('key', 'value # not_a_comment # not_a_comment', None)}, # that's normal behaviour for configparser + {'line': 'key=value#not_a_comment ', 'expected': ('key', 'value#not_a_comment', None)}, + {'line': 'key=value', 'expected': ('key', 'value', None)}, + {'line': 'key =value', 'expected': ('key', 'value', None)}, + {'line': 'key= value', 'expected': ('key', 'value', None)}, + {'line': 'key = value', 'expected': ('key', 'value', None)}, + {'line': 'key = value', 'expected': ('key', 'value', None)}, + {'line': ' key = value ', 'expected': ('key', 'value', None)}, + {'line': 'key:value', 'expected': ('key', 'value', None)}, + {'line': 'key :value', 'expected': ('key', 'value', None)}, + {'line': 'key: value', 'expected': ('key', 'value', None)}, + {'line': 'key : value', 'expected': ('key', 'value', None)}, + {'line': 'key : value', 'expected': ('key', 'value', None)}, + {'line': ' key : value ', 'expected': ('key', 'value', None)}, +] + +INI_QUOTES_CORNER_CASES = [ + {'line': 'key="', 'expected': ('key', '"', None)}, + {'line': 'key = "', 'expected': ('key', '"', None)}, + {'line': ' key = " ', 'expected': ('key', '"', None)}, + {'line': 'key = ""value""', 'expected': ('key', '""value""', None)}, # Not a valid python, so we get the original value, which is normal + {'line': 'key = \'\'value\'\'', 'expected': ('key', "''value''", None)}, # Idem +] + +INI_QUOTED_STRINGS = [ + {'line': 'key="value"', 'expected': ('key', 'value', None)}, + {'line': 'key = "value"', 'expected': ('key', 'value', None)}, + {'line': ' key = "value" ', 'expected': ('key', 'value', None)}, + {'line': 'key=" value "', 'expected': ('key', ' value ', None)}, + {'line': 'key = " value "', 'expected': ('key', ' value ', None)}, + {'line': ' key = " value " ', 'expected': ('key', ' value ', None)}, + {'line': "key='value'", 'expected': ('key', 'value', None)}, + {'line': "key = 'value'", 'expected': ('key', 'value', None)}, + {'line': " key = 'value' ", 'expected': ('key', 'value', None)}, + {'line': "key=' value '", 'expected': ('key', ' value ', None)}, + {'line': "key = ' value '", 'expected': ('key', ' value ', None)}, + {'line': " key = ' value ' ", 'expected': ('key', ' value ', None)}, + {'line': 'key = \'"value"\'', 'expected': ('key', '"value"', None)}, + {'line': 'key = "\'value\'"', 'expected': ('key', "'value'", None)}, +] + +INI_LOOKS_LIKE_QUOTED_STRINGS = [ + {'line': 'key="value', 'expected': ('key', '"value', None)}, + {'line': 'key = "value', 'expected': ('key', '"value', None)}, + {'line': ' key = "value ', 'expected': ('key', '"value', None)}, + {'line': 'key=value"', 'expected': ('key', 'value"', None)}, + {'line': 'key = value"', 'expected': ('key', 'value"', None)}, + {'line': ' key = value " ', 'expected': ('key', 'value "', None)}, + {'line': "key='value", 'expected': ('key', "'value", None)}, + {'line': "key = 'value", 'expected': ('key', "'value", None)}, + {'line': " key = 'value ", 'expected': ('key', "'value", None)}, + {'line': "key=value'", 'expected': ('key', "value'", None)}, + {'line': "key = value'", 'expected': ('key', "value'", None)}, + {'line': " key = value ' ", 'expected': ('key', "value '", None)}, +] + +INI_BLANK_LINES = [ + {'line': 'key=', 'expected': ('key', '', None)}, + {'line': 'key =', 'expected': ('key', '', None)}, + {'line': 'key= ', 'expected': ('key', '', None)}, + {'line': 'key = ', 'expected': ('key', '', None)}, + {'line': 'key = ', 'expected': ('key', '', None)}, + {'line': ' key = ', 'expected': ('key', '', None)}, + {'line': 'key:', 'expected': ('key', '', None)}, + {'line': 'key :', 'expected': ('key', '', None)}, + {'line': 'key: ', 'expected': ('key', '', None)}, + {'line': 'key : ', 'expected': ('key', '', None)}, + {'line': 'key : ', 'expected': ('key', '', None)}, + {'line': ' key : ', 'expected': ('key', '', None)}, +] + +INI_EQUAL_SIGN_VALUE = [ + {'line': 'key=:', 'expected': ('key', ':', None)}, + {'line': 'key =:', 'expected': ('key', ':', None)}, + {'line': 'key= :', 'expected': ('key', ':', None)}, + {'line': 'key = :', 'expected': ('key', ':', None)}, + {'line': 'key = :', 'expected': ('key', ':', None)}, + {'line': ' key = : ', 'expected': ('key', ':', None)}, + {'line': 'key:=', 'expected': ('key', '=', None)}, + {'line': 'key :=', 'expected': ('key', '=', None)}, + {'line': 'key: =', 'expected': ('key', '=', None)}, + {'line': 'key : =', 'expected': ('key', '=', None)}, + {'line': 'key : =', 'expected': ('key', '=', None)}, + {'line': ' key : = ', 'expected': ('key', '=', None)}, + {'line': 'key==', 'expected': ('key', '=', None)}, + {'line': 'key ==', 'expected': ('key', '=', None)}, + {'line': 'key= =', 'expected': ('key', '=', None)}, + {'line': 'key = =', 'expected': ('key', '=', None)}, + {'line': 'key = =', 'expected': ('key', '=', None)}, + {'line': ' key = = ', 'expected': ('key', '=', None)}, + {'line': 'key::', 'expected': ('key', ':', None)}, + {'line': 'key ::', 'expected': ('key', ':', None)}, + {'line': 'key: :', 'expected': ('key', ':', None)}, + {'line': 'key : :', 'expected': ('key', ':', None)}, + {'line': 'key : :', 'expected': ('key', ':', None)}, + {'line': ' key : : ', 'expected': ('key', ':', None)}, +] + +INI_NEGATIVE_VALUES = [ + {'line': 'key = -10', 'expected': ('key', '-10', None)}, + {'line': 'key : -10', 'expected': ('key', '-10', None)}, + # {'line': 'key -10', 'expected': ('key', '-10', None)}, # Not supported + {'line': 'key = "-10"', 'expected': ('key', '-10', None)}, + {'line': "key = '-10'", 'expected': ('key', '-10', None)}, + {'line': 'key=-10', 'expected': ('key', '-10', None)}, +] + +INI_KEY_SYNTAX_EMPTY = [ + {'line': 'key_underscore=', 'expected': ('key_underscore', '', None)}, + {'line': '_key_underscore=', 'expected': ('_key_underscore', '', None)}, + {'line': 'key_underscore_=', 'expected': ('key_underscore_', '', None)}, + {'line': 'key-dash=', 'expected': ('key-dash', '', None)}, + {'line': 'key@word=', 'expected': ('key@word', '', None)}, + {'line': 'key$word=', 'expected': ('key$word', '', None)}, + {'line': 'key.word=', 'expected': ('key.word', '', None)}, +] + +INI_KEY_SYNTAX = [ + {'line': 'key_underscore = value', 'expected': ('key_underscore', 'value', None)}, + # {'line': 'key_underscore', 'expected': ('key_underscore', 'true', None)}, # Not supported + {'line': '_key_underscore = value', 'expected': ('_key_underscore', 'value', None)}, + # {'line': '_key_underscore', 'expected': ('_key_underscore', 'true', None)}, # Idem + {'line': 'key_underscore_ = value', 'expected': ('key_underscore_', 'value', None)}, + # {'line': 'key_underscore_', 'expected': ('key_underscore_', 'true', None)}, Idem + {'line': 'key-dash = value', 'expected': ('key-dash', 'value', None)}, + # {'line': 'key-dash', 'expected': ('key-dash', 'true', None)}, # Idem + {'line': 'key@word = value', 'expected': ('key@word', 'value', None)}, + # {'line': 'key@word', 'expected': ('key@word', 'true', None)}, Idem + {'line': 'key$word = value', 'expected': ('key$word', 'value', None)}, + # {'line': 'key$word', 'expected': ('key$word', 'true', None)}, Idem + {'line': 'key.word = value', 'expected': ('key.word', 'value', None)}, + # {'line': 'key.word', 'expected': ('key.word', 'true', None)}, Idem +] + +INI_LITERAL_LIST = [ + {'line': 'key = [1,2,3]', 'expected': ('key', ['1','2','3'], None)}, + {'line': 'key = []', 'expected': ('key', [], None)}, + {'line': 'key = ["hello", "world", ]', 'expected': ('key', ["hello", "world"], None)}, + {'line': 'key = [\'hello\', \'world\', ]', 'expected': ('key', ["hello", "world"], None)}, + {'line': 'key = [1,2,3] ', 'expected': ('key', ['1','2','3'], None)}, + {'line': 'key = [\n ] \n', 'expected': ('key', [], None)}, + {'line': 'key = [\n "hello", "world", ] \n\n\n\n', 'expected': ('key', ["hello", "world"], None)}, + {'line': 'key = [\n\n \'hello\', \n \'world\', ]', 'expected': ('key', ["hello", "world"], None)}, + {'line': r'key = "[\"hello\", \"world\", ]"', 'expected': ('key', "[\"hello\", \"world\", ]", None)}, +] + +INI_TRIPPLE_QUOTED_STRINGS = [ + {'line': 'key="""value"""', 'expected': ('key', 'value', None)}, + {'line': 'key = """value"""', 'expected': ('key', 'value', None)}, + {'line': ' key = """value""" ', 'expected': ('key', 'value', None)}, + {'line': 'key=""" value """', 'expected': ('key', ' value ', None)}, + {'line': 'key = """ value """', 'expected': ('key', ' value ', None)}, + {'line': ' key = """ value """ ', 'expected': ('key', ' value ', None)}, + {'line': "key='''value'''", 'expected': ('key', 'value', None)}, + {'line': "key = '''value'''", 'expected': ('key', 'value', None)}, + {'line': " key = '''value''' ", 'expected': ('key', 'value', None)}, + {'line': "key=''' value '''", 'expected': ('key', ' value ', None)}, + {'line': "key = ''' value '''", 'expected': ('key', ' value ', None)}, + {'line': " key = ''' value ''' ", 'expected': ('key', ' value ', None)}, + {'line': 'key = \'\'\'"value"\'\'\'', 'expected': ('key', '"value"', None)}, + {'line': 'key = """\'value\'"""', 'expected': ('key', "'value'", None)}, + {'line': 'key = """\\"value\\""""', 'expected': ('key', '"value"', None)}, +] + +# These test does not pass with TOML (even if toml support tripple quoted strings) because indentation +# is lost while parsing the config with configparser. The behaviour is basically the same as +# running textwrap.dedent() on the text. +INI_TRIPPLE_QUOTED_STRINGS_NOT_COMPATIABLE_WITH_TOML = [ + {'line': 'key = """"value\\""""', 'expected': ('key', '"value"', None)}, # This is valid for ast.literal_eval but not for TOML. + {'line': 'key = """"value" """', 'expected': ('key', '"value" ', None)}, # Idem. + + {'line': 'key = \'\'\'\'value\\\'\'\'\'', 'expected': ('key', "'value'", None)}, # The rest of the test cases are not passing for TOML, + # we get the indented string instead, anyway, it's not onus to test TOML. + {'line': 'key="""\n value\n """', 'expected': ('key', '\nvalue\n', None)}, + {'line': 'key = """\n value\n """', 'expected': ('key', '\nvalue\n', None)}, + {'line': ' key = """\n value\n """ ', 'expected': ('key', '\nvalue\n', None)}, + {'line': "key='''\n value\n '''", 'expected': ('key', '\nvalue\n', None)}, + {'line': "key = '''\n value\n '''", 'expected': ('key', '\nvalue\n', None)}, + {'line': " key = '''\n value\n ''' ", 'expected': ('key', '\nvalue\n', None)}, + {'line': 'key= \'\'\'\n """\n \'\'\'', 'expected': ('key', '\n"""\n', None)}, + {'line': 'key = \'\'\'\n """""\n \'\'\'', 'expected': ('key', '\n"""""\n', None)}, + {'line': ' key = \'\'\'\n ""\n \'\'\' ', 'expected': ('key', '\n""\n', None)}, + {'line': 'key = \'\'\'\n "value"\n \'\'\'', 'expected': ('key', '\n"value"\n', None)}, + {'line': 'key = """\n \'value\'\n """', 'expected': ('key', "\n'value'\n", None)}, + {'line': 'key = """"\n value\\"\n """', 'expected': ('key', '"\nvalue"\n', None)}, + {'line': 'key = """\n \\"value\\"\n """', 'expected': ('key', '\n"value"\n', None)}, + {'line': 'key = """\n "value" \n """', 'expected': ('key', '\n"value"\n', None)}, # trailling white spaces are removed by configparser + {'line': 'key = \'\'\'\n \'value\\\'\n \'\'\'', 'expected': ('key', "\n'value'\n", None)}, + +] + +INI_LOOKS_LIKE_TRIPPLE_QUOTED_STRINGS = [ + {'line': 'key= """', 'expected': ('key', '"""', None)}, + {'line': 'key = """""', 'expected': ('key', '"""""', None)}, + {'line': ' key = """" ', 'expected': ('key', '""""', None)}, + {'line': 'key = """"value""""', 'expected': ('key', '""""value""""', None)}, # Not a valid python, so we get the original value, which is normal + {'line': 'key = \'\'\'\'value\'\'\'\'', 'expected': ('key', "''''value''''", None)}, # Idem + {'line': 'key="""value', 'expected': ('key', '"""value', None)}, + {'line': 'key = """value', 'expected': ('key', '"""value', None)}, + {'line': ' key = """value ', 'expected': ('key', '"""value', None)}, + {'line': 'key=value"""', 'expected': ('key', 'value"""', None)}, + {'line': 'key = value"""', 'expected': ('key', 'value"""', None)}, + {'line': ' key = value """ ', 'expected': ('key', 'value """', None)}, + {'line': "key='''value", 'expected': ('key', "'''value", None)}, + {'line': "key = '''value", 'expected': ('key', "'''value", None)}, + {'line': " key = '''value ", 'expected': ('key', "'''value", None)}, + {'line': "key=value'''", 'expected': ('key', "value'''", None)}, + {'line': "key = value'''", 'expected': ('key', "value'''", None)}, + {'line': " key = value ''' ", 'expected': ('key', "value '''", None)}, +] + +INI_BLANK_LINES_QUOTED = [ + {'line': 'key=""', 'expected': ('key', '', None)}, + {'line': 'key =""', 'expected': ('key', '', None)}, + {'line': 'key= ""', 'expected': ('key', '', None)}, + {'line': 'key = ""', 'expected': ('key', '', None)}, + {'line': 'key = \'\'', 'expected': ('key', '', None)}, + {'line': ' key =\'\' ', 'expected': ('key', '', None)}, +] + +INI_BLANK_LINES_QUOTED_COLONS = [ + {'line': 'key:\'\'', 'expected': ('key', '', None)}, + {'line': 'key :\'\'', 'expected': ('key', '', None)}, + {'line': 'key: \'\'', 'expected': ('key', '', None)}, + {'line': 'key : \'\'', 'expected': ('key', '', None)}, + {'line': 'key :\'\' ', 'expected': ('key', '', None)}, + {'line': ' key : "" ', 'expected': ('key', '', None)}, +] + +INI_MULTILINE_STRING_LIST = [ + {'line': 'key = \n hello\n hoho', 'expected': ('key', ["hello", "hoho"], None)}, + {'line': 'key = hello\n hoho', 'expected': ('key', ["hello", "hoho"], None)}, + {'line': 'key : "hello"\n \'hoho\'', 'expected': ('key', ["\"hello\"", "'hoho'"], None)}, # quotes are kept when converting multine strings to list. + {'line': 'key : \n hello\n hoho\n', 'expected': ('key', ["hello", "hoho"], None)}, + {'line': 'key = \n hello\n hoho\n \n\n ', 'expected': ('key', ["hello", "hoho"], None)}, + {'line': 'key = \n hello\n;comment\n\n hoho\n \n\n ', 'expected': ('key', ["hello", "hoho"], None)}, +] + +def get_IniConfigParser_cases(): + return (INI_SIMPLE_STRINGS + + INI_QUOTED_STRINGS + + INI_BLANK_LINES + + INI_NEGATIVE_VALUES + + INI_BLANK_LINES_QUOTED + + INI_BLANK_LINES_QUOTED_COLONS + + INI_KEY_SYNTAX + + INI_KEY_SYNTAX_EMPTY + + INI_LITERAL_LIST + + INI_TRIPPLE_QUOTED_STRINGS + + INI_LOOKS_LIKE_TRIPPLE_QUOTED_STRINGS + + INI_QUOTES_CORNER_CASES + + INI_LOOKS_LIKE_QUOTED_STRINGS) + +def get_IniConfigParser_multiline_text_to_list_cases(): + cases = get_IniConfigParser_cases() + for case in INI_BLANK_LINES + INI_KEY_SYNTAX_EMPTY: # when multiline_text_to_list is enabled blank lines are simply ignored. + cases.remove(case) + cases.extend(INI_MULTILINE_STRING_LIST) + return cases + +def get_TomlConfigParser_cases(): + return (INI_QUOTED_STRINGS + + INI_BLANK_LINES_QUOTED + + INI_LITERAL_LIST + + INI_TRIPPLE_QUOTED_STRINGS) diff --git a/tox.ini b/tox.ini index 6ee0cc1e..a88f4c33 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,9 @@ envlist = py35, py36, py37, py38, py39, py310, py311, pypy, pypy3 [testenv] #setenv = PYTHONPATH = {toxinidir}:{toxinidir}/configmanager - -commands = python setup.py test - # python -m unittest discover +extras = + test +commands = pytest [testenv:py36] basepython=python3.6