diff --git a/.github/workflows/apidocs.yml b/.github/workflows/apidocs.yml index 6c5c862..7670de4 100644 --- a/.github/workflows/apidocs.yml +++ b/.github/workflows/apidocs.yml @@ -13,9 +13,7 @@ jobs: deploy: runs-on: macos-latest permissions: - contents: read - pages: write - id-token: write + contents: write steps: - uses: actions/checkout@v4 @@ -33,7 +31,7 @@ jobs: run: ./apidocs.sh - name: Push API documentation to Github Pages - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./apidocs diff --git a/configargparse.py b/configargparse.py index db92aa6..2d7030b 100644 --- a/configargparse.py +++ b/configargparse.py @@ -516,13 +516,27 @@ def __call__(self): def parse(self, stream): """Parses the keys and values from a TOML config file.""" - # parse with configparser to allow multi-line values - import toml - + # Use tomllib (Python 3.11+) if available, otherwise fall back to toml package try: - config = toml.load(stream) - except Exception as e: - raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) + import tomllib + + # tomllib requires binary mode, but always use loads() for compatibility + content = stream.read() + # If content is bytes, decode it; if string, use as-is + if isinstance(content, bytes): + content = content.decode("utf-8") + try: + config = tomllib.loads(content) + except Exception as e: + raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) + except ImportError: + # Fall back to toml package (supports text mode) + import toml + + try: + config = toml.load(stream) + except Exception as e: + raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) # convert to dict and filter based on section names result = OrderedDict() @@ -550,6 +564,40 @@ def get_syntax_description(self): "See https://github.com/toml-lang/toml/blob/v0.5.0/README.md for details." ) + def serialize(self, items): + """Serialize items to TOML format with section support. + + Note: Requires the 'toml' package for serialization (pip install toml). + Python 3.11's tomllib only supports reading, not writing. + """ + # lazy-import to avoid dependency unless class is used + try: + import toml + except ImportError: + raise ConfigFileParserException( + "The 'toml' package is required for TOML serialization. " + "Install it with: pip install toml" + ) + + # Put items in the first configured section + if not self.sections: + # No sections configured, serialize as flat TOML + return toml.dumps(dict(items)) + + # Create nested dict structure for section + section_name = self.sections[0] + sections = parse_toml_section_name(section_name) + + # Build nested dict from section path + result = {} + current = result + for i, section in enumerate(sections[:-1]): + current[section] = {} + current = current[section] + current[sections[-1]] = dict(items) + + return toml.dumps(result) + class IniConfigParser(ConfigFileParser): """ @@ -598,7 +646,7 @@ class IniConfigParser(ConfigFileParser): def __init__(self, sections, split_ml_text_to_list): """ :param sections: The section names bounded to the new parser. - :param split_ml_text_to_list: Wether to convert multiline strings to list + :param split_ml_text_to_list: Whether to convert multiline strings to list """ super().__init__() self.sections = sections @@ -671,6 +719,37 @@ def get_syntax_description(self): ) return msg + def serialize(self, items): + """Serialize items to INI format with section support.""" + config = configparser.ConfigParser() + + # Use the first configured section + section_name = self.sections[0] if self.sections else "DEFAULT" + + # Add section if not DEFAULT + if section_name != "DEFAULT" and section_name != configparser.DEFAULTSECT: + config.add_section(section_name) + + # Add items to the section + for key, value in items.items(): + if isinstance(value, list): + # Handle lists + if self.split_ml_text_to_list: + # Multi-line format + config.set( + section_name, key, "\n" + "\n".join(str(v) for v in value) + ) + else: + # Python list syntax + config.set(section_name, key, str(value)) + else: + config.set(section_name, key, str(value)) + + stream = StringIO() + config.write(stream) + stream.seek(0) + return stream.read() + class CompositeConfigParser(ConfigFileParser): """ @@ -689,12 +768,22 @@ def __call__(self): def parse(self, stream): errors = [] - for p in self.parsers: + for i, p in enumerate(self.parsers): try: return p.parse(stream) # type: ignore[no-any-return] except Exception as e: - stream.seek(0) errors.append(e) + # Try to seek back to beginning for next parser + # If this is not the last parser and seek fails, we can't continue + if i < len(self.parsers) - 1: + try: + stream.seek(0) + except (AttributeError, OSError) as seek_error: + # Stream doesn't support seeking + raise ConfigFileParserException( + f"Error parsing config with {p.__class__.__name__}: {e}. " + f"Cannot try additional parsers because stream is not seekable." + ) from e raise ConfigFileParserException( f"Error parsing config: {', '.join(repr(str(e)) for e in errors)}" ) @@ -715,6 +804,15 @@ def guess_format_name(classname): msg += f"[{i+1}] {guess_format_name(parser.__class__.__name__)}: {parser.get_syntax_description()} \n" return msg + def serialize(self, items): + """Serialize items using the first parser in the composite.""" + if not self.parsers: + raise ConfigFileParserException( + "No parsers configured in CompositeConfigParser" + ) + # Use the first parser to serialize + return self.parsers[0].serialize(items) # type: ignore[no-any-return] + # used while parsing args to keep track of where they came from _COMMAND_LINE_SOURCE_KEY = "command_line" @@ -844,7 +942,8 @@ def __init__(self, *args, **kwargs): is_write_out_config_file_arg=True, ) - # TODO: delete me! + # Workaround for Python < 3.9: exit_on_error parameter was added in 3.9 + # This can be removed when minimum Python version is raised to 3.9+ if sys.version_info < (3, 9): self.exit_on_error = True @@ -1258,9 +1357,7 @@ def convert_item_to_command_line_arg(self, action, key, value): # --no-foo args.append(action.option_strings[1]) elif isinstance(action, argparse._CountAction): - for arg in args: - if any([arg.startswith(s) for s in action.option_strings]): - value = 0 + # For count actions, repeat the flag the number of times specified args += [action.option_strings[0]] * int(value) else: self.error( @@ -1599,8 +1696,9 @@ def already_on_command_line( ) -# TODO: Update to latest version of pydoctor when https://github.com/twisted/pydoctor/pull/414 has been merged -# such that the alises can be documented automatically. +# NOTE: Aliases below are not auto-documented. This could be improved by updating +# to latest version of pydoctor when https://github.com/twisted/pydoctor/pull/414 +# has been merged, which would allow aliases to be documented automatically. # wrap ArgumentParser's add_argument(..) method with the one above argparse._ActionsContainer.original_add_argument_method = ( diff --git a/tests/test_configargparse.py b/tests/test_configargparse.py index d46fde6..1f7bcb0 100644 --- a/tests/test_configargparse.py +++ b/tests/test_configargparse.py @@ -18,6 +18,12 @@ else: OPTIONAL_ARGS_STRING = "optional arguments" +# In Python 3.13+, short option metavars are no longer repeated +if sys.version_info >= (3, 13): + SHORT_OPT_METAVAR = "" # e.g., "-g, --my-cfg-file MY_CFG_FILE" +else: + SHORT_OPT_METAVAR = "(?: {metavar})?" # e.g., "-g MY_CFG_FILE, --my-cfg-file MY_CFG_FILE" or "-g, --my-cfg-file MY_CFG_FILE" + # set COLUMNS to get expected wrapping os.environ["COLUMNS"] = "80" @@ -315,6 +321,9 @@ def testBasicCase2(self, use_groups=False): ) if not use_groups: + short_g = SHORT_OPT_METAVAR.format(metavar="MY_CFG_FILE") + short_d = SHORT_OPT_METAVAR.format(metavar="DBSNP") + short_f = SHORT_OPT_METAVAR.format(metavar="FRMT") self.assertRegex( self.format_help(), "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?" @@ -325,13 +334,16 @@ def testBasicCase2(self, use_groups=False): " -h, --help \\s+ show this help message and exit\n" " --genome GENOME \\s+ Path to genome file\n" " -v\n" - " -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n" - " -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" - " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" + f" -g{short_g}, --my-cfg-file MY_CFG_FILE\n" + f" -d{short_d}, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" + f" -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" % OPTIONAL_ARGS_STRING + 7 * r"(.+\s*)", ) else: + short_g = SHORT_OPT_METAVAR.format(metavar="MY_CFG_FILE") + short_d = SHORT_OPT_METAVAR.format(metavar="DBSNP") + short_f = SHORT_OPT_METAVAR.format(metavar="FRMT") self.assertRegex( self.format_help(), "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?" @@ -343,10 +355,10 @@ def testBasicCase2(self, use_groups=False): "g1:\n" " --genome GENOME \\s+ Path to genome file\n" " -v\n" - " -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n\n" + f" -g{short_g}, --my-cfg-file MY_CFG_FILE\n\n" "g2:\n" - " -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" - " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" + f" -d{short_d}, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n" + f" -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n" % OPTIONAL_ARGS_STRING + 7 * r"(.+\s*)", ) @@ -469,15 +481,18 @@ def testMutuallyExclusiveArgs(self): " --format: \\s+ BED\n", ) + short_f1 = SHORT_OPT_METAVAR.format(metavar="TYPE1_CFG_FILE") + short_f2 = SHORT_OPT_METAVAR.format(metavar="TYPE2_CFG_FILE") + short_f = SHORT_OPT_METAVAR.format(metavar="FRMT") self.assertRegex( self.format_help(), r"usage: .* \[-h\] --genome GENOME \[-v\]\s+\(-f1 TYPE1_CFG_FILE \|" r"\s+-f2 TYPE2_CFG_FILE\)\s+\(-f FRMT \| -b\)\n\n" "%s:\n" " -h, --help show this help message and exit\n" - " -f1(?: TYPE1_CFG_FILE)?, --type1-cfg-file TYPE1_CFG_FILE\n" - " -f2(?: TYPE2_CFG_FILE)?, --type2-cfg-file TYPE2_CFG_FILE\n" - " -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n" + f" -f1{short_f1}, --type1-cfg-file TYPE1_CFG_FILE\n" + f" -f2{short_f2}, --type2-cfg-file TYPE2_CFG_FILE\n" + f" -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n" " -b, --bam\\s+\\[env var: BAM_FORMAT\\]\n\n" "group1:\n" " --genome GENOME Path to genome file\n" @@ -1054,12 +1069,13 @@ def testConstructor_ConfigFileArgs(self): ns = self.parse("-c " + temp_cfg2.name) self.assertEqual(ns.genome, "hg20") + short_c = SHORT_OPT_METAVAR.format(metavar="CONFIG_FILE") self.assertRegex( self.format_help(), r"usage: .* \[-h\] -c CONFIG_FILE --genome GENOME\n\n" r"%s:\n" r" -h, --help\s+ show this help message and exit\n" - r" -c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+ my config file\n" + rf" -c{short_c}, --config CONFIG_FILE\s+ my config file\n" r" --genome GENOME\s+ Path to genome file\n\n" % OPTIONAL_ARGS_STRING + 5 * r"(.+\s*)", ) @@ -1123,14 +1139,16 @@ def test_FormatHelp(self): self.add_arg("--arg1", help="Arg1 help text", required=True) self.add_arg("--flag", help="Flag help text", action="store_true") + short_c = SHORT_OPT_METAVAR.format(metavar="CONFIG_FILE") + short_w = SHORT_OPT_METAVAR.format(metavar="CONFIG_OUTPUT_PATH") self.assertRegex( self.format_help(), r"usage: .* \[-h\] -c CONFIG_FILE\s+" r"\[-w CONFIG_OUTPUT_PATH\]\s* --arg1\s+ARG1\s*\[--flag\]\s*" "%s:\\s*" "-h, --help \\s* show this help message and exit " - r"-c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+my config file " - r"-w(?: CONFIG_OUTPUT_PATH)?, --write-config CONFIG_OUTPUT_PATH takes " + rf"-c{short_c}, --config CONFIG_FILE\s+my config file " + rf"-w{short_w}, --write-config CONFIG_OUTPUT_PATH takes " r"the current command line args and writes them " r"out to a config file at the given path, then exits " r"--arg1 ARG1 Arg1 help text " @@ -1827,15 +1845,20 @@ def test_advanced(self): parser = configargparse.TomlConfigParser(["tool.section"]) self.assertEqual(parser.parse(f), {"key1": "toml1", "key2": ["1", "2", "3"]}) - def test_fails_binary_read(self): + @unittest.skipIf( + sys.version_info < (3, 11), + "Binary mode only supported with tomllib (Python 3.11+)", + ) + def test_binary_read_works(self): + # Binary mode now works with tomllib (Python 3.11+) f = self.write_toml_file( b"""[tool.section]\nkey1 = "toml1" """, obj=BytesIO, ) parser = configargparse.TomlConfigParser(["tool.section"]) - with self.assertRaises(configargparse.ConfigFileParserException): - parser.parse(f) + # Should successfully parse binary stream + self.assertEqual(parser.parse(f), {"key1": "toml1"}) class TestCompositeConfigParser(unittest.TestCase):