Fix critical bugs and add Python 3.11+ tomllib support

.github/workflows/apidocs.yml

@@ -13,9 +13,7 @@ jobs:
   deploy:
     runs-on: macos-latest
     permissions:
-      contents: read
-      pages: write
-      id-token: write
+      contents: write
 
     steps:
       - uses: actions/checkout@v4
@@ -33,7 +31,7 @@ jobs:
         run: ./apidocs.sh
 
       - name: Push API documentation to Github Pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e   # v4.0.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./apidocs

configargparse.py

@@ -516,13 +516,27 @@ def __call__(self):
 
     def parse(self, stream):
         """Parses the keys and values from a TOML config file."""
-        # parse with configparser to allow multi-line values
-        import toml
-
+        # Use tomllib (Python 3.11+) if available, otherwise fall back to toml package
         try:
-            config = toml.load(stream)
-        except Exception as e:
-            raise ConfigFileParserException("Couldn't parse TOML file: %s" % e)
+            import tomllib
+
+            # tomllib requires binary mode, but always use loads() for compatibility
+            content = stream.read()
+            # If content is bytes, decode it; if string, use as-is
+            if isinstance(content, bytes):
+                content = content.decode("utf-8")
+            try:
+                config = tomllib.loads(content)
+            except Exception as e:
+                raise ConfigFileParserException("Couldn't parse TOML file: %s" % e)
+        except ImportError:
+            # Fall back to toml package (supports text mode)
+            import toml
+
+            try:
+                config = toml.load(stream)
+            except Exception as e:
+                raise ConfigFileParserException("Couldn't parse TOML file: %s" % e)
 
         # convert to dict and filter based on section names
         result = OrderedDict()
@@ -550,6 +564,40 @@ def get_syntax_description(self):
             "See https://github.com/toml-lang/toml/blob/v0.5.0/README.md for details."
         )
 
+    def serialize(self, items):
+        """Serialize items to TOML format with section support.
+
+        Note: Requires the 'toml' package for serialization (pip install toml).
+        Python 3.11's tomllib only supports reading, not writing.
+        """
+        # lazy-import to avoid dependency unless class is used
+        try:
+            import toml
+        except ImportError:
+            raise ConfigFileParserException(
+                "The 'toml' package is required for TOML serialization. "
+                "Install it with: pip install toml"
+            )
+
+        # Put items in the first configured section
+        if not self.sections:
+            # No sections configured, serialize as flat TOML
+            return toml.dumps(dict(items))
+
+        # Create nested dict structure for section
+        section_name = self.sections[0]
+        sections = parse_toml_section_name(section_name)
+
+        # Build nested dict from section path
+        result = {}
+        current = result
+        for i, section in enumerate(sections[:-1]):
+            current[section] = {}
+            current = current[section]
+        current[sections[-1]] = dict(items)
+
+        return toml.dumps(result)
+
 
 class IniConfigParser(ConfigFileParser):
     """
@@ -598,7 +646,7 @@ class IniConfigParser(ConfigFileParser):
     def __init__(self, sections, split_ml_text_to_list):
         """
         :param sections: The section names bounded to the new parser.
-        :param split_ml_text_to_list: Wether to convert multiline strings to list
+        :param split_ml_text_to_list: Whether to convert multiline strings to list
         """
         super().__init__()
         self.sections = sections
@@ -671,6 +719,37 @@ def get_syntax_description(self):
             )
         return msg
 
+    def serialize(self, items):
+        """Serialize items to INI format with section support."""
+        config = configparser.ConfigParser()
+
+        # Use the first configured section
+        section_name = self.sections[0] if self.sections else "DEFAULT"
+
+        # Add section if not DEFAULT
+        if section_name != "DEFAULT" and section_name != configparser.DEFAULTSECT:
+            config.add_section(section_name)
+
+        # Add items to the section
+        for key, value in items.items():
+            if isinstance(value, list):
+                # Handle lists
+                if self.split_ml_text_to_list:
+                    # Multi-line format
+                    config.set(
+                        section_name, key, "\n" + "\n".join(str(v) for v in value)
+                    )
+                else:
+                    # Python list syntax
+                    config.set(section_name, key, str(value))
+            else:
+                config.set(section_name, key, str(value))
+
+        stream = StringIO()
+        config.write(stream)
+        stream.seek(0)
+        return stream.read()
+
 
 class CompositeConfigParser(ConfigFileParser):
     """
@@ -689,12 +768,22 @@ def __call__(self):
 
     def parse(self, stream):
         errors = []
-        for p in self.parsers:
+        for i, p in enumerate(self.parsers):
             try:
                 return p.parse(stream)  # type: ignore[no-any-return]
             except Exception as e:
-                stream.seek(0)
                 errors.append(e)
+                # Try to seek back to beginning for next parser
+                # If this is not the last parser and seek fails, we can't continue
+                if i < len(self.parsers) - 1:
+                    try:
+                        stream.seek(0)
+                    except (AttributeError, OSError) as seek_error:
+                        # Stream doesn't support seeking
+                        raise ConfigFileParserException(
+                            f"Error parsing config with {p.__class__.__name__}: {e}. "
+                            f"Cannot try additional parsers because stream is not seekable."
+                        ) from e
         raise ConfigFileParserException(
             f"Error parsing config: {', '.join(repr(str(e)) for e in errors)}"
         )
@@ -715,6 +804,15 @@ def guess_format_name(classname):
             msg += f"[{i+1}] {guess_format_name(parser.__class__.__name__)}: {parser.get_syntax_description()} \n"
         return msg
 
+    def serialize(self, items):
+        """Serialize items using the first parser in the composite."""
+        if not self.parsers:
+            raise ConfigFileParserException(
+                "No parsers configured in CompositeConfigParser"
+            )
+        # Use the first parser to serialize
+        return self.parsers[0].serialize(items)  # type: ignore[no-any-return]
+
 
 # used while parsing args to keep track of where they came from
 _COMMAND_LINE_SOURCE_KEY = "command_line"
@@ -844,7 +942,8 @@ def __init__(self, *args, **kwargs):
                 is_write_out_config_file_arg=True,
             )
 
-        # TODO: delete me!
+        # Workaround for Python < 3.9: exit_on_error parameter was added in 3.9
+        # This can be removed when minimum Python version is raised to 3.9+
         if sys.version_info < (3, 9):
             self.exit_on_error = True
 
@@ -1258,9 +1357,7 @@ def convert_item_to_command_line_arg(self, action, key, value):
                     # --no-foo
                     args.append(action.option_strings[1])
             elif isinstance(action, argparse._CountAction):
-                for arg in args:
-                    if any([arg.startswith(s) for s in action.option_strings]):
-                        value = 0
+                # For count actions, repeat the flag the number of times specified
                 args += [action.option_strings[0]] * int(value)
             else:
                 self.error(
@@ -1599,8 +1696,9 @@ def already_on_command_line(
     )
 
 
-# TODO: Update to latest version of pydoctor when https://github.com/twisted/pydoctor/pull/414 has been merged
-# such that the alises can be documented automatically.
+# NOTE: Aliases below are not auto-documented. This could be improved by updating
+# to latest version of pydoctor when https://github.com/twisted/pydoctor/pull/414
+# has been merged, which would allow aliases to be documented automatically.
 
 # wrap ArgumentParser's add_argument(..) method with the one above
 argparse._ActionsContainer.original_add_argument_method = (

tests/test_configargparse.py

@@ -18,6 +18,12 @@
 else:
     OPTIONAL_ARGS_STRING = "optional arguments"
 
+# In Python 3.13+, short option metavars are no longer repeated
+if sys.version_info >= (3, 13):
+    SHORT_OPT_METAVAR = ""  # e.g., "-g, --my-cfg-file MY_CFG_FILE"
+else:
+    SHORT_OPT_METAVAR = "(?: {metavar})?"  # e.g., "-g MY_CFG_FILE, --my-cfg-file MY_CFG_FILE" or "-g, --my-cfg-file MY_CFG_FILE"
+
 # set COLUMNS to get expected wrapping
 os.environ["COLUMNS"] = "80"
 
@@ -315,6 +321,9 @@ def testBasicCase2(self, use_groups=False):
         )
 
         if not use_groups:
+            short_g = SHORT_OPT_METAVAR.format(metavar="MY_CFG_FILE")
+            short_d = SHORT_OPT_METAVAR.format(metavar="DBSNP")
+            short_f = SHORT_OPT_METAVAR.format(metavar="FRMT")
             self.assertRegex(
                 self.format_help(),
                 "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?"
@@ -325,13 +334,16 @@ def testBasicCase2(self, use_groups=False):
                 "  -h, --help \\s+ show this help message and exit\n"
                 "  --genome GENOME \\s+ Path to genome file\n"
                 "  -v\n"
-                "  -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n"
-                "  -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n"
-                "  -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n"
+                f"  -g{short_g}, --my-cfg-file MY_CFG_FILE\n"
+                f"  -d{short_d}, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n"
+                f"  -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n"
                 % OPTIONAL_ARGS_STRING
                 + 7 * r"(.+\s*)",
             )
         else:
+            short_g = SHORT_OPT_METAVAR.format(metavar="MY_CFG_FILE")
+            short_d = SHORT_OPT_METAVAR.format(metavar="DBSNP")
+            short_f = SHORT_OPT_METAVAR.format(metavar="FRMT")
             self.assertRegex(
                 self.format_help(),
                 "usage: .* \\[-h\\] --genome GENOME \\[-v\\] -g MY_CFG_FILE\n?"
@@ -343,10 +355,10 @@ def testBasicCase2(self, use_groups=False):
                 "g1:\n"
                 "  --genome GENOME \\s+ Path to genome file\n"
                 "  -v\n"
-                "  -g(?: MY_CFG_FILE)?, --my-cfg-file MY_CFG_FILE\n\n"
+                f"  -g{short_g}, --my-cfg-file MY_CFG_FILE\n\n"
                 "g2:\n"
-                "  -d(?: DBSNP)?, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n"
-                "  -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n"
+                f"  -d{short_d}, --dbsnp DBSNP\\s+\\[env var: DBSNP_PATH\\]\n"
+                f"  -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n\n"
                 % OPTIONAL_ARGS_STRING
                 + 7 * r"(.+\s*)",
             )
@@ -469,15 +481,18 @@ def testMutuallyExclusiveArgs(self):
             "  --format: \\s+ BED\n",
         )
 
+        short_f1 = SHORT_OPT_METAVAR.format(metavar="TYPE1_CFG_FILE")
+        short_f2 = SHORT_OPT_METAVAR.format(metavar="TYPE2_CFG_FILE")
+        short_f = SHORT_OPT_METAVAR.format(metavar="FRMT")
         self.assertRegex(
             self.format_help(),
             r"usage: .* \[-h\] --genome GENOME \[-v\]\s+\(-f1 TYPE1_CFG_FILE \|"
             r"\s+-f2 TYPE2_CFG_FILE\)\s+\(-f FRMT \| -b\)\n\n"
             "%s:\n"
             "  -h, --help            show this help message and exit\n"
-            "  -f1(?: TYPE1_CFG_FILE)?, --type1-cfg-file TYPE1_CFG_FILE\n"
-            "  -f2(?: TYPE2_CFG_FILE)?, --type2-cfg-file TYPE2_CFG_FILE\n"
-            "  -f(?: FRMT)?, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n"
+            f"  -f1{short_f1}, --type1-cfg-file TYPE1_CFG_FILE\n"
+            f"  -f2{short_f2}, --type2-cfg-file TYPE2_CFG_FILE\n"
+            f"  -f{short_f}, --format FRMT\\s+\\[env var: OUTPUT_FORMAT\\]\n"
             "  -b, --bam\\s+\\[env var: BAM_FORMAT\\]\n\n"
             "group1:\n"
             "  --genome GENOME       Path to genome file\n"
@@ -1054,12 +1069,13 @@ def testConstructor_ConfigFileArgs(self):
         ns = self.parse("-c " + temp_cfg2.name)
         self.assertEqual(ns.genome, "hg20")
 
+        short_c = SHORT_OPT_METAVAR.format(metavar="CONFIG_FILE")
         self.assertRegex(
             self.format_help(),
             r"usage: .* \[-h\] -c CONFIG_FILE --genome GENOME\n\n"
             r"%s:\n"
             r"  -h, --help\s+ show this help message and exit\n"
-            r"  -c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+ my config file\n"
+            rf"  -c{short_c}, --config CONFIG_FILE\s+ my config file\n"
             r"  --genome GENOME\s+ Path to genome file\n\n" % OPTIONAL_ARGS_STRING
             + 5 * r"(.+\s*)",
         )
@@ -1123,14 +1139,16 @@ def test_FormatHelp(self):
         self.add_arg("--arg1", help="Arg1 help text", required=True)
         self.add_arg("--flag", help="Flag help text", action="store_true")
 
+        short_c = SHORT_OPT_METAVAR.format(metavar="CONFIG_FILE")
+        short_w = SHORT_OPT_METAVAR.format(metavar="CONFIG_OUTPUT_PATH")
         self.assertRegex(
             self.format_help(),
             r"usage: .* \[-h\] -c CONFIG_FILE\s+"
             r"\[-w CONFIG_OUTPUT_PATH\]\s* --arg1\s+ARG1\s*\[--flag\]\s*"
             "%s:\\s*"
             "-h, --help \\s* show this help message and exit "
-            r"-c(?: CONFIG_FILE)?, --config CONFIG_FILE\s+my config file "
-            r"-w(?: CONFIG_OUTPUT_PATH)?, --write-config CONFIG_OUTPUT_PATH takes "
+            rf"-c{short_c}, --config CONFIG_FILE\s+my config file "
+            rf"-w{short_w}, --write-config CONFIG_OUTPUT_PATH takes "
             r"the current command line args and writes them "
             r"out to a config file at the given path, then exits "
             r"--arg1 ARG1 Arg1 help text "
@@ -1827,15 +1845,20 @@ def test_advanced(self):
         parser = configargparse.TomlConfigParser(["tool.section"])
         self.assertEqual(parser.parse(f), {"key1": "toml1", "key2": ["1", "2", "3"]})
 
-    def test_fails_binary_read(self):
+    @unittest.skipIf(
+        sys.version_info < (3, 11),
+        "Binary mode only supported with tomllib (Python 3.11+)",
+    )
+    def test_binary_read_works(self):
+        # Binary mode now works with tomllib (Python 3.11+)
         f = self.write_toml_file(
             b"""[tool.section]\nkey1 = "toml1"
             """,
             obj=BytesIO,
         )
         parser = configargparse.TomlConfigParser(["tool.section"])
-        with self.assertRaises(configargparse.ConfigFileParserException):
-            parser.parse(f)
+        # Should successfully parse binary stream
+        self.assertEqual(parser.parse(f), {"key1": "toml1"})
 
 
 class TestCompositeConfigParser(unittest.TestCase):