Merge pull request matplotlib#22950 from anntzer/mathtextsymbols

MNT: Simplify definition of mathtext symbols & correctly end tokens in mathtext parsing

Author: tacaswell

doc/api/next_api_changes/behavior/22950-AL.rst

@@ -0,0 +1,3 @@
+``$\doteq \doteqdot \dotminus \dotplus \dots$`` are now surrounded by extra space
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+... because they are correctly treated as relational or binary operators.

lib/matplotlib/_mathtext.py

@@ -8,6 +8,7 @@
 import functools
 import logging
 import os
+import re
 import types
 import unicodedata
 
@@ -1648,7 +1649,8 @@ class _MathStyle(enum.Enum):
       \cdot           \bigtriangledown         \bigcirc
       \cap            \triangleleft            \dagger
       \cup            \triangleright           \ddagger
-      \uplus          \lhd                     \amalg'''.split())
+      \uplus          \lhd                     \amalg
+      \dotplus        \dotminus'''.split())
 
     _relation_symbols = set(r'''
       = < > :
@@ -1661,7 +1663,7 @@ class _MathStyle(enum.Enum):
       \sqsubset   \sqsupset   \neq     \smile
       \sqsubseteq \sqsupseteq \doteq   \frown
       \in         \ni         \propto  \vdash
-      \dashv      \dots       \dotplus \doteqdot'''.split())
+      \dashv      \dots       \doteqdot'''.split())
 
     _arrow_symbols = set(r'''
       \leftarrow              \longleftarrow           \uparrow
@@ -1717,24 +1719,36 @@ def set_names_and_parse_actions():
 
         # Root definitions.
 
+        # In TeX parlance, a csname is a control sequence name (a "\foo").
+        def csnames(group, names):
+            ends_with_alpha = []
+            ends_with_nonalpha = []
+            for name in names:
+                if name[-1].isalpha():
+                    ends_with_alpha.append(name)
+                else:
+                    ends_with_nonalpha.append(name)
+            return Regex(r"\\(?P<{}>(?:{})(?![A-Za-z]){})".format(
+                group,
+                "|".join(map(re.escape, ends_with_alpha)),
+                "".join(f"|{s}" for s in map(re.escape, ends_with_nonalpha)),
+            ))
+
         p.float_literal  = Regex(r"[-+]?([0-9]+\.?[0-9]*|\.[0-9]+)")
         p.space          = oneOf(self._space_widths)("space")
 
         p.style_literal  = oneOf(
             [str(e.value) for e in self._MathStyle])("style_literal")
 
-        p.single_symbol  = Regex(
-            r"([a-zA-Z0-9 +\-*/<>=:,.;!\?&'@()\[\]|%s])|(\\[%%${}\[\]_|])" %
-            "\U00000080-\U0001ffff"  # unicode range
-        )("sym")
-        p.accentprefixed = "\\" + oneOf(self._accentprefixed)("sym")
-        p.symbol_name    = (
-            oneOf([rf"\{sym}" for sym in tex2uni])("sym")
-            + Regex("(?=[^A-Za-z]|$)").leaveWhitespace())
-        p.symbol         = (p.single_symbol | p.symbol_name).leaveWhitespace()
+        p.symbol         = Regex(
+            r"[a-zA-Z0-9 +\-*/<>=:,.;!\?&'@()\[\]|\U00000080-\U0001ffff]"
+            r"|\\[%${}\[\]_|]"
+            + r"|\\(?:{})(?![A-Za-z])".format(
+                "|".join(map(re.escape, tex2uni)))
+        )("sym").leaveWhitespace()
         p.unknown_symbol = Regex(r"\\[A-Za-z]*")("name")
 
-        p.font           = "\\" + oneOf(self._fontnames)("font")
+        p.font           = csnames("font", self._fontnames)
         p.start_group    = (
             Optional(r"\math" + oneOf(self._fontnames)("font")) + "{")
         p.end_group      = Literal("}")
@@ -1771,11 +1785,10 @@ def set_names_and_parse_actions():
         p.customspace <<= cmd(r"\hspace", "{" + p.float_literal("space") + "}")
 
         p.accent <<= (
-            "\\"
-            + oneOf([*self._accent_map, *self._wide_accents])("accent")
+            csnames("accent", [*self._accent_map, *self._wide_accents])
             - p.placeable("sym"))
 
-        p.function     <<= "\\" + oneOf(self._function_names)("name")
+        p.function <<= csnames("name", self._function_names)
         p.operatorname <<= cmd(
             r"\operatorname",
             "{" + ZeroOrMore(p.simple | p.unknown_symbol)("name") + "}")
@@ -1816,10 +1829,8 @@ def set_names_and_parse_actions():
             p.optional_group("annotation") + p.optional_group("body"))
 
         p.placeable     <<= (
-            p.accentprefixed  # Must be before accent so named symbols that are
-                              # prefixed with an accent name work
-            | p.accent   # Must be before symbol as all accents are symbols
-            | p.symbol   # Must be third to catch all named symbols and single
+            p.accent     # Must be before symbol as all accents are symbols
+            | p.symbol   # Must be second to catch all named symbols and single
                          # chars not in a group
             | p.function
             | p.operatorname
@@ -2019,8 +2030,6 @@ def symbol(self, s, loc, toks):
                 return [Hlist([char, self._make_space(0.2)], do_kern=True)]
         return [char]
 
-    accentprefixed = symbol
-
     def unknown_symbol(self, s, loc, toks):
         raise ParseFatalException(s, loc, f"Unknown symbol: {toks['name']}")
 
@@ -2049,12 +2058,6 @@ def unknown_symbol(self, s, loc, toks):
 
     _wide_accents = set(r"widehat widetilde widebar".split())
 
-    # make a lambda and call it to get the namespace right
-    _accentprefixed = (lambda am: [
-        p for p in tex2uni
-        if any(p.startswith(a) and a != p for a in am)
-    ])(set(_accent_map))
-
     def accent(self, s, loc, toks):
         state = self.get_state()
         thickness = state.get_current_underline_thickness()