Add color to tokens

tests/test_web_driver.py

@@ -16,24 +16,27 @@
 class WebDriverTests(unittest.TestCase):
     def test_view_tokens_includes_token_names(self) -> None:
         rendered = driver.view_tokens("x = 1\n")
-        self.assertIn("NAME", rendered)
-        self.assertIn("NUMBER", rendered)
+        self.assertIn("NAME", rendered["text"])
+        self.assertIn("NUMBER", rendered["text"])
 
     def test_view_ast_returns_module_dump(self) -> None:
         rendered = driver.view_ast("x = 1\n", optimize=False)
         self.assertTrue(rendered["html"])
-        self.assertIn(">Module</span>(", rendered["text"])
-        self.assertIn(">Assign</span>(", rendered["text"])
+        self.assertIn("Module", rendered["text"])
+        self.assertIn("Assign", rendered["text"])
         # Every row maps back to source line 1 via lineno propagation.
         self.assertTrue(all(ln == 1 for ln in rendered["lines"] if ln is not None))
 
     def test_view_pseudo_smoke(self) -> None:
         rendered = driver.view_pseudo("def f(x):\n    return x\n\nprint(f(42))\n")
-        self.assertIn("LOAD_CONST", rendered)
+        self.assertIn("LOAD_CONST", rendered["text"])
 
     def test_view_compiled_smoke(self) -> None:
         rendered = driver.view_compiled("x = 1\n")
-        self.assertIn("LOAD_CONST", rendered)
+        if sys.version_info < (3, 15):
+            self.assertIn("LOAD_CONST", rendered["text"])
+        else:
+            self.assertIn("LOAD_SMALL_INT", rendered["text"])
 
     def test_instruction_items_supports_list_and_get_instructions(self) -> None:
         inst = dis.Instruction(

web/driver.py

@@ -16,44 +16,58 @@
 from _testinternalcapi import compiler_codegen, optimize_cfg
 
 
-def _as_view(rows: list[tuple[str, int | None]]) -> dict[str, Any]:
-    text_lines = [row[0] for row in rows]
-    src_lines = [row[1] for row in rows]
-    return {"text": "\n".join(text_lines), "lines": src_lines}
+def view_tokens(code: str) -> dict[str, Any]:
+    if color := sys.version_info >= (3, 15):
+        import _colorize
 
+        theme = _colorize.get_theme(force_color=True)
+        syntax = theme.syntax
+        token_colors = tokenize._get_token_colors(syntax, theme.tokenize)
+        reset = syntax.reset
 
-def view_tokens(code: str) -> dict[str, Any]:
-    rows = []
-    toks = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
+    toks = list(tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline))
+    html_lines = []
+    src_lines = []
     current_line = 0
-    for t in toks:
-        line, end = t.start[0], t.end[0]
+    for tok in toks:
+        line, end = tok.start[0], tok.end[0]
         if end != line:
             marker = f"{line:4d}-{end}: "
         elif line != current_line:
             marker = f"{line:4d}: "
         else:
             marker = "      "
-        rows.append(
-            (
-                f"{marker}{tok_name[t.exact_type]:10} {t.string!r}",
-                line if line > 0 else None,
+        if color:
+            ansi = (
+                f"{_colorize.ANSIColors.RED}{marker}{reset}"
+                f"{token_colors.get(tok.type, reset)}{tok_name[tok.exact_type]:10}{reset} "
+                f"{tok.string!r}"
             )
-        )
+            html_lines.append(_ansi_to_html(ansi))
+        else:
+            plain = f"{marker}{tok_name[tok.exact_type]:10} {tok.string!r}"
+            html_lines.append(html.escape(plain))
+        src_lines.append(line if line > 0 else None)
         current_line = line
-    return _as_view(rows)
+    return {
+        "text": "\n".join(html_lines),
+        "lines": src_lines,
+        "html": True,
+    }
 
 
 _ANSI_RE = re.compile(r"\x1b\[([0-9;]*)m")
 _LINENO_RE = re.compile(r"\blineno=(\d+)")
 _ATTR_ROW_RE = re.compile(r"^\s*(?:lineno|col_offset|end_lineno|end_col_offset)=\d+")
 _ANSI_CLASS = {
-    "36": "ast-node",
-    "34": "ast-field",
-    "90": "ast-attribute",
-    "32": "ast-string",
-    "33": "ast-number",
-    "1;34": "ast-keyword",
+    "31": "ansi-red",
+    "32": "ansi-green",
+    "33": "ansi-yellow",
+    "34": "ansi-blue",
+    "36": "ansi-cyan",
+    "90": "ansi-grey",
+    "1;31": "ansi-bold-red",
+    "1;34": "ansi-bold-blue",
 }
 
 

web/index.html

@@ -132,12 +132,17 @@
             .panel > .content .line.highlight {
                 background: rgba(255, 215, 0, 0.35);
             }
-            .ast-node      { color: #5fc1e0; }
-            .ast-field     { color: #6cb6ff; }
-            .ast-attribute { color: #8a93a0; }
-            .ast-string    { color: #b5e890; }
-            .ast-number    { color: #f0c674; }
-            .ast-keyword   { color: #b9a0ff; font-weight: bold; }
+            /* mirrors cpython's default Theme (ANSIColors); palette tracks
+               the VS Code Dark+ terminal, with bold codes using the bright
+               variants per GNOME Terminal / iTerm2 convention. */
+            .ansi-red       { color: #cd3131; }
+            .ansi-green     { color: #0dbc79; }
+            .ansi-yellow    { color: #e5e510; }
+            .ansi-blue      { color: #2472c8; }
+            .ansi-cyan      { color: #11a8cd; }
+            .ansi-grey      { color: #767676; }
+            .ansi-bold-red  { color: #f14c4c; font-weight: bold; }
+            .ansi-bold-blue { color: #3b8eea; font-weight: bold; }
             .ace-codoscope-highlight {
                 position: absolute;
                 background: rgba(255, 215, 0, 0.35);