From 434e1279a84f3a8305b20fb309fc5ba23675588b Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 26 Apr 2026 20:47:27 +0100 Subject: [PATCH 1/4] Add color to tokens --- web/driver.py | 56 +++++++++++++++++++++++++++++++------------------- web/index.html | 17 +++++++++------ 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/web/driver.py b/web/driver.py index 47f2619..507d5f5 100644 --- a/web/driver.py +++ b/web/driver.py @@ -16,44 +16,58 @@ from _testinternalcapi import compiler_codegen, optimize_cfg -def _as_view(rows: list[tuple[str, int | None]]) -> dict[str, Any]: - text_lines = [row[0] for row in rows] - src_lines = [row[1] for row in rows] - return {"text": "\n".join(text_lines), "lines": src_lines} +def view_tokens(code: str) -> dict[str, Any]: + if color := sys.version_info >= (3, 15): + import _colorize + theme = _colorize.get_theme(force_color=True) + syntax = theme.syntax + token_colors = tokenize._get_token_colors(syntax, theme.tokenize) + reset = syntax.reset -def view_tokens(code: str) -> dict[str, Any]: - rows = [] - toks = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline) + toks = list(tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)) + html_lines = [] + src_lines = [] current_line = 0 - for t in toks: - line, end = t.start[0], t.end[0] + for tok in toks: + line, end = tok.start[0], tok.end[0] if end != line: marker = f"{line:4d}-{end}: " elif line != current_line: marker = f"{line:4d}: " else: marker = " " - rows.append( - ( - f"{marker}{tok_name[t.exact_type]:10} {t.string!r}", - line if line > 0 else None, + if color: + ansi = ( + f"{theme.tokenize.position}{marker}{reset}" + f"{token_colors.get(tok.type, reset)}{tok_name[tok.exact_type]:10}{reset} " + f"{tok.string!r}" ) - ) + html_lines.append(_ansi_to_html(ansi)) + else: + plain = f"{marker}{tok_name[tok.exact_type]:10} {tok.string!r}" + html_lines.append(html.escape(plain)) + src_lines.append(line if line > 0 else None) current_line = line - return _as_view(rows) + return { + "text": "\n".join(html_lines), + "lines": src_lines, + "html": True, + } _ANSI_RE = re.compile(r"\x1b\[([0-9;]*)m") _LINENO_RE = re.compile(r"\blineno=(\d+)") _ATTR_ROW_RE = re.compile(r"^\s*(?:lineno|col_offset|end_lineno|end_col_offset)=\d+") _ANSI_CLASS = { - "36": "ast-node", - "34": "ast-field", - "90": "ast-attribute", - "32": "ast-string", - "33": "ast-number", - "1;34": "ast-keyword", + "31": "ansi-red", + "32": "ansi-green", + "33": "ansi-yellow", + "34": "ansi-blue", + "36": "ansi-cyan", + "90": "ansi-grey", + "1;31": "ansi-bold-red", + "1;34": "ansi-bold-blue", } diff --git a/web/index.html b/web/index.html index 67e2380..249ca5d 100644 --- a/web/index.html +++ b/web/index.html @@ -132,12 +132,17 @@ .panel > .content .line.highlight { background: rgba(255, 215, 0, 0.35); } - .ast-node { color: #5fc1e0; } - .ast-field { color: #6cb6ff; } - .ast-attribute { color: #8a93a0; } - .ast-string { color: #b5e890; } - .ast-number { color: #f0c674; } - .ast-keyword { color: #b9a0ff; font-weight: bold; } + /* mirrors cpython's default Theme (ANSIColors); palette tracks + the VS Code Dark+ terminal, with bold codes using the bright + variants per GNOME Terminal / iTerm2 convention. */ + .ansi-red { color: #cd3131; } + .ansi-green { color: #0dbc79; } + .ansi-yellow { color: #e5e510; } + .ansi-blue { color: #2472c8; } + .ansi-cyan { color: #11a8cd; } + .ansi-grey { color: #767676; } + .ansi-bold-red { color: #f14c4c; font-weight: bold; } + .ansi-bold-blue { color: #3b8eea; font-weight: bold; } .ace-codoscope-highlight { position: absolute; background: rgba(255, 215, 0, 0.35); From c3aac36e20a340e9284de6112e39dd8d7e9b7d31 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 26 Apr 2026 21:03:25 +0100 Subject: [PATCH 2/4] Make numbers in the gutter red like Godbolt --- web/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/driver.py b/web/driver.py index 507d5f5..4917e96 100644 --- a/web/driver.py +++ b/web/driver.py @@ -39,7 +39,7 @@ def view_tokens(code: str) -> dict[str, Any]: marker = " " if color: ansi = ( - f"{theme.tokenize.position}{marker}{reset}" + f"{_colorize.ANSIColors.RED}{marker}{reset}" f"{token_colors.get(tok.type, reset)}{tok_name[tok.exact_type]:10}{reset} " f"{tok.string!r}" ) From 5aaf620edcc65c114c145e89913da3565c1be4a6 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 3 May 2026 11:03:35 +0100 Subject: [PATCH 3/4] Fix em tests --- tests/test_web_driver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_web_driver.py b/tests/test_web_driver.py index 691e718..13e7bda 100644 --- a/tests/test_web_driver.py +++ b/tests/test_web_driver.py @@ -16,8 +16,8 @@ class WebDriverTests(unittest.TestCase): def test_view_tokens_includes_token_names(self) -> None: rendered = driver.view_tokens("x = 1\n") - self.assertIn("NAME", rendered) - self.assertIn("NUMBER", rendered) + self.assertIn("NAME", rendered['text']) + self.assertIn("NUMBER", rendered['text']) def test_view_ast_returns_module_dump(self) -> None: rendered = driver.view_ast("x = 1\n", optimize=False) @@ -29,11 +29,14 @@ def test_view_ast_returns_module_dump(self) -> None: def test_view_pseudo_smoke(self) -> None: rendered = driver.view_pseudo("def f(x):\n return x\n\nprint(f(42))\n") - self.assertIn("LOAD_CONST", rendered) + self.assertIn("LOAD_CONST", rendered['text']) def test_view_compiled_smoke(self) -> None: rendered = driver.view_compiled("x = 1\n") - self.assertIn("LOAD_CONST", rendered) + if sys.version_info < (3, 15): + self.assertIn("LOAD_CONST", rendered['text']) + else: + self.assertIn("LOAD_SMALL_INT", rendered['text']) def test_instruction_items_supports_list_and_get_instructions(self) -> None: inst = dis.Instruction( From 97dfee024d77bf3f402a74342f9ba647cec6933c Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 3 May 2026 11:19:20 +0100 Subject: [PATCH 4/4] Fix em tests (more) --- tests/test_web_driver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_web_driver.py b/tests/test_web_driver.py index 13e7bda..aaa8066 100644 --- a/tests/test_web_driver.py +++ b/tests/test_web_driver.py @@ -16,27 +16,27 @@ class WebDriverTests(unittest.TestCase): def test_view_tokens_includes_token_names(self) -> None: rendered = driver.view_tokens("x = 1\n") - self.assertIn("NAME", rendered['text']) - self.assertIn("NUMBER", rendered['text']) + self.assertIn("NAME", rendered["text"]) + self.assertIn("NUMBER", rendered["text"]) def test_view_ast_returns_module_dump(self) -> None: rendered = driver.view_ast("x = 1\n", optimize=False) self.assertTrue(rendered["html"]) - self.assertIn(">Module(", rendered["text"]) - self.assertIn(">Assign(", rendered["text"]) + self.assertIn("Module", rendered["text"]) + self.assertIn("Assign", rendered["text"]) # Every row maps back to source line 1 via lineno propagation. self.assertTrue(all(ln == 1 for ln in rendered["lines"] if ln is not None)) def test_view_pseudo_smoke(self) -> None: rendered = driver.view_pseudo("def f(x):\n return x\n\nprint(f(42))\n") - self.assertIn("LOAD_CONST", rendered['text']) + self.assertIn("LOAD_CONST", rendered["text"]) def test_view_compiled_smoke(self) -> None: rendered = driver.view_compiled("x = 1\n") if sys.version_info < (3, 15): - self.assertIn("LOAD_CONST", rendered['text']) + self.assertIn("LOAD_CONST", rendered["text"]) else: - self.assertIn("LOAD_SMALL_INT", rendered['text']) + self.assertIn("LOAD_SMALL_INT", rendered["text"]) def test_instruction_items_supports_list_and_get_instructions(self) -> None: inst = dis.Instruction(