From d3bb94f31a2635368c63ff7b4b3743b0ecf0c12f Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:03:39 -0700 Subject: [PATCH 1/6] Add CircuitPython module highlighting overlay CodeMirror 6 dropped the simple extra_keywords mechanism that CM5 had, so instead of forking @codemirror/lang-python we layer a small ViewPlugin on top of the existing Python syntax tree. It walks the visible tree, finds VariableName nodes whose text matches a known CircuitPython core module or common Adafruit library, and tags them with a tok-cp-module class so the theme can color them distinctly. This avoids: - Forking lang-python and tracking upstream changes - Re-tokenizing strings/comments (we only mark identifier nodes) - Adding any per-keystroke parsing cost (decoration set is rebuilt only on doc/viewport/tree changes) Adds a magenta accent (#FF79C6) for .tok-cp-module in the editor theme so CircuitPython modules pop out from regular Python identifiers. Closes #363 --- js/common/circuitpython_highlight.js | 200 +++++++++++++++++++++++++++ js/script.js | 2 + sass/layout/_themes.scss | 7 + 3 files changed, 209 insertions(+) create mode 100644 js/common/circuitpython_highlight.js diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js new file mode 100644 index 0000000..60ca642 --- /dev/null +++ b/js/common/circuitpython_highlight.js @@ -0,0 +1,200 @@ +// CircuitPython syntax highlighting overlay for CodeMirror 6. +// +// CodeMirror 6 dropped the simple `extra_keywords` mechanism that CM5 had, +// so instead of forking @codemirror/lang-python we layer extra decorations +// on top of the existing Python syntax tree. We walk the tree inside the +// viewport, find identifier nodes whose text matches a CircuitPython name, +// and tag them with a CSS class that the theme can style. + +import { ViewPlugin, Decoration } from "@codemirror/view"; +import { syntaxTree } from "@codemirror/language"; + +// CircuitPython core/built-in modules. These are the names that appear in +// `import foo` / `from foo import ...` inside CircuitPython code and are +// the most consistent thing we can match without parsing semantics. +// +// Keep this list focused on modules that ship with CircuitPython (or are +// extremely common Adafruit libraries). Anything we add here will be +// highlighted whenever the identifier appears, so we want low false-positive +// risk against regular Python code. +const CIRCUITPYTHON_MODULES = new Set([ + // Core built-in CircuitPython modules + "adafruit_bus_device", + "aesio", + "alarm", + "analogbufio", + "analogio", + "atexit", + "audiobusio", + "audiocore", + "audioio", + "audiomixer", + "audiomp3", + "audiopwmio", + "bitbangio", + "bitmapfilter", + "bitmaptools", + "bitops", + "board", + "busdisplay", + "busio", + "canio", + "codeop", + "countio", + "digitalio", + "displayio", + "dotclockframebuffer", + "dualbank", + "epaperdisplay", + "espidf", + "espnow", + "espulp", + "floppyio", + "fontio", + "framebufferio", + "frequencyio", + "getpass", + "gifio", + "i2cdisplaybus", + "i2cperipheral", + "i2ctarget", + "imagecapture", + "is31fl3741", + "jpegio", + "keypad", + "keypad_demux", + "lsm6ds", + "max3421e", + "mdns", + "memorymap", + "memorymonitor", + "microcontroller", + "msgpack", + "neopixel_write", + "nvm", + "onewireio", + "paralleldisplay", + "paralleldisplaybus", + "picodvi", + "pulseio", + "pwmio", + "qrio", + "rainbowio", + "rgbmatrix", + "rotaryio", + "rp2pio", + "rtc", + "sdcardio", + "sdioio", + "sharpdisplay", + "socketpool", + "spitarget", + "ssl", + "storage", + "supervisor", + "synthio", + "terminalio", + "tilepalettemapper", + "touchio", + "traceback", + "uheap", + "ulab", + "usb", + "usb_cdc", + "usb_hid", + "usb_host", + "usb_midi", + "usb_video", + "ustack", + "vectorio", + "warnings", + "watchdog", + "wifi", + "zlib", + + // Very common Adafruit/CircuitPython community libraries that users see + // imported all the time. Underscore-prefixed Adafruit names are already + // distinctive enough that false positives are essentially nil. + "adafruit_ble", + "adafruit_connection_manager", + "adafruit_datetime", + "adafruit_display_shapes", + "adafruit_display_text", + "adafruit_displayio_layout", + "adafruit_displayio_sh1106", + "adafruit_displayio_ssd1306", + "adafruit_dotstar", + "adafruit_fakerequests", + "adafruit_framebuf", + "adafruit_hid", + "adafruit_httpserver", + "adafruit_imageload", + "adafruit_io", + "adafruit_logging", + "adafruit_matrixportal", + "adafruit_minimqtt", + "adafruit_motor", + "adafruit_ntp", + "adafruit_pixelbuf", + "adafruit_pixelmap", + "adafruit_portalbase", + "adafruit_register", + "adafruit_requests", + "adafruit_sdcard", + "adafruit_seesaw", + "adafruit_simplemath", + "adafruit_ticks", + "neopixel", + "simpleio", +]); + +const moduleMark = Decoration.mark({ class: "tok-cp-module" }); + +// Build the decoration set for the part of the document currently visible. +// Walking only visible ranges keeps this cheap on big files. +function buildDecorations(view) { + const builder = []; + for (const { from, to } of view.visibleRanges) { + syntaxTree(view.state).iterate({ + from, + to, + enter(node) { + // We care about identifier-like leaves only. Lezer Python emits + // `VariableName` for bare identifiers (including module names + // in `import foo` and `from foo import ...`). Module names + // accessed as attributes (e.g. `adafruit_io.MQTT`) come in as + // `VariableName` for the leftmost part, then `PropertyName` + // children — we only mark the root reference. + if (node.name !== "VariableName") return; + const text = view.state.doc.sliceString(node.from, node.to); + if (CIRCUITPYTHON_MODULES.has(text)) { + builder.push(moduleMark.range(node.from, node.to)); + } + }, + }); + } + // Decoration ranges must be sorted by `from`, which they already are + // because we iterate the tree in document order. + return Decoration.set(builder); +} + +// ViewPlugin keeps decorations in sync with viewport / document changes. +export const circuitpythonHighlight = ViewPlugin.fromClass( + class { + constructor(view) { + this.decorations = buildDecorations(view); + } + update(update) { + if ( + update.docChanged || + update.viewportChanged || + syntaxTree(update.startState) !== syntaxTree(update.state) + ) { + this.decorations = buildDecorations(update.view); + } + } + }, + { + decorations: (v) => v.decorations, + }, +); diff --git a/js/script.js b/js/script.js index db7cc75..7a2d501 100644 --- a/js/script.js +++ b/js/script.js @@ -5,6 +5,7 @@ import { indentWithTab } from "@codemirror/commands" import { python } from "@codemirror/lang-python"; import { syntaxHighlighting, indentUnit } from "@codemirror/language"; import { classHighlighter } from "@lezer/highlight"; +import { circuitpythonHighlight } from "./common/circuitpython_highlight.js"; import { getFileIcon } from "./common/file_dialog.js"; import { Terminal } from '@xterm/xterm'; @@ -405,6 +406,7 @@ const editorExtensions = [ python(), editorTheme, syntaxHighlighting(classHighlighter), + circuitpythonHighlight, EditorView.updateListener.of(onTextChange) ]; diff --git a/sass/layout/_themes.scss b/sass/layout/_themes.scss index 500f1fd..770f991 100644 --- a/sass/layout/_themes.scss +++ b/sass/layout/_themes.scss @@ -174,4 +174,11 @@ .tok-bool { color: #E06C75; } + + // CircuitPython-specific module names (overlay added by + // js/common/circuitpython_highlight.js). Distinct from regular + // variableName so users can spot CircuitPython modules at a glance. + .tok-cp-module { + color: #FF79C6; + } } From e00c925c55ce8d4098a6864709269005c81e4728 Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:20:28 -0700 Subject: [PATCH 2/6] Match adafruit_ libraries by prefix instead of explicit list Drops the hand-curated list of adafruit_* community libraries in favour of a startsWith("adafruit_") check. New Adafruit CircuitPython libraries will be highlighted automatically as they ship, with no maintenance needed in this file. The core CircuitPython module set still uses an explicit allowlist because those names don't share a distinguishing prefix. --- js/common/circuitpython_highlight.js | 66 ++++++++++------------------ 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js index 60ca642..812b172 100644 --- a/js/common/circuitpython_highlight.js +++ b/js/common/circuitpython_highlight.js @@ -9,17 +9,15 @@ import { ViewPlugin, Decoration } from "@codemirror/view"; import { syntaxTree } from "@codemirror/language"; -// CircuitPython core/built-in modules. These are the names that appear in -// `import foo` / `from foo import ...` inside CircuitPython code and are -// the most consistent thing we can match without parsing semantics. +// Core/built-in CircuitPython modules. These are the identifiers that show +// up in `import foo` / `from foo import ...` inside CircuitPython code. // -// Keep this list focused on modules that ship with CircuitPython (or are -// extremely common Adafruit libraries). Anything we add here will be -// highlighted whenever the identifier appears, so we want low false-positive -// risk against regular Python code. -const CIRCUITPYTHON_MODULES = new Set([ - // Core built-in CircuitPython modules - "adafruit_bus_device", +// Anything in this set is highlighted whenever it appears as a bare +// identifier, so it should stay focused on names that ship with +// CircuitPython itself. Third-party Adafruit libraries are matched by +// the `adafruit_` prefix below instead of being listed individually, +// which avoids list maintenance every time a new library lands on PyPI. +const CIRCUITPYTHON_CORE_MODULES = new Set([ "aesio", "alarm", "analogbufio", @@ -112,42 +110,24 @@ const CIRCUITPYTHON_MODULES = new Set([ "wifi", "zlib", - // Very common Adafruit/CircuitPython community libraries that users see - // imported all the time. Underscore-prefixed Adafruit names are already - // distinctive enough that false positives are essentially nil. - "adafruit_ble", - "adafruit_connection_manager", - "adafruit_datetime", - "adafruit_display_shapes", - "adafruit_display_text", - "adafruit_displayio_layout", - "adafruit_displayio_sh1106", - "adafruit_displayio_ssd1306", - "adafruit_dotstar", - "adafruit_fakerequests", - "adafruit_framebuf", - "adafruit_hid", - "adafruit_httpserver", - "adafruit_imageload", - "adafruit_io", - "adafruit_logging", - "adafruit_matrixportal", - "adafruit_minimqtt", - "adafruit_motor", - "adafruit_ntp", - "adafruit_pixelbuf", - "adafruit_pixelmap", - "adafruit_portalbase", - "adafruit_register", - "adafruit_requests", - "adafruit_sdcard", - "adafruit_seesaw", - "adafruit_simplemath", - "adafruit_ticks", + // Bare-named community modules without the `adafruit_` prefix that are + // common enough to be worth recognising explicitly. "neopixel", "simpleio", ]); +// Returns true when `name` is a CircuitPython module worth highlighting. +// Wildcard-matches anything starting with `adafruit_` so new libraries +// (e.g. `adafruit_foo_bar` shipped next month) light up automatically +// without touching this file. +function isCircuitPythonModule(name) { + if (CIRCUITPYTHON_CORE_MODULES.has(name)) return true; + if (name.startsWith("adafruit_") && name.length > "adafruit_".length) { + return true; + } + return false; +} + const moduleMark = Decoration.mark({ class: "tok-cp-module" }); // Build the decoration set for the part of the document currently visible. @@ -167,7 +147,7 @@ function buildDecorations(view) { // children — we only mark the root reference. if (node.name !== "VariableName") return; const text = view.state.doc.sliceString(node.from, node.to); - if (CIRCUITPYTHON_MODULES.has(text)) { + if (isCircuitPythonModule(text)) { builder.push(moduleMark.range(node.from, node.to)); } }, From 53505988dc6c7b386dd78566047d6070926d973d Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:30:24 -0700 Subject: [PATCH 3/6] Also wildcard-match circuitpython_ libraries The CircuitPython Community Bundle ships ~10 libraries with a `circuitpython_` prefix (circuitpython_csv, circuitpython_schedule, circuitpython_functools, etc.). Same prefix-match approach as adafruit_*: new community libraries are highlighted automatically. --- js/common/circuitpython_highlight.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js index 812b172..30da5a0 100644 --- a/js/common/circuitpython_highlight.js +++ b/js/common/circuitpython_highlight.js @@ -117,14 +117,22 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ ]); // Returns true when `name` is a CircuitPython module worth highlighting. -// Wildcard-matches anything starting with `adafruit_` so new libraries -// (e.g. `adafruit_foo_bar` shipped next month) light up automatically -// without touching this file. +// Wildcard-matches anything starting with `adafruit_` (Adafruit-maintained +// libraries) or `circuitpython_` (community bundle libraries) so new +// libraries light up automatically without touching this file. Both +// prefixes are distinctive enough that false positives against ordinary +// Python code are essentially nil. function isCircuitPythonModule(name) { if (CIRCUITPYTHON_CORE_MODULES.has(name)) return true; if (name.startsWith("adafruit_") && name.length > "adafruit_".length) { return true; } + if ( + name.startsWith("circuitpython_") && + name.length > "circuitpython_".length + ) { + return true; + } return false; } From c65d6000447a78b596744c8f485b94f6eda477d8 Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:34:07 -0700 Subject: [PATCH 4/6] Sync core module list with upstream shared-bindings Refreshed CIRCUITPYTHON_CORE_MODULES against the current contents of adafruit/circuitpython:shared-bindings/. Added 15 modules that have landed since the initial list was written: audiodelays, audiofilters, audiofreeverb, audiospeed, aurora_epaper, camera, fourwire, gnss, i2cioexpander, lvfontio, mcp4822, mipidsi, ps2io, qspibus, rclcpy. Removed the deprecated paralleldisplay alias (paralleldisplaybus is the modern name and is already listed) and the stray lsm6ds entry, which isn't a shared binding and isn't in the community bundle either. Standard-Python modules that CircuitPython also exposes (math, os, time, random, struct, hashlib, ipaddress, locale, __future__) are intentionally left out so we don't recolour those names in plain Python code. Underscore-prefixed internal bindings (_bleio etc.) are also omitted; users reach for the adafruit_ wrappers, which the prefix wildcard already matches. --- js/common/circuitpython_highlight.js | 44 +++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js index 30da5a0..aa4b722 100644 --- a/js/common/circuitpython_highlight.js +++ b/js/common/circuitpython_highlight.js @@ -12,11 +12,22 @@ import { syntaxTree } from "@codemirror/language"; // Core/built-in CircuitPython modules. These are the identifiers that show // up in `import foo` / `from foo import ...` inside CircuitPython code. // -// Anything in this set is highlighted whenever it appears as a bare -// identifier, so it should stay focused on names that ship with -// CircuitPython itself. Third-party Adafruit libraries are matched by -// the `adafruit_` prefix below instead of being listed individually, -// which avoids list maintenance every time a new library lands on PyPI. +// Sourced from the upstream `shared-bindings/` directory in +// adafruit/circuitpython, plus port-specific bindings that are widely used +// (espidf/espnow/espulp on ESP, picodvi/rp2pio on RP2). Standard-Python +// modules that CircuitPython also exposes (math, os, time, random, struct, +// hashlib, ipaddress, locale, __future__) are intentionally omitted — they +// aren't CircuitPython-specific and highlighting them as such would be +// noisy in regular Python code shown in the editor. +// +// Underscore-prefixed internal bindings (_bleio, _eve, _pew, _pixelmap, +// _stage) are also omitted; users access those via the corresponding +// `adafruit_*` libraries which are matched by the prefix wildcard below. +// +// Third-party Adafruit libraries are matched by the `adafruit_` prefix +// instead of being listed individually, and community-bundle libraries by +// the `circuitpython_` prefix, so this set only needs updating when a new +// shared binding lands upstream. const CIRCUITPYTHON_CORE_MODULES = new Set([ "aesio", "alarm", @@ -25,10 +36,15 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ "atexit", "audiobusio", "audiocore", + "audiodelays", + "audiofilters", + "audiofreeverb", "audioio", "audiomixer", "audiomp3", "audiopwmio", + "audiospeed", + "aurora_epaper", "bitbangio", "bitmapfilter", "bitmaptools", @@ -36,6 +52,7 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ "board", "busdisplay", "busio", + "camera", "canio", "codeop", "countio", @@ -49,35 +66,41 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ "espulp", "floppyio", "fontio", + "fourwire", "framebufferio", "frequencyio", "getpass", "gifio", + "gnss", "i2cdisplaybus", - "i2cperipheral", + "i2cioexpander", "i2ctarget", "imagecapture", "is31fl3741", "jpegio", "keypad", "keypad_demux", - "lsm6ds", + "lvfontio", "max3421e", + "mcp4822", "mdns", "memorymap", "memorymonitor", "microcontroller", + "mipidsi", "msgpack", "neopixel_write", "nvm", "onewireio", - "paralleldisplay", "paralleldisplaybus", "picodvi", + "ps2io", "pulseio", "pwmio", "qrio", + "qspibus", "rainbowio", + "rclcpy", "rgbmatrix", "rotaryio", "rp2pio", @@ -110,8 +133,9 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ "wifi", "zlib", - // Bare-named community modules without the `adafruit_` prefix that are - // common enough to be worth recognising explicitly. + // Bare-named community modules without the `adafruit_` or + // `circuitpython_` prefix that are common enough to recognise + // explicitly. "neopixel", "simpleio", ]); From b56f5190dc607e8278ee2d24966be3dee39eccd4 Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:47:51 -0700 Subject: [PATCH 5/6] Clarify neopixel/simpleio comment These are early Adafruit-maintained libraries that predate the adafruit_ naming convention, not community-bundle modules. --- js/common/circuitpython_highlight.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js index aa4b722..18184bf 100644 --- a/js/common/circuitpython_highlight.js +++ b/js/common/circuitpython_highlight.js @@ -133,9 +133,9 @@ const CIRCUITPYTHON_CORE_MODULES = new Set([ "wifi", "zlib", - // Bare-named community modules without the `adafruit_` or - // `circuitpython_` prefix that are common enough to recognise - // explicitly. + // Early Adafruit-maintained libraries that predate the `adafruit_` + // naming convention and shipped without a prefix. Listed explicitly + // because the prefix wildcard below can't catch them. "neopixel", "simpleio", ]); From c57e60cd5594fc46f5ff0929cce26e2dbccb7b77 Mon Sep 17 00:00:00 2001 From: Piclaw Date: Wed, 29 Apr 2026 15:56:40 -0700 Subject: [PATCH 6/6] Use Prec.highest so cp-module color wins over variableName When an identifier matches both classHighlighter's VariableName tag and our CircuitPython overlay, CodeMirror renders nested mark spans. The inner span (closer to the text) is what the browser uses for `color`, and it's chosen by decoration precedence \u2014 so without an explicit precedence wrapper, the syntax highlighter's blue stayed on top and `neopixel` rendered the same color as any other variable. Wrapping the ViewPlugin with Prec.highest puts the tok-cp-module span inside the tok-variableName span, letting the pink override the blue without touching the CSS or using !important. --- js/common/circuitpython_highlight.js | 11 ++++++++++- sass/layout/_themes.scss | 5 +++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/js/common/circuitpython_highlight.js b/js/common/circuitpython_highlight.js index 18184bf..8b51313 100644 --- a/js/common/circuitpython_highlight.js +++ b/js/common/circuitpython_highlight.js @@ -8,6 +8,7 @@ import { ViewPlugin, Decoration } from "@codemirror/view"; import { syntaxTree } from "@codemirror/language"; +import { Prec } from "@codemirror/state"; // Core/built-in CircuitPython modules. These are the identifiers that show // up in `import foo` / `from foo import ...` inside CircuitPython code. @@ -191,7 +192,7 @@ function buildDecorations(view) { } // ViewPlugin keeps decorations in sync with viewport / document changes. -export const circuitpythonHighlight = ViewPlugin.fromClass( +const circuitpythonHighlightPlugin = ViewPlugin.fromClass( class { constructor(view) { this.decorations = buildDecorations(view); @@ -210,3 +211,11 @@ export const circuitpythonHighlight = ViewPlugin.fromClass( decorations: (v) => v.decorations, }, ); + +// Wrap the plugin with Prec.highest so its decoration nests inside the +// classHighlighter span. CodeMirror renders overlapping mark decorations +// as nested spans where higher-precedence decorations end up closer to +// the text. The inner span’s `color` is what the user sees, so making +// `tok-cp-module` the inner class is what lets our pink override the +// underlying `tok-variableName` blue without resorting to !important. +export const circuitpythonHighlight = Prec.highest(circuitpythonHighlightPlugin); diff --git a/sass/layout/_themes.scss b/sass/layout/_themes.scss index 770f991..b4479bf 100644 --- a/sass/layout/_themes.scss +++ b/sass/layout/_themes.scss @@ -176,8 +176,9 @@ } // CircuitPython-specific module names (overlay added by - // js/common/circuitpython_highlight.js). Distinct from regular - // variableName so users can spot CircuitPython modules at a glance. + // js/common/circuitpython_highlight.js). The overlay extension is + // wrapped with Prec.highest so its decoration nests inside the + // classHighlighter span, letting this color win without !important. .tok-cp-module { color: #FF79C6; }