diff --git a/.changeset/violet-pigs-sink.md b/.changeset/violet-pigs-sink.md new file mode 100644 index 00000000..59d3a25f --- /dev/null +++ b/.changeset/violet-pigs-sink.md @@ -0,0 +1,5 @@ +--- +'@ryanatkn/fuz_code': minor +--- + +add range highlighting diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..339ae527 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,256 @@ +# fuz_code - Syntax Highlighter + +A performance-focused fork of PrismJS for syntax highlighting, +optimized for runtime use with optional CSS Custom Highlight API support. + +## Quick Start + +```typescript +import Code from '$lib/Code.svelte'; +import {syntax_styler_global} from '$lib/syntax_styler_global.js'; + +// Auto-detects and uses CSS Highlight API when available + + +// Force specific rendering mode + // Traditional HTML + // CSS Highlights + +// Direct usage +const html = syntax_styler_global.stylize(code, 'ts'); +``` + +## Commands + +```bash +gro test # Run all tests +gro test src/fixtures/check.test.ts # Verify fixture generation +npm run update-generated-fixtures # Regenerate fixtures +npm run benchmark # Run performance benchmarks +npm run benchmark-compare # Compare performance with Prism and Shiki +``` + +## Architecture + +### Core System + +**Syntax Styler** - A PrismJS fork with two rendering modes: + +1. **HTML Mode** - Traditional token-based HTML generation with CSS classes +2. **Range Mode** - CSS Custom Highlight API for native browser highlighting + +The system uses regex-based tokenization inherited from PrismJS, maintaining compatibility with existing language definitions while adding position tracking for range highlighting. + +### Key Components + +#### Tokenization Engine + +- `src/lib/syntax_styler.ts` - Core tokenization engine with linked list processing +- `src/lib/syntax_styler_global.ts` - Pre-configured global instance +- `tokenize_syntax()` from `src/lib/syntax_styler.ts` - Main tokenization function + +#### Language Definitions + +- `src/lib/grammar_ts.ts` - TypeScript/JS +- `src/lib/grammar_css.ts` - CSS stylesheets +- `src/lib/grammar_markup.ts` - HTML/XML markup +- `src/lib/grammar_json.ts` - JSON data +- `src/lib/grammar_svelte.ts` - Svelte components +- `src/lib/grammar_clike.ts` - Base for C-like languages + +#### Range Highlighting + +- `src/lib/highlight_manager.ts` - Direct tree traversal for range creation +- `src/lib/Highlight_Manager` - Manages CSS Custom Highlights per element + +#### Components + +- `src/lib/Code.svelte` - Hybrid component supporting both HTML and range modes with auto-detection + +#### Themes + +- `src/lib/theme.css` - Traditional CSS classes for HTML mode (requires Moss or theme_variables.css) +- `src/lib/theme_variables.css` - CSS variable definitions for non-Moss users + +## How It Works + +### Token Tree Structure + +Syntax styler creates a hierarchical token tree where tokens can contain nested tokens: + +```typescript +interface Syntax_Token { + type: string; // Token type (e.g., 'keyword', 'string') + content: string | Syntax_Token_Stream; // Text or nested tokens + alias: string | Array; // CSS class aliases + length: number; // Token text length +} +``` + +### Range Creation + +For CSS Custom Highlights, ranges are created directly from the token tree during a single traversal: + +```typescript +// Generate tokens from syntax styler +const tokens = tokenize_syntax(code, grammar); +// Highlight manager creates ranges directly from the token tree +highlight_manager.highlight_from_syntax_tokens(element, tokens); +``` + +### CSS Custom Highlights + +When supported, the browser's native highlighting is used: + +```JS +// Create ranges for each token +const range = new Range(); +range.setStart(textNode, token.start); +range.setEnd(textNode, token.end); + +// Add to CSS Highlight +const highlight = CSS.highlights.get(token.type) || new Highlight(); +highlight.add(range); +CSS.highlights.set(token.type, highlight); +``` + +## Supported Languages + +- `ts` - TypeScript +- `js` - JS +- `css` - CSS +- `html` - HTML/XML +- `json` - JSON +- `svelte` - Svelte components + +## API Reference + +### Syntax_Styler + +```typescript +class Syntax_Styler { + // Generate HTML with syntax highlighting + stylize(text: string, lang: string): string; + + // Get language grammar + get_lang(id: string): Grammar; + + // Add new language + add_lang(id: string, grammar: Grammar, aliases?: Array): void; +} +``` + +### Pre-configured instance + +```typescript +import {syntax_styler_global} from '$lib/syntax_styler_global.js'; + +const html = syntax_styler_global.stylize(code, 'ts'); +``` + +### Highlight_Manager + +Manages CSS Custom Highlights for an element: + +```typescript +const manager = new Highlight_Manager(); + +// Apply highlights from tokens +manager.highlight_from_syntax_tokens(element, tokens); + +// Clear highlights +manager.clear_element_ranges(); + +// Clean up +manager.destroy(); +``` + +## Testing + +### Sample Files + +Test samples in `src/lib/samples/sample_*.{lang}` are the source of truth. + +### Fixtures + +Generated fixtures in `src/fixtures/{lang}/`: + +- `{lang}_{variant}.json` - Token data and HTML output +- `{lang}_{variant}.txt` - Human-readable debug output + +### Workflow + +1. Edit samples in `src/lib/samples/` +2. Run `npm run update-generated-fixtures` to regenerate +3. Run `gro test src/fixtures/check.test.ts` to verify +4. Review changes with `git diff src/fixtures/` + +## Performance + +### Benchmarking + +```bash +npm run benchmark # Internal performance benchmark +npm run benchmark-compare # Compare with Prism and Shiki +``` + +**Internal benchmark** tests fuz_code performance across all sample files with small and large (100x) content. + +**Comparison benchmark** (`./benchmark/compare/`) tests fuz_code against: + +- Prism - Similar regex-based approach +- Shiki JS - JS regex engine +- Shiki Oniguruma - Full TextMate grammar engine + +Results show relative performance (% of fastest) for each language and content size. + +### Optimization Notes + +- **HTML Mode**: Proven PrismJS approach, good for SSR +- **Range Mode**: Native browser highlighting, better for large documents +- **Auto Mode**: Best of both worlds, progressive enhancement + +## Color Variables + +Theme uses CSS variables from Moss: + +- `--color_a` - Keywords, tags +- `--color_b` - Strings, selectors +- `--color_c` - Types (TypeScript) +- `--color_d` - Functions, classes +- `--color_e` - Numbers, regex +- `--color_f` - Operators, keywords +- `--color_g` - Attributes +- `--color_h` - Properties +- `--color_i` - Booleans, comments + +## Development Guidelines + +1. **Maintain PrismJS compatibility** - Language definitions should work with upstream +2. **Test with fixtures** - All changes must pass fixture tests +3. **No automated commits** - Manual review required +4. **Support both modes** - Features should work in HTML and range modes +5. **Follow patterns** - Use existing language definitions as templates + +## Demo Pages + +- `/samples` - Code samples in all supported languages +- `/benchmark` - Performance testing + +## Troubleshooting + +### Positions Don't Match + +The position calculation happens during range creation. If positions are wrong: + +1. Check `highlight_manager.ts` range creation logic +2. Verify token tree structure with debug output +3. Look for nested tokens that might be miscounted + +### Adding a New Language + +1. Create `src/lib/grammar_{lang}.ts` +2. Define grammar patterns (see existing languages) +3. Register in `syntax_styler_global.ts` +4. Add samples in `src/lib/samples/sample_{variant}.{lang}` +5. Generate fixtures and test diff --git a/README.md b/README.md index 03bfdc9f..acd80751 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The main changes: - has a minimal and explicit API to generate stylized HTML, and knows nothing about the DOM - uses stateless ES modules, instead of globals with side effects and pseudo-module behaviors - has various incompatible changes, so using Prism grammars requires some tweaks -- smaller (by 7kB minified and 3kB gzipped, ~1/3 less) and [faster](#benchmarks) +- smaller (by 7kB minified and 3kB gzipped, ~1/3 less) - written in TypeScript - is a fork, see the [MIT license](https://github.com/ryanatkn/fuz_code/blob/main/LICENSE) @@ -25,20 +25,21 @@ but there are two optional dependencies: based on [`prism-svelte`](https://github.com/pngwn/prism-svelte) and a [Svelte component](src/lib/Code.svelte) for convenient usage. - The [default theme](src/lib/theme.css) integrates - with my CSS library [Moss](https://github.com/ryanatkn/moss) for colors that adapt to the user's runtime `color-scheme` preference. - A [zero-dependency theme](src/lib/theme_standalone.css) - is also provided that uses the less-customizable `light-dark()`, see below for more. + with my CSS library [Moss](https://github.com/ryanatkn/moss) for colors that adapt to the user's runtime `color-scheme` preference, + and [theme_variables.css](src/lib/theme_variables.css) + is also included that uses the less-customizable `light-dark()`. Compared to [Shiki](https://github.com/shikijs/shiki), this library is much lighter (with its faster `shiki/engine/javascript`, 503kB minified to 16kB, 63kb gzipped to 5.6kB), -and [vastly faster](#benchmarks) +and [vastly faster](./benchmark/compare/results.md) for runtime usage because it uses JS regexps instead of the [Onigurama regexp engine](https://shiki.matsu.io/guide/regex-engines) used by TextMate grammars. Shiki also has 38 dependencies instead of 0. -However this is not a fair comparison because Shiki is designed mainly for buildtime usage, -and Prism grammars are much simpler and less powerful than TextMate's. +However this is not a fair comparison because +Prism grammars are much simpler and less powerful than TextMate's, +and Shiki is designed mainly for buildtime usage. ## Usage @@ -47,11 +48,31 @@ npm i -D @ryanatkn/fuz_code ``` ```ts -import {syntax_styler} from '@ryanatkn/fuz_code'; +import {syntax_styler_global} from '@ryanatkn/fuz_code/syntax_styler_global.js'; -syntax_styler.stylize('

hello world

', 'svelte'); +syntax_styler_global.stylize('

hello world

', 'svelte'); ``` +```svelte + + + + + + + + + + +``` + +By default the `Code` component automatically uses the +[CSS Custom Highlight API](https://developer.mozilla.org/en-US/docs/Web/API/CSS_Custom_Highlight_API) +when available for improved performance, +falling back to HTML generation for non-browser runtimes and older browsers. + Themes are just CSS files, so they work with any JS framework. With SvelteKit: @@ -66,35 +87,34 @@ on my CSS library [Moss](https://github.com/ryanatkn/moss) for [color-scheme](https://moss.ryanatkn.com/docs/themes) awareness. See the [Moss docs](https://moss.ryanatkn.com/) for its usage. -[A dependency-free version](src/lib/theme_standalone.css) of the default theme is provided, -but note that the colors are staticly defined instead of using -Moss' [style variables](https://moss.ryanatkn.com/docs/variables). -They use [`light-dark()`](https://developer.mozilla.org/en-US/docs/Web/CSS/color_value/light-dark) -which means they're hardcoded to the inferred `color-scheme`, -rather than being settable by a user option unlike the Moss version. This could be improved using a class convention like `.dark`. +If you're not using Moss, import `theme_variables.css` alongside `theme.css`: ```ts -// no dependencies: -import '@ryanatkn/fuz_code/theme_standalone.css'; +// Without Moss: +import '@ryanatkn/fuz_code/theme.css'; +import '@ryanatkn/fuz_code/theme_variables.css'; ``` ### Modules -- [@ryanatkn/fuz_code](src/lib/index.ts) - index with default grammars, - use this as a guide if you want custom grammars -- [@ryanatkn/fuz_code/syntax_styler.js](src/lib/syntax_styler.ts) - utilities for custom grammars +- [@ryanatkn/fuz_code/syntax_styler_global.js](src/lib/syntax_styler_global.ts) - pre-configured instance with all grammars +- [@ryanatkn/fuz_code/syntax_styler.js](src/lib/syntax_styler.ts) - base class for custom grammars - [@ryanatkn/fuz_code/theme.css](src/lib/theme.css) - default theme that depends on [Moss](https://github.com/ryanatkn/moss) -- [@ryanatkn/fuz_code/theme_standalone.css](src/lib/theme_standalone.css) - - default theme with no dependencies +- [@ryanatkn/fuz_code/theme_variables.css](src/lib/theme_variables.css) - + CSS variables for non-Moss users - [@ryanatkn/fuz_code/Code.svelte](src/lib/Code.svelte) - - Svelte component with a convenient API + Svelte component supporting both HTML generation and native browser highlights +- [@ryanatkn/fuz_code/highlight_manager.js](src/lib/highlight_manager.ts) - + uses the browser [`Highlight`](https://developer.mozilla.org/en-US/docs/Web/API/Highlight) + and [`Range`](https://developer.mozilla.org/en-US/docs/Web/API/Range) APIs + as a faster alternative to generating spans with classes I encourage you to poke around [`src/lib`](src/lib) if you're interested in using fuz_code. ### Grammars -Enabled [by default](src/lib/index.ts): +Enabled by default in `syntax_styler_global`: - [`markup`](src/lib/grammar_markup.ts) (html, xml, etc) - [`svelte`](src/lib/grammar_svelte.ts) @@ -109,10 +129,10 @@ Enabled [by default](src/lib/index.ts): Docs are a work in progress: - this readme has basic usage instructions +- [CLAUDE.md](./CLAUDE.md) has more high-level docs including benchmarks - [code.fuz.dev](https://code.fuz.dev/) has usage examples with the Svelte component - [samples](https://code.fuz.dev/samples) on the website - (also see the [inputs](src/lib/code_sample_inputs.ts) - and [outputs](src/lib/code_sample_outputs.ts)) + (also see the [sample files](src/lib/samples/)) - [tests](src/lib/syntax_styler.test.ts) Please open issues if you need any help. @@ -129,86 +149,6 @@ Please open issues if you need any help. - improve the TypeScript grammar to tokenize types - improve the grammars in subtle ways -## Benchmarks - -Performance is a high priority to best support runtime usage. -This project is still early and there are more gains to be had. - -Note that this benchmark is somewhat unfair to Shiki -because it's not designed for runtime usage, -and it probably does a significantly better job at the task at hand -because it uses TextMate grammars. - -Results styling the [Svelte sample](src/lib/code_sample_inputs.ts): - -| Task name | Throughput average (ops/s) | Throughput median (ops/s) | Samples | -| ----------------------- | -------------------------- | ------------------------- | ------- | -| syntax_styler.stylize | 3149 ± 0.56% | 3333 | 6004 | -| Prism.highlight | 2748 ± 0.51% | 2500 | 5293 | -| Shiki engine/javascript | 69 ± 0.59% | 69 | 138 | -| Shiki engine/oniguruma | 41 ± 0.27% | 40 | 82 | - -Directly runnable benchmarks are not included yet - -I don't know if I'll add them here or make a separate project. - -To run the benchmarks yourself: - -```bash -npm i -D shiki prismjs prism-svelte @types/prismjs @ryanatkn/fuz_code -``` - -Then add this file and import it somewhere like `$routes/+page.svelte`: - -```ts -// $lib/benchmark.ts - -import {Bench} from 'tinybench'; -import Prism from 'prismjs'; -import 'prism-svelte'; -import {createHighlighterCoreSync} from 'shiki/core'; -import {createJavaScriptRegexEngine} from 'shiki/engine/javascript'; -import svelte_shiki from 'shiki/langs/svelte.mjs'; -import nord from 'shiki/themes/nord.mjs'; -import {createOnigurumaEngine} from 'shiki/engine/oniguruma'; - -import {syntax_styler} from '$lib/index.js'; -import {sample_svelte_code} from '$lib/code_sample_inputs.js'; - -console.log('benchmarking'); -const bench = new Bench({name: 'syntax styling', time: 2000}); - -const shiki_highlighter_js = createHighlighterCoreSync({ - themes: [nord], - langs: [svelte_shiki], - engine: createJavaScriptRegexEngine(), -}); - -const shiki_highlighter_builtin = createHighlighterCoreSync({ - themes: [nord], - langs: [svelte_shiki], - engine: await createOnigurumaEngine(import('shiki/wasm')), -}); - -bench - .add('syntax_styler.stylize', () => { - syntax_styler.stylize(sample_svelte_code, 'svelte'); - }) - .add('Prism.highlight', () => { - Prism.highlight(sample_svelte_code, Prism.langs.svelte, 'svelte'); - }) - .add('Shiki engine/javascript', () => { - shiki_highlighter_js.codeToHtml(sample_svelte_code, {lang: 'svelte', theme: 'nord'}); - }) - .add('Shiki engine/oniguruma', () => { - shiki_highlighter_builtin.codeToHtml(sample_svelte_code, {lang: 'svelte', theme: 'nord'}); - }); - -await bench.run(); - -console.log(bench.name); -console.table(bench.table()); -``` - ## License [🐦](https://wikipedia.org/wiki/Free_and_open-source_software) based on [Prism](https://github.com/PrismJS/prism) diff --git a/benchmark/benchmarks.ts b/benchmark/benchmarks.ts new file mode 100644 index 00000000..da73efdd --- /dev/null +++ b/benchmark/benchmarks.ts @@ -0,0 +1,109 @@ +import {Bench} from 'tinybench'; + +import {samples as all_samples} from '../src/lib/samples/all.js'; +import {syntax_styler_global} from '../src/lib/syntax_styler_global.js'; + +/* eslint-disable no-console */ + +const BENCHMARK_TIME = 10000; +const WARMUP_TIME = 1000; +const WARMUP_ITERATIONS = 50; +const LARGE_CONTENT_MULTIPLIER = 100; + +export interface Benchmark_Result { + name: string; + ops_per_sec: number; + mean_time: number; + samples: number; +} + +export const run_benchmark = async (filter?: string): Promise> => { + const bench = new Bench({ + time: BENCHMARK_TIME, + warmupTime: WARMUP_TIME, + warmupIterations: WARMUP_ITERATIONS, + }); + + const samples = Object.values(all_samples); + const samples_to_run = filter + ? samples.filter((s) => s.name.includes(filter) || s.lang === filter) + : samples; + + // Add baseline benchmarks (existing behavior) + for (const sample of samples_to_run) { + bench.add(`baseline:${sample.name}`, () => { + syntax_styler_global.stylize(sample.content, sample.lang); + }); + } + + // Add large content benchmarks (100x repetition) + const complex_samples = Object.values(all_samples).filter((s) => s.name.includes('complex')); + for (const sample of complex_samples) { + // Skip if filter is specified and doesn't match this sample + if ( + filter && + !sample.name.includes(filter) && + !sample.lang.includes(filter) && + filter !== sample.lang + ) { + continue; + } + + const large_content = sample.content.repeat(LARGE_CONTENT_MULTIPLIER); + + bench.add(`large:${sample.name}`, () => { + syntax_styler_global.stylize(large_content, sample.lang); + }); + } + + await bench.run(); + + const results: Array = []; + + for (const task of bench.tasks) { + if (task.result) { + results.push({ + name: task.name, + ops_per_sec: task.result.throughput.mean, + mean_time: task.result.latency.mean, + samples: task.result.latency.samples.length, + }); + } + } + + return results; +}; + +export const format_benchmark_results = (results: Array): string => { + const lines: Array = [ + '## Benchmark Results', + '', + '| Sample | Ops/sec | Mean Time (ms) | Samples |', + '|--------|---------|----------------|---------|', + ]; + + for (const result of results) { + const name = result.name.replace('baseline:', ''); + const ops_per_sec = result.ops_per_sec.toFixed(2); + const mean_time = result.mean_time.toFixed(4); + lines.push(`| ${name} | ${ops_per_sec} | ${mean_time} | ${result.samples} |`); + } + + lines.push(''); + lines.push(`**Total samples benchmarked:** ${results.length}`); + + const avg_ops = results.reduce((sum, r) => sum + r.ops_per_sec, 0) / results.length; + lines.push(`**Average ops/sec:** ${avg_ops.toFixed(2)}`); + + return lines.join('\n'); +}; + +export const run_and_print_benchmark = async (filter?: string): Promise => { + console.log('Starting benchmark...\n'); + + const results = await run_benchmark(filter); + + console.log(format_benchmark_results(results)); + + console.log('\n✅ All samples validated successfully'); +}; diff --git a/benchmark/compare/compare.ts b/benchmark/compare/compare.ts new file mode 100644 index 00000000..76f7dfe2 --- /dev/null +++ b/benchmark/compare/compare.ts @@ -0,0 +1,320 @@ +// TODO this is a workaround for eslint failng without `"benchmark/**/*.ts"` in tsconfig.json +// This allows CI to pass without running `npm install` for the benchmarks. +// @ts-nocheck + +import {Bench} from 'tinybench'; + +// Prism imports +import Prism from 'prismjs'; +import 'prismjs/components/prism-typescript.js'; +import 'prismjs/components/prism-css.js'; +import 'prismjs/components/prism-markup.js'; +import 'prismjs/components/prism-json.js'; +import 'prism-svelte'; + +// Shiki imports +import {createHighlighterCoreSync} from 'shiki/core'; +import {createJavaScriptRegexEngine} from 'shiki/engine/javascript'; +import {createOnigurumaEngine} from 'shiki/engine/oniguruma'; +import typescript from 'shiki/langs/typescript.mjs'; +import javascript from 'shiki/langs/javascript.mjs'; +import css from 'shiki/langs/css.mjs'; +import html from 'shiki/langs/html.mjs'; +import json from 'shiki/langs/json.mjs'; +import svelte from 'shiki/langs/svelte.mjs'; +import nord from 'shiki/themes/nord.mjs'; + +// Fuz Code imports +import {samples as all_samples} from '../../src/lib/samples/all.js'; +import {syntax_styler_global} from '../../src/lib/syntax_styler_global.js'; +import {tokenize_syntax} from '../../src/lib/syntax_styler.js'; + +/* eslint-disable no-console */ + +const BENCHMARK_TIME = 10000; // 10000 +const WARMUP_TIME = 1000; // 1000 +const WARMUP_ITERATIONS = 20; // 20 +const LARGE_CONTENT_MULTIPLIER = 100; // 100 +const MIN_ITERATIONS = 3; // Tiny minimum samples cause of Shiki's pathological cases with TS + +export interface Comparison_Result { + implementation: string; + language: string; + ops_per_sec: number; + mean_time: number; + samples: number; + content_size: 'small' | 'large'; + total_time: number; + operation: 'tokenize' | 'stylize'; +} + +const LANGUAGE_MAP = { + ts: { + prism: 'typescript', + shiki: 'typescript', + fuz: 'ts', + }, + js: { + prism: 'javascript', + shiki: 'javascript', + fuz: 'js', + }, + css: { + prism: 'css', + shiki: 'css', + fuz: 'css', + }, + html: { + prism: 'markup', + shiki: 'html', + fuz: 'html', + }, + json: { + prism: 'json', + shiki: 'json', + fuz: 'json', + }, + svelte: { + prism: 'svelte', + shiki: 'svelte', + fuz: 'svelte', + }, +} as const; + +type SupportedLanguage = keyof typeof LANGUAGE_MAP; + +const setupShiki = async () => { + const langs = [typescript, javascript, css, html, json, svelte]; + + const shiki_js = createHighlighterCoreSync({ + themes: [nord], + langs, + engine: createJavaScriptRegexEngine(), + }); + + const shiki_oniguruma = createHighlighterCoreSync({ + themes: [nord], + langs, + engine: await createOnigurumaEngine(import('shiki/wasm')), + }); + + return {shiki_js, shiki_oniguruma}; +}; + +const getSampleContent = (lang: SupportedLanguage, large = false) => { + const sample = Object.values(all_samples).find((s) => s.lang === LANGUAGE_MAP[lang].fuz); + if (!sample) { + throw new Error(`No sample found for language: ${lang}`); + } + return large ? sample.content.repeat(LARGE_CONTENT_MULTIPLIER) : sample.content; +}; + +export const run_comparison_benchmark = async ( + filter?: string, +): Promise> => { + const bench = new Bench({ + time: BENCHMARK_TIME, + warmupTime: WARMUP_TIME, + warmupIterations: WARMUP_ITERATIONS, + iterations: MIN_ITERATIONS, + }); + + // Setup Shiki + console.log('Setting up Shiki highlighters...'); + const {shiki_js, shiki_oniguruma} = await setupShiki(); + console.log('Shiki setup complete'); + + // Determine languages to test + const supported_languages: Array = ['ts', 'css', 'html', 'json', 'svelte']; + const languages_to_test = filter + ? supported_languages.filter((lang) => lang === filter || lang.includes(filter)) + : supported_languages; + + console.log(`Testing languages: ${languages_to_test.join(', ')}`); + + // Add benchmarks for each language and content size + for (const lang of languages_to_test) { + const prism_lang = LANGUAGE_MAP[lang].prism; + const shiki_lang = LANGUAGE_MAP[lang].shiki; + const fuz_lang = LANGUAGE_MAP[lang].fuz; + + // Test both small and large content + for (const large of [false, true]) { + const content = getSampleContent(lang, large); + const size_label = large ? 'large' : 'small'; + + // Tokenization benchmarks (fuz_code and Prism only) + // Fuz Code tokenize benchmark + bench.add(`fuz_code_tokenize_${lang}_${size_label}`, () => { + tokenize_syntax(content, syntax_styler_global.get_lang(fuz_lang)); + }); + + // Prism tokenize benchmark + if (Prism.languages[prism_lang]) { + bench.add(`prism_tokenize_${lang}_${size_label}`, () => { + Prism.tokenize(content, Prism.languages[prism_lang]); + }); + } + + // Stylization benchmarks (all implementations) + // Fuz Code stylize benchmark + bench.add(`fuz_code_stylize_${lang}_${size_label}`, () => { + syntax_styler_global.stylize(content, fuz_lang); + }); + + // Prism stylize benchmark + if (Prism.languages[prism_lang]) { + bench.add(`prism_stylize_${lang}_${size_label}`, () => { + Prism.highlight(content, Prism.languages[prism_lang], prism_lang); + }); + } else { + throw new Error(`Prism language not available: ${prism_lang}`); + } + + // Shiki JS engine benchmark + bench.add(`shiki_js_stylize_${lang}_${size_label}`, () => { + shiki_js.codeToHtml(content, {lang: shiki_lang, theme: 'nord'}); + }); + + // Shiki Oniguruma engine benchmark + bench.add(`shiki_oniguruma_stylize_${lang}_${size_label}`, () => { + shiki_oniguruma.codeToHtml(content, {lang: shiki_lang, theme: 'nord'}); + }); + } + } + + console.log('Running benchmarks...'); + await bench.run(); + + // Process results + const results: Array = []; + + for (const task of bench.tasks) { + if (task.result) { + // Parse benchmark name: implementation_operation_language_size + // Handle multi-word implementations like 'fuz_code' and 'shiki_js' + const parts = task.name.split('_'); + let implementation: string; + let operation: 'tokenize' | 'stylize'; + let language: string; + let content_size: 'small' | 'large'; + + if (task.name.startsWith('fuz_code_tokenize_')) { + implementation = 'fuz_code'; + operation = 'tokenize'; + language = parts[3]; + content_size = parts[4] as 'small' | 'large'; + } else if (task.name.startsWith('fuz_code_stylize_')) { + implementation = 'fuz_code'; + operation = 'stylize'; + language = parts[3]; + content_size = parts[4] as 'small' | 'large'; + } else if (task.name.startsWith('prism_tokenize_')) { + implementation = 'prism'; + operation = 'tokenize'; + language = parts[2]; + content_size = parts[3] as 'small' | 'large'; + } else if (task.name.startsWith('prism_stylize_')) { + implementation = 'prism'; + operation = 'stylize'; + language = parts[2]; + content_size = parts[3] as 'small' | 'large'; + } else if (task.name.startsWith('shiki_js_stylize_')) { + implementation = 'shiki_js'; + operation = 'stylize'; + language = parts[3]; + content_size = parts[4] as 'small' | 'large'; + } else if (task.name.startsWith('shiki_oniguruma_stylize_')) { + implementation = 'shiki_oniguruma'; + operation = 'stylize'; + language = parts[3]; + content_size = parts[4] as 'small' | 'large'; + } else { + console.warn(`Unknown benchmark name format: ${task.name}`); + continue; + } + + results.push({ + implementation, + language, + ops_per_sec: task.result.throughput.mean, + mean_time: task.result.latency.mean, + samples: task.result.latency.samples.length, + content_size, + total_time: task.result.totalTime, + operation, + }); + } + } + + return results; +}; + +export const format_comparison_results = (results: Array): string => { + const lines: Array = [ + '# Syntax Highlighting Performance Comparison', + '', + 'Comparing fuz_code vs Prism vs Shiki across multiple languages and content sizes.', + '', + '## Results', + '', + '| Language+Operation+Size | Implementation | % | Ops/sec | Mean Time (ms) |', + '|-------------------------|----------------|---|---------|----------------|', + ]; + + // Group results by language+operation+size to find fastest in each group + const grouped: Map> = new Map(); + for (const result of results) { + const key = `${result.language}_${result.operation}_${result.content_size}`; + const group = grouped.get(key) || []; + group.push(result); + grouped.set(key, group); + } + + // Calculate fastest ops/sec for each group + const fastest_by_group: Map = new Map(); + for (const [key, group] of grouped) { + const fastest = Math.max(...group.map((r) => r.ops_per_sec)); + fastest_by_group.set(key, fastest); + } + + // Sort by: language -> operation (tokenize first) -> content_size (small first) -> ops/sec (fastest first) + const sorted_results = results.sort((a, b) => { + if (a.language !== b.language) return a.language.localeCompare(b.language); + if (a.operation !== b.operation) return a.operation === 'tokenize' ? -1 : 1; + if (a.content_size !== b.content_size) return a.content_size === 'small' ? -1 : 1; + return b.ops_per_sec - a.ops_per_sec; // Fastest first + }); + + for (const result of sorted_results) { + const group_key = `${result.language}_${result.operation}_${result.content_size}`; + const fastest = fastest_by_group.get(group_key) || 1; + const percent = ((result.ops_per_sec / fastest) * 100).toFixed(0); + + const ops = result.ops_per_sec.toFixed(2); + const time = result.mean_time.toFixed(4); + // Use this to diagnose pathological cases to tweak `MIN_ITERATIONS` + // const total_time = result.total_time.toFixed(1); + + lines.push( + `| ${result.language} ${result.operation} ${result.content_size} | ${result.implementation} | ${percent}% | ${ops} | ${time} |`, + ); + } + + return lines.join('\n'); +}; + +export const run_and_print_comparison = async (filter?: string): Promise => { + console.log('Starting comparison benchmark...\n'); + + try { + const results = await run_comparison_benchmark(filter); + const report = format_comparison_results(results); + + console.log(report); + console.log('\n✅ Comparison benchmark complete'); + } catch (error) { + console.error('Comparison benchmark failed:', error); + throw error; + } +}; diff --git a/benchmark/compare/package-lock.json b/benchmark/compare/package-lock.json new file mode 100644 index 00000000..cba287fd --- /dev/null +++ b/benchmark/compare/package-lock.json @@ -0,0 +1,606 @@ +{ + "name": "fuz_code_benchmark", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fuz_code_benchmark", + "devDependencies": { + "@types/prismjs": "^1.26.5", + "prism-svelte": "^0.5.0", + "prismjs": "^1.30.0", + "shiki": "^3.13.0" + } + }, + "node_modules/@shikijs/core": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-3.13.0.tgz", + "integrity": "sha512-3P8rGsg2Eh2qIHekwuQjzWhKI4jV97PhvYjYUzGqjvJfqdQPz+nMlfWahU24GZAyW1FxFI1sYjyhfh5CoLmIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.13.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.5" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-3.13.0.tgz", + "integrity": "sha512-Ty7xv32XCp8u0eQt8rItpMs6rU9Ki6LJ1dQOW3V/56PKDcpvfHPnYFbsx5FFUP2Yim34m/UkazidamMNVR4vKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.13.0", + "@shikijs/vscode-textmate": "^10.0.2", + "oniguruma-to-es": "^4.3.3" + } + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-3.13.0.tgz", + "integrity": "sha512-O42rBGr4UDSlhT2ZFMxqM7QzIU+IcpoTMzb3W7AlziI1ZF7R8eS2M0yt5Ry35nnnTX/LTLXFPUjRFCIW+Operg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.13.0", + "@shikijs/vscode-textmate": "^10.0.2" + } + }, + "node_modules/@shikijs/langs": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.13.0.tgz", + "integrity": "sha512-672c3WAETDYHwrRP0yLy3W1QYB89Hbpj+pO4KhxK6FzIrDI2FoEXNiNCut6BQmEApYLfuYfpgOZaqbY+E9b8wQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.13.0" + } + }, + "node_modules/@shikijs/themes": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.13.0.tgz", + "integrity": "sha512-Vxw1Nm1/Od8jyA7QuAenaV78BG2nSr3/gCGdBkLpfLscddCkzkL36Q5b67SrLLfvAJTOUzW39x4FHVCFriPVgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.13.0" + } + }, + "node_modules/@shikijs/types": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.13.0.tgz", + "integrity": "sha512-oM9P+NCFri/mmQ8LoFGVfVyemm5Hi27330zuOBp0annwJdKH1kOLndw3zCtAVDehPLg9fKqoEx3Ht/wNZxolfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/prismjs": { + "version": "1.26.5", + "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.5.tgz", + "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true, + "license": "ISC" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-to-html": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", + "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", + "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/oniguruma-parser": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.1.tgz", + "integrity": "sha512-8Unqkvk1RYc6yq2WBYRj4hdnsAxVze8i7iPfQr8e4uSP3tRv0rpZcbGUDvxfQQcdwHt/e9PrMvGCsa8OqG9X3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/oniguruma-to-es": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.3.tgz", + "integrity": "sha512-rPiZhzC3wXwE59YQMRDodUwwT9FZ9nNBwQQfsd1wfdtlKEyCdRV0avrTcSZ5xlIvGRVPd/cx6ZN45ECmS39xvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "oniguruma-parser": "^0.12.1", + "regex": "^6.0.1", + "regex-recursion": "^6.0.2" + } + }, + "node_modules/prism-svelte": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/prism-svelte/-/prism-svelte-0.5.0.tgz", + "integrity": "sha512-db91Bf3pRGKDPz1lAqLFSJXeW13mulUJxhycysFpfXV5MIK7RgWWK2E5aPAa71s8TCzQUXxF5JOV42/iOs6QkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/prismjs": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", + "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/property-information": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", + "integrity": "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/regex/-/regex-6.0.1.tgz", + "integrity": "sha512-uorlqlzAKjKQZ5P+kTJr3eeJGSVroLKoHmquUj4zHWuR+hEyNqlXsSKlYYF5F4NI6nl7tWCs0apKJ0lmfsXAPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", + "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "dev": true, + "license": "MIT" + }, + "node_modules/shiki": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-3.13.0.tgz", + "integrity": "sha512-aZW4l8Og16CokuCLf8CF8kq+KK2yOygapU5m3+hoGw0Mdosc6fPitjM+ujYarppj5ZIKGyPDPP1vqmQhr+5/0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/core": "3.13.0", + "@shikijs/engine-javascript": "3.13.0", + "@shikijs/engine-oniguruma": "3.13.0", + "@shikijs/langs": "3.13.0", + "@shikijs/themes": "3.13.0", + "@shikijs/types": "3.13.0", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "dev": true, + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/benchmark/compare/package.json b/benchmark/compare/package.json new file mode 100644 index 00000000..88aeb590 --- /dev/null +++ b/benchmark/compare/package.json @@ -0,0 +1,10 @@ +{ + "name": "fuz_code_benchmark", + "type": "module", + "devDependencies": { + "@types/prismjs": "^1.26.5", + "prism-svelte": "^0.5.0", + "prismjs": "^1.30.0", + "shiki": "^3.13.0" + } +} diff --git a/benchmark/compare/results.md b/benchmark/compare/results.md new file mode 100644 index 00000000..5a08c7d3 --- /dev/null +++ b/benchmark/compare/results.md @@ -0,0 +1,66 @@ +# Syntax Highlighting Performance Comparison + +## Results + +| Language+Operation+Size | Implementation | % | Ops/sec | Mean Time (ms) | +| ----------------------- | --------------- | ---- | -------- | -------------- | +| css tokenize small | fuz_code | 100% | 58435.54 | 0.0174 | +| css tokenize small | prism | 95% | 55478.47 | 0.0183 | +| css tokenize large | fuz_code | 100% | 572.29 | 1.7902 | +| css tokenize large | prism | 96% | 549.77 | 1.8614 | +| css stylize small | fuz_code | 100% | 25824.88 | 0.0393 | +| css stylize small | prism | 93% | 24059.34 | 0.0421 | +| css stylize small | shiki_js | 5% | 1359.65 | 0.7490 | +| css stylize small | shiki_oniguruma | 2% | 632.21 | 1.5908 | +| css stylize large | fuz_code | 100% | 225.48 | 4.4705 | +| css stylize large | prism | 85% | 192.03 | 5.3467 | +| css stylize large | shiki_js | 6% | 13.09 | 76.5671 | +| css stylize large | shiki_oniguruma | 3% | 6.20 | 161.4719 | +| html tokenize small | prism | 100% | 28071.50 | 0.0362 | +| html tokenize small | fuz_code | 80% | 22353.53 | 0.0454 | +| html tokenize large | prism | 100% | 261.66 | 3.8329 | +| html tokenize large | fuz_code | 67% | 175.24 | 5.7740 | +| html stylize small | prism | 100% | 13185.90 | 0.0772 | +| html stylize small | fuz_code | 92% | 12163.60 | 0.0837 | +| html stylize small | shiki_oniguruma | 6% | 794.71 | 1.2666 | +| html stylize small | shiki_js | 6% | 774.19 | 1.3020 | +| html stylize large | prism | 100% | 83.83 | 12.3378 | +| html stylize large | fuz_code | 94% | 79.04 | 12.8017 | +| html stylize large | shiki_oniguruma | 9% | 7.83 | 127.7735 | +| html stylize large | shiki_js | 9% | 7.82 | 127.9444 | +| json tokenize small | prism | 100% | 73749.21 | 0.0138 | +| json tokenize small | fuz_code | 99% | 73235.11 | 0.0140 | +| json tokenize large | prism | 100% | 694.84 | 1.5010 | +| json tokenize large | fuz_code | 99% | 686.16 | 1.5154 | +| json stylize small | fuz_code | 100% | 28875.11 | 0.0354 | +| json stylize small | prism | 92% | 26525.47 | 0.0384 | +| json stylize small | shiki_js | 7% | 1957.01 | 0.5252 | +| json stylize small | shiki_oniguruma | 6% | 1731.06 | 0.5960 | +| json stylize large | fuz_code | 100% | 239.85 | 4.2362 | +| json stylize large | prism | 87% | 207.78 | 5.0337 | +| json stylize large | shiki_js | 8% | 18.99 | 52.7780 | +| json stylize large | shiki_oniguruma | 7% | 17.10 | 58.6602 | +| svelte tokenize small | fuz_code | 100% | 3976.35 | 0.2547 | +| svelte tokenize small | prism | 90% | 3565.41 | 0.2832 | +| svelte tokenize large | fuz_code | 100% | 34.76 | 28.9565 | +| svelte tokenize large | prism | 87% | 30.35 | 33.1966 | +| svelte stylize small | fuz_code | 100% | 2360.10 | 0.4321 | +| svelte stylize small | prism | 86% | 2021.82 | 0.5108 | +| svelte stylize small | shiki_oniguruma | 3% | 61.53 | 16.3099 | +| svelte stylize small | shiki_js | 2% | 50.00 | 20.0518 | +| svelte stylize large | fuz_code | 100% | 13.09 | 76.8902 | +| svelte stylize large | prism | 84% | 11.03 | 91.0119 | +| svelte stylize large | shiki_oniguruma | 5% | 0.63 | 1585.1294 | +| svelte stylize large | shiki_js | 4% | 0.52 | 1916.0590 | +| ts tokenize small | fuz_code | 100% | 8619.08 | 0.1165 | +| ts tokenize small | prism | 90% | 7757.57 | 0.1293 | +| ts tokenize large | fuz_code | 100% | 54.49 | 18.3994 | +| ts tokenize large | prism | 94% | 51.31 | 19.5324 | +| ts stylize small | fuz_code | 100% | 5516.74 | 0.1822 | +| ts stylize small | prism | 88% | 4829.14 | 0.2086 | +| ts stylize small | shiki_oniguruma | 4% | 201.12 | 4.9793 | +| ts stylize small | shiki_js | 3% | 138.64 | 7.2199 | +| ts stylize large | fuz_code | 100% | 33.44 | 30.3301 | +| ts stylize large | prism | 70% | 23.29 | 43.0380 | +| ts stylize large | shiki_oniguruma | 6% | 1.98 | 505.1610 | +| ts stylize large | shiki_js | 4% | 1.37 | 732.2407 | diff --git a/benchmark/compare/run_compare.ts b/benchmark/compare/run_compare.ts new file mode 100644 index 00000000..402b8c86 --- /dev/null +++ b/benchmark/compare/run_compare.ts @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import {run_and_print_comparison} from './compare.js'; + +const filter = process.argv[2]; + +run_and_print_comparison(filter) + .then(() => { + process.exit(0); + }) + .catch((error) => { + console.error('comparison benchmark failed:', error); // eslint-disable-line no-console + process.exit(1); + }); \ No newline at end of file diff --git a/benchmark/results.md b/benchmark/results.md new file mode 100644 index 00000000..0405cb67 --- /dev/null +++ b/benchmark/results.md @@ -0,0 +1,34 @@ +# Benchmark Baseline Results + +## Benchmark Results + +| Sample | Ops/sec | Mean Time (ms) | Samples | +| -------------------- | -------- | -------------- | ------- | +| json_complex | 29821.82 | 0.0341 | 293671 | +| css_complex | 26826.07 | 0.0381 | 262506 | +| ts_complex | 5479.79 | 0.1845 | 54198 | +| html_complex | 12268.33 | 0.0832 | 120121 | +| svelte_complex | 2404.65 | 0.4203 | 23791 | +| large:json_complex | 258.37 | 3.9305 | 2545 | +| large:css_complex | 239.62 | 4.2260 | 2367 | +| large:ts_complex | 30.66 | 33.1579 | 302 | +| large:html_complex | 84.14 | 12.1397 | 824 | +| large:svelte_complex | 13.55 | 74.7600 | 134 | + +**Total samples benchmarked:** 10 +**Average ops/sec:** 7742.70 + +## Browser Benchmark Results + +| Language | Implementation | Mean (ms) | Median (ms) | Std Dev | CV | P95 (ms) | Ops/sec | Outliers | Failed | Stability | +| -------- | -------------- | --------- | ----------- | ------- | ----- | -------- | ------- | -------- | ------ | --------- | +| ts | html | 82.39 | 80.95 | 3.98 | 4.8% | 87.60 | 12 | 0/10 | 0 | 100% | +| ts | ranges | 38.74 | 38.60 | 3.02 | 7.8% | 43.80 | 26 | 0/10 | 0 | 100% | +| css | html | 840.65 | 840.20 | 9.26 | 1.1% | 854.80 | 1 | 0/10 | 0 | 90% | +| css | ranges | 14.01 | 14.30 | 0.78 | 5.6% | 14.90 | 71 | 1/10 | 0 | 90% | +| html | html | 62.01 | 64.90 | 9.28 | 15.0% | 71.30 | 16 | 0/10 | 0 | 100% | +| html | ranges | 20.65 | 21.45 | 2.26 | 10.9% | 23.60 | 48 | 0/10 | 0 | 100% | +| json | html | 402.64 | 401.80 | 3.07 | 0.8% | 407.80 | 2 | 3/10 | 0 | 90% | +| json | ranges | 13.29 | 13.40 | 0.74 | 5.6% | 14.20 | 75 | 1/10 | 0 | 90% | +| svelte | html | 175.48 | 166.10 | 21.47 | 12.2% | 218.10 | 6 | 0/10 | 0 | 100% | +| svelte | ranges | 100.58 | 101.70 | 7.36 | 7.3% | 113.70 | 10 | 1/10 | 0 | 100% | diff --git a/benchmark/run_benchmarks.ts b/benchmark/run_benchmarks.ts new file mode 100644 index 00000000..e0362aea --- /dev/null +++ b/benchmark/run_benchmarks.ts @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import {run_and_print_benchmark} from './benchmarks.js'; + +const filter = process.argv[2]; + +run_and_print_benchmark(filter) + .then(() => { + process.exit(0); + }) + .catch((error) => { + console.error('benchmark failed:', error); // eslint-disable-line no-console + process.exit(1); + }); diff --git a/fixtures/generated/css/css_complex.json b/fixtures/generated/css/css_complex.json new file mode 100644 index 00000000..caf89022 --- /dev/null +++ b/fixtures/generated/css/css_complex.json @@ -0,0 +1,88 @@ +{ + "sample": { + "lang": "css", + "variant": "complex", + "content": ".some_class {\n\tcolor: red;\n}\n\n.hypen-class {\n\tfont-size: 16px;\n}\n\np {\n\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n}\n\n/* comment */\n\n/*\nmulti\n.line {\n\ti: 100px\n\n\n\n@media*/\n\n#id {\n\tbackground-color: blue;\n}\n\ndiv > p {\n\tmargin: 10px;\n}\n\n@media (max-width: 600px) {\n\tbody {\n\t\tbackground-color: lightblue;\n\t}\n}\n\n/* patterns in strings are not falsely detected */\n.content::before {\n\tcontent: ' /* not a comment */';\n}\n\n.attr[title='Click: here'] {\n\tcursor: pointer;\n}\n\n.background /* comment*/ {\n\tbackground-image: url('data:image/svg+xml...');\n}\n", + "filepath": "src/lib/samples/sample_complex.css" + }, + "tokens": [ + {"type": "selector", "start": 0, "end": 11}, + {"type": "punctuation", "start": 12, "end": 13}, + {"type": "property", "start": 15, "end": 20}, + {"type": "punctuation", "start": 20, "end": 21}, + {"type": "punctuation", "start": 25, "end": 26}, + {"type": "punctuation", "start": 27, "end": 28}, + {"type": "selector", "start": 30, "end": 42}, + {"type": "punctuation", "start": 43, "end": 44}, + {"type": "property", "start": 46, "end": 55}, + {"type": "punctuation", "start": 55, "end": 56}, + {"type": "punctuation", "start": 61, "end": 62}, + {"type": "punctuation", "start": 63, "end": 64}, + {"type": "selector", "start": 66, "end": 67}, + {"type": "punctuation", "start": 68, "end": 69}, + {"type": "property", "start": 71, "end": 81}, + {"type": "punctuation", "start": 81, "end": 82}, + {"type": "function", "start": 92, "end": 96}, + {"type": "punctuation", "start": 96, "end": 97}, + {"type": "punctuation", "start": 98, "end": 99}, + {"type": "punctuation", "start": 101, "end": 102}, + {"type": "punctuation", "start": 104, "end": 105}, + {"type": "punctuation", "start": 109, "end": 110}, + {"type": "punctuation", "start": 110, "end": 111}, + {"type": "punctuation", "start": 112, "end": 113}, + {"type": "comment", "start": 115, "end": 128}, + {"type": "comment", "start": 130, "end": 176}, + {"type": "selector", "start": 178, "end": 181}, + {"type": "punctuation", "start": 182, "end": 183}, + {"type": "property", "start": 185, "end": 201}, + {"type": "punctuation", "start": 201, "end": 202}, + {"type": "punctuation", "start": 207, "end": 208}, + {"type": "punctuation", "start": 209, "end": 210}, + {"type": "selector", "start": 212, "end": 219}, + {"type": "punctuation", "start": 220, "end": 221}, + {"type": "property", "start": 223, "end": 229}, + {"type": "punctuation", "start": 229, "end": 230}, + {"type": "punctuation", "start": 235, "end": 236}, + {"type": "punctuation", "start": 237, "end": 238}, + {"type": "atrule", "start": 240, "end": 265}, + {"type": "rule", "start": 240, "end": 246}, + {"type": "punctuation", "start": 247, "end": 248}, + {"type": "property", "start": 248, "end": 257}, + {"type": "punctuation", "start": 257, "end": 258}, + {"type": "punctuation", "start": 264, "end": 265}, + {"type": "punctuation", "start": 266, "end": 267}, + {"type": "selector", "start": 269, "end": 273}, + {"type": "punctuation", "start": 274, "end": 275}, + {"type": "property", "start": 278, "end": 294}, + {"type": "punctuation", "start": 294, "end": 295}, + {"type": "punctuation", "start": 305, "end": 306}, + {"type": "punctuation", "start": 308, "end": 309}, + {"type": "punctuation", "start": 310, "end": 311}, + {"type": "comment", "start": 313, "end": 363}, + {"type": "selector", "start": 364, "end": 380}, + {"type": "punctuation", "start": 381, "end": 382}, + {"type": "property", "start": 384, "end": 391}, + {"type": "punctuation", "start": 391, "end": 392}, + {"type": "string", "start": 393, "end": 423}, + {"type": "punctuation", "start": 423, "end": 424}, + {"type": "punctuation", "start": 425, "end": 426}, + {"type": "selector", "start": 428, "end": 454}, + {"type": "punctuation", "start": 455, "end": 456}, + {"type": "property", "start": 458, "end": 464}, + {"type": "punctuation", "start": 464, "end": 465}, + {"type": "punctuation", "start": 473, "end": 474}, + {"type": "punctuation", "start": 475, "end": 476}, + {"type": "comment", "start": 491, "end": 503}, + {"type": "punctuation", "start": 504, "end": 505}, + {"type": "property", "start": 507, "end": 523}, + {"type": "punctuation", "start": 523, "end": 524}, + {"type": "url", "start": 525, "end": 553}, + {"type": "function", "start": 525, "end": 528}, + {"type": "punctuation", "start": 528, "end": 529}, + {"type": "string", "start": 529, "end": 552}, + {"type": "punctuation", "start": 552, "end": 553}, + {"type": "punctuation", "start": 553, "end": 554}, + {"type": "punctuation", "start": 555, "end": 556} + ], + "html": ".some_class {\n\tcolor: red;\n}\n\n.hypen-class {\n\tfont-size: 16px;\n}\n\np {\n\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n}\n\n/* comment */\n\n/*\nmulti\n.line {\n\ti: 100px\n\n</style>\n\n@media*/\n\n#id {\n\tbackground-color: blue;\n}\n\ndiv > p {\n\tmargin: 10px;\n}\n\n@media (max-width: 600px) {\n\tbody {\n\t\tbackground-color: lightblue;\n\t}\n}\n\n/* patterns in strings are not falsely detected */\n.content::before {\n\tcontent: '</style> /* not a comment */';\n}\n\n.attr[title='Click: here'] {\n\tcursor: pointer;\n}\n\n.background /* comment*/ {\n\tbackground-image: url('data:image/svg+xml...');\n}\n" +} diff --git a/fixtures/generated/css/css_complex.txt b/fixtures/generated/css/css_complex.txt new file mode 100644 index 00000000..9ae37ae4 --- /dev/null +++ b/fixtures/generated/css/css_complex.txt @@ -0,0 +1,93 @@ +=== TOKENS === + 0-11 selector .some_class + 12-13 punctuation { + 15-20 property color + 20-21 punctuation : + 25-26 punctuation ; + 27-28 punctuation } + 30-42 selector .hypen-class + 43-44 punctuation { + 46-55 property font-size + 55-56 punctuation : + 61-62 punctuation ; + 63-64 punctuation } + 66-67 selector p + 68-69 punctuation { + 71-81 property box-shadow + 81-82 punctuation : + 92-96 function rgba + 96-97 punctuation ( + 98-99 punctuation , + 101-102 punctuation , + 104-105 punctuation , + 109-110 punctuation ) + 110-111 punctuation ; + 112-113 punctuation } + 115-128 comment /* comment */ + 130-176 comment /*\nmulti\n.line {\n\ti: 100px\n\n\n\n@media*/ + 178-181 selector #id + 182-183 punctuation { + 185-201 property background-color + 201-202 punctuation : + 207-208 punctuation ; + 209-210 punctuation } + 212-219 selector div > p + 220-221 punctuation { + 223-229 property margin + 229-230 punctuation : + 235-236 punctuation ; + 237-238 punctuation } + 240-265 atrule @media (max-width: 600px) + 240-246 rule @media + 247-248 punctuation ( + 248-257 property max-width + 257-258 punctuation : + 264-265 punctuation ) + 266-267 punctuation { + 269-273 selector body + 274-275 punctuation { + 278-294 property background-color + 294-295 punctuation : + 305-306 punctuation ; + 308-309 punctuation } + 310-311 punctuation } + 313-363 comment /* patterns in strings are not falsely detected */ + 364-380 selector .content::before + 381-382 punctuation { + 384-391 property content + 391-392 punctuation : + 393-423 string ' /* not a comment */' + 423-424 punctuation ; + 425-426 punctuation } + 428-454 selector .attr[title='Click: here'] + 455-456 punctuation { + 458-464 property cursor + 464-465 punctuation : + 473-474 punctuation ; + 475-476 punctuation } + 491-503 comment /* comment*/ + 504-505 punctuation { + 507-523 property background-image + 523-524 punctuation : + 525-553 url url('data:image/svg+xml...') + 525-528 function url + 528-529 punctuation ( + 529-552 string 'data:image/svg+xml...' + 552-553 punctuation ) + 553-554 punctuation ; + 555-556 punctuation } + +=== STATS === +Total tokens: 77 +Sample length: 557 characters + +Token types: + punctuation: 48 + property: 10 + selector: 8 + comment: 4 + function: 2 + string: 2 + atrule: 1 + rule: 1 + url: 1 diff --git a/fixtures/generated/html/html_complex.json b/fixtures/generated/html/html_complex.json new file mode 100644 index 00000000..72a85484 --- /dev/null +++ b/fixtures/generated/html/html_complex.json @@ -0,0 +1,162 @@ +{ + "sample": { + "lang": "html", + "variant": "complex", + "content": "\n\n
\n\t

hello world!

\n
\n\n

some text

\n\n\n\n\n\n
\n\n
\n\n\"access\"\n\n
    \n\t
  • list item 1
    • \n\t
  • list item 2
    • \n
\n\n\n\n\n\n ]]>\n", + "filepath": "src/lib/samples/sample_complex.html" + }, + "tokens": [ + {"type": "doctype", "start": 0, "end": 15}, + {"type": "tag", "start": 17, "end": 35}, + {"type": "tag", "start": 17, "end": 21}, + {"type": "punctuation", "start": 17, "end": 18}, + {"type": "attr_name", "start": 22, "end": 27}, + {"type": "attr_value", "start": 27, "end": 34}, + {"type": "punctuation", "start": 27, "end": 28}, + {"type": "punctuation", "start": 28, "end": 29}, + {"type": "punctuation", "start": 33, "end": 34}, + {"type": "punctuation", "start": 34, "end": 35}, + {"type": "tag", "start": 37, "end": 40}, + {"type": "tag", "start": 37, "end": 39}, + {"type": "punctuation", "start": 37, "end": 38}, + {"type": "punctuation", "start": 39, "end": 40}, + {"type": "tag", "start": 52, "end": 56}, + {"type": "tag", "start": 52, "end": 55}, + {"type": "punctuation", "start": 52, "end": 54}, + {"type": "punctuation", "start": 55, "end": 56}, + {"type": "tag", "start": 57, "end": 63}, + {"type": "tag", "start": 57, "end": 62}, + {"type": "punctuation", "start": 57, "end": 59}, + {"type": "punctuation", "start": 62, "end": 63}, + {"type": "tag", "start": 65, "end": 99}, + {"type": "tag", "start": 65, "end": 67}, + {"type": "punctuation", "start": 65, "end": 66}, + {"type": "attr_name", "start": 68, "end": 73}, + {"type": "attr_value", "start": 73, "end": 98}, + {"type": "punctuation", "start": 73, "end": 74}, + {"type": "punctuation", "start": 74, "end": 75}, + {"type": "punctuation", "start": 97, "end": 98}, + {"type": "punctuation", "start": 98, "end": 99}, + {"type": "tag", "start": 104, "end": 124}, + {"type": "tag", "start": 104, "end": 109}, + {"type": "punctuation", "start": 104, "end": 105}, + {"type": "attr_name", "start": 110, "end": 115}, + {"type": "attr_value", "start": 115, "end": 123}, + {"type": "punctuation", "start": 115, "end": 116}, + {"type": "punctuation", "start": 116, "end": 117}, + {"type": "punctuation", "start": 122, "end": 123}, + {"type": "punctuation", "start": 123, "end": 124}, + {"type": "tag", "start": 128, "end": 135}, + {"type": "tag", "start": 128, "end": 134}, + {"type": "punctuation", "start": 128, "end": 130}, + {"type": "punctuation", "start": 134, "end": 135}, + {"type": "tag", "start": 135, "end": 139}, + {"type": "tag", "start": 135, "end": 138}, + {"type": "punctuation", "start": 135, "end": 137}, + {"type": "punctuation", "start": 138, "end": 139}, + {"type": "tag", "start": 141, "end": 172}, + {"type": "tag", "start": 141, "end": 148}, + {"type": "punctuation", "start": 141, "end": 142}, + {"type": "attr_name", "start": 149, "end": 153}, + {"type": "attr_value", "start": 153, "end": 162}, + {"type": "punctuation", "start": 153, "end": 154}, + {"type": "punctuation", "start": 154, "end": 155}, + {"type": "punctuation", "start": 161, "end": 162}, + {"type": "attr_name", "start": 163, "end": 171}, + {"type": "punctuation", "start": 171, "end": 172}, + {"type": "tag", "start": 180, "end": 189}, + {"type": "tag", "start": 180, "end": 188}, + {"type": "punctuation", "start": 180, "end": 182}, + {"type": "punctuation", "start": 188, "end": 189}, + {"type": "comment", "start": 191, "end": 237}, + {"type": "tag", "start": 239, "end": 245}, + {"type": "tag", "start": 239, "end": 242}, + {"type": "punctuation", "start": 239, "end": 240}, + {"type": "punctuation", "start": 243, "end": 245}, + {"type": "tag", "start": 247, "end": 253}, + {"type": "tag", "start": 247, "end": 250}, + {"type": "punctuation", "start": 247, "end": 248}, + {"type": "punctuation", "start": 251, "end": 253}, + {"type": "tag", "start": 255, "end": 291}, + {"type": "tag", "start": 255, "end": 259}, + {"type": "punctuation", "start": 255, "end": 256}, + {"type": "attr_name", "start": 260, "end": 263}, + {"type": "attr_value", "start": 263, "end": 275}, + {"type": "punctuation", "start": 263, "end": 264}, + {"type": "punctuation", "start": 264, "end": 265}, + {"type": "punctuation", "start": 274, "end": 275}, + {"type": "attr_name", "start": 276, "end": 279}, + {"type": "attr_value", "start": 279, "end": 288}, + {"type": "punctuation", "start": 279, "end": 280}, + {"type": "punctuation", "start": 280, "end": 281}, + {"type": "punctuation", "start": 287, "end": 288}, + {"type": "punctuation", "start": 289, "end": 291}, + {"type": "tag", "start": 293, "end": 297}, + {"type": "tag", "start": 293, "end": 296}, + {"type": "punctuation", "start": 293, "end": 294}, + {"type": "punctuation", "start": 296, "end": 297}, + {"type": "tag", "start": 299, "end": 303}, + {"type": "tag", "start": 299, "end": 302}, + {"type": "punctuation", "start": 299, "end": 300}, + {"type": "punctuation", "start": 302, "end": 303}, + {"type": "tag", "start": 314, "end": 319}, + {"type": "tag", "start": 314, "end": 318}, + {"type": "punctuation", "start": 314, "end": 316}, + {"type": "punctuation", "start": 318, "end": 319}, + {"type": "tag", "start": 321, "end": 325}, + {"type": "tag", "start": 321, "end": 324}, + {"type": "punctuation", "start": 321, "end": 322}, + {"type": "punctuation", "start": 324, "end": 325}, + {"type": "tag", "start": 336, "end": 341}, + {"type": "tag", "start": 336, "end": 340}, + {"type": "punctuation", "start": 336, "end": 338}, + {"type": "punctuation", "start": 340, "end": 341}, + {"type": "tag", "start": 342, "end": 347}, + {"type": "tag", "start": 342, "end": 346}, + {"type": "punctuation", "start": 342, "end": 344}, + {"type": "punctuation", "start": 346, "end": 347}, + {"type": "tag", "start": 349, "end": 380}, + {"type": "tag", "start": 349, "end": 356}, + {"type": "punctuation", "start": 349, "end": 350}, + {"type": "attr_name", "start": 357, "end": 361}, + {"type": "attr_value", "start": 361, "end": 379}, + {"type": "punctuation", "start": 361, "end": 362}, + {"type": "punctuation", "start": 362, "end": 363}, + {"type": "punctuation", "start": 378, "end": 379}, + {"type": "punctuation", "start": 379, "end": 380}, + {"type": "script", "start": 380, "end": 404}, + {"type": "lang_js", "start": 380, "end": 404}, + {"type": "keyword", "start": 382, "end": 387}, + {"type": "operator", "start": 391, "end": 392}, + {"type": "string", "start": 393, "end": 402}, + {"type": "punctuation", "start": 402, "end": 403}, + {"type": "tag", "start": 404, "end": 413}, + {"type": "tag", "start": 404, "end": 412}, + {"type": "punctuation", "start": 404, "end": 406}, + {"type": "punctuation", "start": 412, "end": 413}, + {"type": "tag", "start": 415, "end": 438}, + {"type": "tag", "start": 415, "end": 421}, + {"type": "punctuation", "start": 415, "end": 416}, + {"type": "attr_name", "start": 422, "end": 426}, + {"type": "attr_value", "start": 426, "end": 437}, + {"type": "punctuation", "start": 426, "end": 427}, + {"type": "punctuation", "start": 427, "end": 428}, + {"type": "punctuation", "start": 436, "end": 437}, + {"type": "punctuation", "start": 437, "end": 438}, + {"type": "style", "start": 438, "end": 482}, + {"type": "lang_css", "start": 438, "end": 482}, + {"type": "selector", "start": 440, "end": 456}, + {"type": "punctuation", "start": 457, "end": 458}, + {"type": "property", "start": 461, "end": 468}, + {"type": "punctuation", "start": 468, "end": 469}, + {"type": "string", "start": 470, "end": 477}, + {"type": "punctuation", "start": 477, "end": 478}, + {"type": "punctuation", "start": 480, "end": 481}, + {"type": "tag", "start": 482, "end": 490}, + {"type": "tag", "start": 482, "end": 489}, + {"type": "punctuation", "start": 482, "end": 484}, + {"type": "punctuation", "start": 489, "end": 490}, + {"type": "cdata", "start": 492, "end": 540} + ], + "html": "<!doctype html>\n\n<div class=\"test\">\n\t<p>hello world!</p>\n</div>\n\n<p class=\"some_class hypen-class\">some <span class=\"a b c\">text</span></p>\n\n<button type=\"button\" disabled>click me</button>\n\n<!-- comment <div>a<br /> b</div> <script> -->\n\n<br />\n\n<hr />\n\n<img src=\"image.jpg\" alt=\"access\" />\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<script type=\"text/javascript\">\n\tconst ok = '<style>';\n</script>\n\n<style type=\"text/css\">\n\t.special::before {\n\t\tcontent: '< & >';\n\t}\n</style>\n\n<![CDATA[ if (a < 0) alert(\"b\"); <not-a-tag> ]]>\n" +} diff --git a/fixtures/generated/html/html_complex.txt b/fixtures/generated/html/html_complex.txt new file mode 100644 index 00000000..cd8b4eba --- /dev/null +++ b/fixtures/generated/html/html_complex.txt @@ -0,0 +1,174 @@ +=== TOKENS === + 0-15 doctype + 17-35 tag
+ 17-21 tag
+ 37-40 tag

+ 37-39 tag

+ 52-56 tag

+ 52-55 tag

+ 57-63 tag
+ 57-62 tag
+ 65-99 tag

+ 65-67 tag

+ 104-124 tag + 104-109 tag + 128-135 tag + 128-134 tag + 135-139 tag

+ 135-138 tag

+ 141-172 tag + 180-188 tag + 191-237 comment + 239-245 tag
+ 239-242 tag
+ 247-253 tag
+ 247-250 tag
+ 255-291 tag access + 255-259 tag + 293-297 tag
    + 293-296 tag
      + 299-303 tag
    • + 299-302 tag
    • + 314-319 tag
      • + 314-318 tag + 321-325 tag
    • + 321-324 tag
    • + 336-341 tag
      • + 336-340 tag + 342-347 tag
    + 342-346 tag
+ 349-380 tag + 404-412 tag + 415-438 tag + 482-489 tag + 492-540 cdata ]]> + +=== STATS === +Total tokens: 151 +Sample length: 541 characters + +Token types: + punctuation: 75 + tag: 46 + attr_name: 9 + attr_value: 8 + string: 2 + doctype: 1 + comment: 1 + script: 1 + lang_js: 1 + keyword: 1 + operator: 1 + style: 1 + lang_css: 1 + selector: 1 + property: 1 + cdata: 1 diff --git a/fixtures/generated/json/json_complex.json b/fixtures/generated/json/json_complex.json new file mode 100644 index 00000000..2b43c70d --- /dev/null +++ b/fixtures/generated/json/json_complex.json @@ -0,0 +1,94 @@ +{ + "sample": { + "lang": "json", + "variant": "complex", + "content": "{\n\t\"string\": \"a string\",\n\t\"number\": 12345,\n\t\"boolean\": true,\n\t\"null\": null,\n\t\"empty\": \"\",\n\t\"escaped\": \"quote: \\\"test\\\" and backslash: \\\\\",\n\t\"object\": {\n\t\t\"array\": [1, \"b\", false],\n\t\t\"strings\": [\"1\", \"2\", \"3\"],\n\t\t\"mixed\": [\"start\", 123, true, \"middle\", null, \"end\"],\n\t\t\"nested\": [[\"a\", \"str\", \"\"], {\"key\": \"nested value\"}]\n\t}\n}\n", + "filepath": "src/lib/samples/sample_complex.json" + }, + "tokens": [ + {"type": "punctuation", "start": 0, "end": 1}, + {"type": "property", "start": 3, "end": 11}, + {"type": "operator", "start": 11, "end": 12}, + {"type": "string", "start": 13, "end": 23}, + {"type": "punctuation", "start": 23, "end": 24}, + {"type": "property", "start": 26, "end": 34}, + {"type": "operator", "start": 34, "end": 35}, + {"type": "number", "start": 36, "end": 41}, + {"type": "punctuation", "start": 41, "end": 42}, + {"type": "property", "start": 44, "end": 53}, + {"type": "operator", "start": 53, "end": 54}, + {"type": "boolean", "start": 55, "end": 59}, + {"type": "punctuation", "start": 59, "end": 60}, + {"type": "property", "start": 62, "end": 68}, + {"type": "operator", "start": 68, "end": 69}, + {"type": "null", "start": 70, "end": 74}, + {"type": "punctuation", "start": 74, "end": 75}, + {"type": "property", "start": 77, "end": 84}, + {"type": "operator", "start": 84, "end": 85}, + {"type": "string", "start": 86, "end": 88}, + {"type": "punctuation", "start": 88, "end": 89}, + {"type": "property", "start": 91, "end": 100}, + {"type": "operator", "start": 100, "end": 101}, + {"type": "string", "start": 102, "end": 137}, + {"type": "punctuation", "start": 137, "end": 138}, + {"type": "property", "start": 140, "end": 148}, + {"type": "operator", "start": 148, "end": 149}, + {"type": "punctuation", "start": 150, "end": 151}, + {"type": "property", "start": 154, "end": 161}, + {"type": "operator", "start": 161, "end": 162}, + {"type": "punctuation", "start": 163, "end": 164}, + {"type": "number", "start": 164, "end": 165}, + {"type": "punctuation", "start": 165, "end": 166}, + {"type": "string", "start": 167, "end": 170}, + {"type": "punctuation", "start": 170, "end": 171}, + {"type": "boolean", "start": 172, "end": 177}, + {"type": "punctuation", "start": 177, "end": 178}, + {"type": "punctuation", "start": 178, "end": 179}, + {"type": "property", "start": 182, "end": 191}, + {"type": "operator", "start": 191, "end": 192}, + {"type": "punctuation", "start": 193, "end": 194}, + {"type": "string", "start": 194, "end": 197}, + {"type": "punctuation", "start": 197, "end": 198}, + {"type": "string", "start": 199, "end": 202}, + {"type": "punctuation", "start": 202, "end": 203}, + {"type": "string", "start": 204, "end": 207}, + {"type": "punctuation", "start": 207, "end": 208}, + {"type": "punctuation", "start": 208, "end": 209}, + {"type": "property", "start": 212, "end": 219}, + {"type": "operator", "start": 219, "end": 220}, + {"type": "punctuation", "start": 221, "end": 222}, + {"type": "string", "start": 222, "end": 229}, + {"type": "punctuation", "start": 229, "end": 230}, + {"type": "number", "start": 231, "end": 234}, + {"type": "punctuation", "start": 234, "end": 235}, + {"type": "boolean", "start": 236, "end": 240}, + {"type": "punctuation", "start": 240, "end": 241}, + {"type": "string", "start": 242, "end": 250}, + {"type": "punctuation", "start": 250, "end": 251}, + {"type": "null", "start": 252, "end": 256}, + {"type": "punctuation", "start": 256, "end": 257}, + {"type": "string", "start": 258, "end": 263}, + {"type": "punctuation", "start": 263, "end": 264}, + {"type": "punctuation", "start": 264, "end": 265}, + {"type": "property", "start": 268, "end": 276}, + {"type": "operator", "start": 276, "end": 277}, + {"type": "punctuation", "start": 278, "end": 279}, + {"type": "punctuation", "start": 279, "end": 280}, + {"type": "string", "start": 280, "end": 283}, + {"type": "punctuation", "start": 283, "end": 284}, + {"type": "string", "start": 285, "end": 290}, + {"type": "punctuation", "start": 290, "end": 291}, + {"type": "string", "start": 292, "end": 294}, + {"type": "punctuation", "start": 294, "end": 295}, + {"type": "punctuation", "start": 295, "end": 296}, + {"type": "punctuation", "start": 297, "end": 298}, + {"type": "property", "start": 298, "end": 303}, + {"type": "operator", "start": 303, "end": 304}, + {"type": "string", "start": 305, "end": 319}, + {"type": "punctuation", "start": 319, "end": 320}, + {"type": "punctuation", "start": 320, "end": 321}, + {"type": "punctuation", "start": 323, "end": 324}, + {"type": "punctuation", "start": 325, "end": 326} + ], + "html": "{\n\t\"string\": \"a string\",\n\t\"number\": 12345,\n\t\"boolean\": true,\n\t\"null\": null,\n\t\"empty\": \"\",\n\t\"escaped\": \"quote: \\\"test\\\" and backslash: \\\\\",\n\t\"object\": {\n\t\t\"array\": [1, \"b\", false],\n\t\t\"strings\": [\"1\", \"2\", \"3\"],\n\t\t\"mixed\": [\"start\", 123, true, \"middle\", null, \"end\"],\n\t\t\"nested\": [[\"a\", \"str\", \"\"], {\"key\": \"nested value\"}]\n\t}\n}\n" +} diff --git a/fixtures/generated/json/json_complex.txt b/fixtures/generated/json/json_complex.txt new file mode 100644 index 00000000..082fc2fa --- /dev/null +++ b/fixtures/generated/json/json_complex.txt @@ -0,0 +1,97 @@ +=== TOKENS === + 0-1 punctuation { + 3-11 property "string" + 11-12 operator : + 13-23 string "a string" + 23-24 punctuation , + 26-34 property "number" + 34-35 operator : + 36-41 number 12345 + 41-42 punctuation , + 44-53 property "boolean" + 53-54 operator : + 55-59 boolean true + 59-60 punctuation , + 62-68 property "null" + 68-69 operator : + 70-74 null null + 74-75 punctuation , + 77-84 property "empty" + 84-85 operator : + 86-88 string "" + 88-89 punctuation , + 91-100 property "escaped" + 100-101 operator : + 102-137 string "quote: \"test\" and backslash: \\" + 137-138 punctuation , + 140-148 property "object" + 148-149 operator : + 150-151 punctuation { + 154-161 property "array" + 161-162 operator : + 163-164 punctuation [ + 164-165 number 1 + 165-166 punctuation , + 167-170 string "b" + 170-171 punctuation , + 172-177 boolean false + 177-178 punctuation ] + 178-179 punctuation , + 182-191 property "strings" + 191-192 operator : + 193-194 punctuation [ + 194-197 string "1" + 197-198 punctuation , + 199-202 string "2" + 202-203 punctuation , + 204-207 string "3" + 207-208 punctuation ] + 208-209 punctuation , + 212-219 property "mixed" + 219-220 operator : + 221-222 punctuation [ + 222-229 string "start" + 229-230 punctuation , + 231-234 number 123 + 234-235 punctuation , + 236-240 boolean true + 240-241 punctuation , + 242-250 string "middle" + 250-251 punctuation , + 252-256 null null + 256-257 punctuation , + 258-263 string "end" + 263-264 punctuation ] + 264-265 punctuation , + 268-276 property "nested" + 276-277 operator : + 278-279 punctuation [ + 279-280 punctuation [ + 280-283 string "a" + 283-284 punctuation , + 285-290 string "str" + 290-291 punctuation , + 292-294 string "" + 294-295 punctuation ] + 295-296 punctuation , + 297-298 punctuation { + 298-303 property "key" + 303-304 operator : + 305-319 string "nested value" + 319-320 punctuation } + 320-321 punctuation ] + 323-324 punctuation } + 325-326 punctuation } + +=== STATS === +Total tokens: 83 +Sample length: 327 characters + +Token types: + punctuation: 37 + string: 14 + property: 12 + operator: 12 + number: 3 + boolean: 3 + null: 2 diff --git a/fixtures/generated/svelte/svelte_complex.json b/fixtures/generated/svelte/svelte_complex.json new file mode 100644 index 00000000..16d8e811 --- /dev/null +++ b/fixtures/generated/svelte/svelte_complex.json @@ -0,0 +1,587 @@ +{ + "sample": { + "lang": "svelte", + "variant": "complex", + "content": "\n\n\n\n

hello {HELLO}!

\n\n{#each thing_keys as key (key)}\n\t{@const value = thing[key]}\n\t{value}\n{/each}\n\n{#if c}\n\t\n{:else}\n\t (c = !c)}>\n\t\t{@render children()}\n\t\n{/if}\n\n\n\n
\n\t

hello world!

\n
\n\n

\n\tsome text\n

\n\n\n\n\n{a}\n{b}\n{bound}\n{D}\n\n
\n\n
\n\n\"access\"\n\n
    \n\t
  • list item 1
    • \n\t
  • list item 2
    • \n
\n\n\n", + "filepath": "src/lib/samples/sample_complex.svelte" + }, + "tokens": [ + {"type": "tag", "start": 0, "end": 25}, + {"type": "tag", "start": 0, "end": 7}, + {"type": "punctuation", "start": 0, "end": 1}, + {"type": "attr_name", "start": 8, "end": 12}, + {"type": "attr_value", "start": 12, "end": 17}, + {"type": "punctuation", "start": 12, "end": 13}, + {"type": "punctuation", "start": 13, "end": 14}, + {"type": "punctuation", "start": 16, "end": 17}, + {"type": "attr_name", "start": 18, "end": 24}, + {"type": "punctuation", "start": 24, "end": 25}, + {"type": "script", "start": 25, "end": 57}, + {"type": "lang_ts", "start": 25, "end": 57}, + {"type": "keyword", "start": 27, "end": 33}, + {"type": "keyword", "start": 34, "end": 39}, + {"type": "constant", "start": 40, "end": 45}, + {"type": "operator", "start": 46, "end": 47}, + {"type": "string", "start": 48, "end": 55}, + {"type": "punctuation", "start": 55, "end": 56}, + {"type": "tag", "start": 57, "end": 66}, + {"type": "tag", "start": 57, "end": 65}, + {"type": "punctuation", "start": 57, "end": 59}, + {"type": "punctuation", "start": 65, "end": 66}, + {"type": "tag", "start": 68, "end": 86}, + {"type": "tag", "start": 68, "end": 75}, + {"type": "punctuation", "start": 68, "end": 69}, + {"type": "attr_name", "start": 76, "end": 80}, + {"type": "attr_value", "start": 80, "end": 85}, + {"type": "punctuation", "start": 80, "end": 81}, + {"type": "punctuation", "start": 81, "end": 82}, + {"type": "punctuation", "start": 84, "end": 85}, + {"type": "punctuation", "start": 85, "end": 86}, + {"type": "script", "start": 86, "end": 1268}, + {"type": "lang_ts", "start": 86, "end": 1268}, + {"type": "comment", "start": 88, "end": 107}, + {"type": "keyword", "start": 109, "end": 115}, + {"type": "keyword", "start": 122, "end": 126}, + {"type": "string", "start": 127, "end": 146}, + {"type": "punctuation", "start": 146, "end": 147}, + {"type": "keyword", "start": 149, "end": 155}, + {"type": "keyword", "start": 156, "end": 160}, + {"type": "punctuation", "start": 161, "end": 162}, + {"type": "punctuation", "start": 169, "end": 170}, + {"type": "keyword", "start": 171, "end": 175}, + {"type": "string", "start": 176, "end": 184}, + {"type": "punctuation", "start": 184, "end": 185}, + {"type": "keyword", "start": 188, "end": 193}, + {"type": "punctuation", "start": 194, "end": 195}, + {"type": "punctuation", "start": 203, "end": 204}, + {"type": "operator", "start": 213, "end": 214}, + {"type": "function", "start": 215, "end": 224}, + {"type": "punctuation", "start": 224, "end": 225}, + {"type": "boolean", "start": 225, "end": 229}, + {"type": "punctuation", "start": 229, "end": 230}, + {"type": "punctuation", "start": 230, "end": 231}, + {"type": "punctuation", "start": 242, "end": 243}, + {"type": "punctuation", "start": 245, "end": 246}, + {"type": "operator", "start": 246, "end": 247}, + {"type": "punctuation", "start": 248, "end": 249}, + {"type": "operator", "start": 257, "end": 258}, + {"type": "operator", "start": 265, "end": 266}, + {"type": "builtin", "start": 266, "end": 272}, + {"type": "punctuation", "start": 272, "end": 273}, + {"type": "builtin", "start": 274, "end": 277}, + {"type": "operator", "start": 277, "end": 278}, + {"type": "punctuation", "start": 278, "end": 279}, + {"type": "operator", "start": 287, "end": 288}, + {"type": "operator", "start": 288, "end": 289}, + {"type": "builtin", "start": 290, "end": 297}, + {"type": "punctuation", "start": 297, "end": 298}, + {"type": "operator", "start": 309, "end": 310}, + {"type": "punctuation", "start": 318, "end": 319}, + {"type": "punctuation", "start": 321, "end": 322}, + {"type": "operator", "start": 323, "end": 324}, + {"type": "function", "start": 325, "end": 331}, + {"type": "punctuation", "start": 331, "end": 332}, + {"type": "punctuation", "start": 332, "end": 333}, + {"type": "punctuation", "start": 333, "end": 334}, + {"type": "keyword", "start": 337, "end": 342}, + {"type": "operator", "start": 354, "end": 355}, + {"type": "function", "start": 356, "end": 364}, + {"type": "punctuation", "start": 364, "end": 365}, + {"type": "punctuation", "start": 371, "end": 372}, + {"type": "function", "start": 372, "end": 376}, + {"type": "punctuation", "start": 376, "end": 377}, + {"type": "punctuation", "start": 382, "end": 383}, + {"type": "punctuation", "start": 383, "end": 384}, + {"type": "punctuation", "start": 384, "end": 385}, + {"type": "keyword", "start": 388, "end": 393}, + {"type": "operator", "start": 396, "end": 397}, + {"type": "number", "start": 398, "end": 399}, + {"type": "punctuation", "start": 399, "end": 400}, + {"type": "keyword", "start": 403, "end": 408}, + {"type": "operator", "start": 411, "end": 412}, + {"type": "string", "start": 413, "end": 416}, + {"type": "punctuation", "start": 416, "end": 417}, + {"type": "keyword", "start": 420, "end": 423}, + {"type": "operator", "start": 425, "end": 426}, + {"type": "builtin", "start": 427, "end": 434}, + {"type": "operator", "start": 435, "end": 436}, + {"type": "function", "start": 437, "end": 443}, + {"type": "punctuation", "start": 443, "end": 444}, + {"type": "boolean", "start": 444, "end": 448}, + {"type": "punctuation", "start": 448, "end": 449}, + {"type": "punctuation", "start": 449, "end": 450}, + {"type": "keyword", "start": 453, "end": 459}, + {"type": "keyword", "start": 460, "end": 464}, + {"type": "class_name", "start": 465, "end": 474}, + {"type": "operator", "start": 475, "end": 476}, + {"type": "number", "start": 477, "end": 478}, + {"type": "operator", "start": 479, "end": 480}, + {"type": "string", "start": 481, "end": 484}, + {"type": "operator", "start": 485, "end": 486}, + {"type": "boolean", "start": 487, "end": 491}, + {"type": "punctuation", "start": 491, "end": 492}, + {"type": "keyword", "start": 495, "end": 500}, + {"type": "class_name", "start": 501, "end": 502}, + {"type": "constant", "start": 501, "end": 502}, + {"type": "punctuation", "start": 503, "end": 504}, + {"type": "operator", "start": 509, "end": 510}, + {"type": "builtin", "start": 511, "end": 517}, + {"type": "operator", "start": 518, "end": 519}, + {"type": "string", "start": 520, "end": 523}, + {"type": "punctuation", "start": 523, "end": 524}, + {"type": "operator", "start": 529, "end": 530}, + {"type": "builtin", "start": 531, "end": 537}, + {"type": "punctuation", "start": 537, "end": 538}, + {"type": "operator", "start": 544, "end": 545}, + {"type": "function", "start": 546, "end": 552}, + {"type": "punctuation", "start": 552, "end": 553}, + {"type": "boolean", "start": 553, "end": 558}, + {"type": "punctuation", "start": 558, "end": 559}, + {"type": "punctuation", "start": 559, "end": 560}, + {"type": "function", "start": 564, "end": 575}, + {"type": "punctuation", "start": 575, "end": 576}, + {"type": "operator", "start": 578, "end": 579}, + {"type": "builtin", "start": 580, "end": 586}, + {"type": "punctuation", "start": 586, "end": 587}, + {"type": "punctuation", "start": 588, "end": 589}, + {"type": "keyword", "start": 593, "end": 597}, + {"type": "punctuation", "start": 597, "end": 598}, + {"type": "operator", "start": 601, "end": 602}, + {"type": "punctuation", "start": 605, "end": 606}, + {"type": "punctuation", "start": 609, "end": 610}, + {"type": "function", "start": 614, "end": 626}, + {"type": "punctuation", "start": 626, "end": 627}, + {"type": "punctuation", "start": 627, "end": 628}, + {"type": "operator", "start": 628, "end": 629}, + {"type": "builtin", "start": 630, "end": 636}, + {"type": "punctuation", "start": 637, "end": 638}, + {"type": "keyword", "start": 642, "end": 648}, + {"type": "template_string", "start": 649, "end": 668}, + {"type": "template_punctuation", "start": 649, "end": 650}, + {"type": "string", "start": 650, "end": 657}, + {"type": "interpolation", "start": 657, "end": 667}, + {"type": "interpolation_punctuation", "start": 657, "end": 659}, + {"type": "keyword", "start": 659, "end": 663}, + {"type": "punctuation", "start": 663, "end": 664}, + {"type": "interpolation_punctuation", "start": 666, "end": 667}, + {"type": "template_punctuation", "start": 667, "end": 668}, + {"type": "punctuation", "start": 668, "end": 669}, + {"type": "punctuation", "start": 672, "end": 673}, + {"type": "function_variable", "start": 677, "end": 692}, + {"type": "operator", "start": 693, "end": 694}, + {"type": "punctuation", "start": 695, "end": 696}, + {"type": "punctuation", "start": 696, "end": 697}, + {"type": "operator", "start": 698, "end": 700}, + {"type": "punctuation", "start": 701, "end": 702}, + {"type": "comment", "start": 706, "end": 715}, + {"type": "keyword", "start": 719, "end": 723}, + {"type": "punctuation", "start": 723, "end": 724}, + {"type": "function", "start": 724, "end": 739}, + {"type": "punctuation", "start": 739, "end": 740}, + {"type": "punctuation", "start": 740, "end": 741}, + {"type": "punctuation", "start": 741, "end": 742}, + {"type": "comment", "start": 746, "end": 752}, + {"type": "punctuation", "start": 755, "end": 756}, + {"type": "punctuation", "start": 756, "end": 757}, + {"type": "function", "start": 761, "end": 776}, + {"type": "punctuation", "start": 776, "end": 777}, + {"type": "punctuation", "start": 777, "end": 778}, + {"type": "punctuation", "start": 779, "end": 780}, + {"type": "keyword", "start": 784, "end": 789}, + {"type": "keyword", "start": 790, "end": 793}, + {"type": "class_name", "start": 794, "end": 799}, + {"type": "punctuation", "start": 799, "end": 800}, + {"type": "template_string", "start": 800, "end": 816}, + {"type": "template_punctuation", "start": 800, "end": 801}, + {"type": "interpolation", "start": 801, "end": 811}, + {"type": "interpolation_punctuation", "start": 801, "end": 803}, + {"type": "keyword", "start": 803, "end": 807}, + {"type": "punctuation", "start": 807, "end": 808}, + {"type": "interpolation_punctuation", "start": 810, "end": 811}, + {"type": "string", "start": 811, "end": 815}, + {"type": "template_punctuation", "start": 815, "end": 816}, + {"type": "punctuation", "start": 816, "end": 817}, + {"type": "punctuation", "start": 817, "end": 818}, + {"type": "punctuation", "start": 821, "end": 822}, + {"type": "keyword", "start": 826, "end": 835}, + {"type": "function", "start": 836, "end": 852}, + {"type": "punctuation", "start": 852, "end": 853}, + {"type": "punctuation", "start": 853, "end": 854}, + {"type": "punctuation", "start": 855, "end": 856}, + {"type": "builtin", "start": 860, "end": 867}, + {"type": "punctuation", "start": 867, "end": 868}, + {"type": "function", "start": 868, "end": 871}, + {"type": "punctuation", "start": 871, "end": 872}, + {"type": "keyword", "start": 872, "end": 875}, + {"type": "class_name", "start": 876, "end": 880}, + {"type": "punctuation", "start": 880, "end": 881}, + {"type": "number", "start": 881, "end": 884}, + {"type": "punctuation", "start": 884, "end": 885}, + {"type": "punctuation", "start": 885, "end": 886}, + {"type": "punctuation", "start": 886, "end": 887}, + {"type": "comment", "start": 888, "end": 921}, + {"type": "punctuation", "start": 924, "end": 925}, + {"type": "punctuation", "start": 927, "end": 928}, + {"type": "comment", "start": 931, "end": 941}, + {"type": "comment", "start": 944, "end": 990}, + {"type": "comment", "start": 993, "end": 1019}, + {"type": "keyword", "start": 1022, "end": 1028}, + {"type": "keyword", "start": 1029, "end": 1038}, + {"type": "class_name", "start": 1039, "end": 1045}, + {"type": "punctuation", "start": 1046, "end": 1047}, + {"type": "operator", "start": 1054, "end": 1055}, + {"type": "builtin", "start": 1056, "end": 1062}, + {"type": "punctuation", "start": 1062, "end": 1063}, + {"type": "operator", "start": 1069, "end": 1070}, + {"type": "builtin", "start": 1071, "end": 1077}, + {"type": "punctuation", "start": 1077, "end": 1078}, + {"type": "punctuation", "start": 1080, "end": 1081}, + {"type": "keyword", "start": 1084, "end": 1090}, + {"type": "keyword", "start": 1091, "end": 1096}, + {"type": "operator", "start": 1103, "end": 1104}, + {"type": "operator", "start": 1112, "end": 1113}, + {"type": "punctuation", "start": 1114, "end": 1115}, + {"type": "operator", "start": 1119, "end": 1120}, + {"type": "string", "start": 1121, "end": 1128}, + {"type": "punctuation", "start": 1128, "end": 1129}, + {"type": "operator", "start": 1133, "end": 1134}, + {"type": "number", "start": 1135, "end": 1138}, + {"type": "punctuation", "start": 1138, "end": 1139}, + {"type": "punctuation", "start": 1139, "end": 1140}, + {"type": "keyword", "start": 1143, "end": 1149}, + {"type": "keyword", "start": 1150, "end": 1158}, + {"type": "function", "start": 1159, "end": 1162}, + {"type": "punctuation", "start": 1162, "end": 1163}, + {"type": "operator", "start": 1164, "end": 1165}, + {"type": "builtin", "start": 1166, "end": 1172}, + {"type": "punctuation", "start": 1172, "end": 1173}, + {"type": "operator", "start": 1175, "end": 1176}, + {"type": "builtin", "start": 1177, "end": 1183}, + {"type": "punctuation", "start": 1183, "end": 1184}, + {"type": "operator", "start": 1184, "end": 1185}, + {"type": "builtin", "start": 1186, "end": 1192}, + {"type": "punctuation", "start": 1193, "end": 1194}, + {"type": "keyword", "start": 1197, "end": 1203}, + {"type": "operator", "start": 1206, "end": 1207}, + {"type": "punctuation", "start": 1209, "end": 1210}, + {"type": "punctuation", "start": 1212, "end": 1213}, + {"type": "keyword", "start": 1216, "end": 1222}, + {"type": "keyword", "start": 1223, "end": 1228}, + {"type": "operator", "start": 1234, "end": 1235}, + {"type": "punctuation", "start": 1236, "end": 1237}, + {"type": "operator", "start": 1238, "end": 1239}, + {"type": "builtin", "start": 1240, "end": 1243}, + {"type": "punctuation", "start": 1243, "end": 1244}, + {"type": "operator", "start": 1246, "end": 1247}, + {"type": "builtin", "start": 1248, "end": 1251}, + {"type": "punctuation", "start": 1251, "end": 1252}, + {"type": "operator", "start": 1252, "end": 1253}, + {"type": "builtin", "start": 1254, "end": 1257}, + {"type": "operator", "start": 1258, "end": 1260}, + {"type": "operator", "start": 1263, "end": 1264}, + {"type": "punctuation", "start": 1266, "end": 1267}, + {"type": "tag", "start": 1268, "end": 1277}, + {"type": "tag", "start": 1268, "end": 1276}, + {"type": "punctuation", "start": 1268, "end": 1270}, + {"type": "punctuation", "start": 1276, "end": 1277}, + {"type": "tag", "start": 1279, "end": 1283}, + {"type": "tag", "start": 1279, "end": 1282}, + {"type": "punctuation", "start": 1279, "end": 1280}, + {"type": "punctuation", "start": 1282, "end": 1283}, + {"type": "lang_ts", "start": 1289, "end": 1296}, + {"type": "punctuation", "start": 1289, "end": 1290}, + {"type": "constant", "start": 1290, "end": 1295}, + {"type": "punctuation", "start": 1295, "end": 1296}, + {"type": "tag", "start": 1297, "end": 1302}, + {"type": "tag", "start": 1297, "end": 1301}, + {"type": "punctuation", "start": 1297, "end": 1299}, + {"type": "punctuation", "start": 1301, "end": 1302}, + {"type": "each", "start": 1304, "end": 1335}, + {"type": "punctuation", "start": 1304, "end": 1305}, + {"type": "keyword", "start": 1305, "end": 1310}, + {"type": "lang_ts", "start": 1311, "end": 1322}, + {"type": "keyword", "start": 1322, "end": 1324}, + {"type": "lang_ts", "start": 1325, "end": 1329}, + {"type": "lang_ts", "start": 1329, "end": 1334}, + {"type": "punctuation", "start": 1329, "end": 1330}, + {"type": "punctuation", "start": 1333, "end": 1334}, + {"type": "punctuation", "start": 1334, "end": 1335}, + {"type": "lang_ts", "start": 1337, "end": 1364}, + {"type": "punctuation", "start": 1337, "end": 1338}, + {"type": "keyword", "start": 1339, "end": 1344}, + {"type": "operator", "start": 1351, "end": 1352}, + {"type": "punctuation", "start": 1358, "end": 1359}, + {"type": "punctuation", "start": 1362, "end": 1363}, + {"type": "punctuation", "start": 1363, "end": 1364}, + {"type": "lang_ts", "start": 1366, "end": 1373}, + {"type": "punctuation", "start": 1366, "end": 1367}, + {"type": "punctuation", "start": 1372, "end": 1373}, + {"type": "each", "start": 1374, "end": 1381}, + {"type": "punctuation", "start": 1374, "end": 1375}, + {"type": "keyword", "start": 1375, "end": 1380}, + {"type": "punctuation", "start": 1380, "end": 1381}, + {"type": "lang_ts", "start": 1383, "end": 1390}, + {"type": "punctuation", "start": 1383, "end": 1384}, + {"type": "keyword", "start": 1385, "end": 1387}, + {"type": "punctuation", "start": 1389, "end": 1390}, + {"type": "tag", "start": 1392, "end": 1433}, + {"type": "tag", "start": 1392, "end": 1398}, + {"type": "punctuation", "start": 1392, "end": 1393}, + {"type": "attr_name", "start": 1399, "end": 1410}, + {"type": "attr_value", "start": 1410, "end": 1414}, + {"type": "punctuation", "start": 1410, "end": 1411}, + {"type": "punctuation", "start": 1411, "end": 1412}, + {"type": "punctuation", "start": 1413, "end": 1414}, + {"type": "attr_name", "start": 1415, "end": 1427}, + {"type": "lang_ts", "start": 1427, "end": 1430}, + {"type": "punctuation", "start": 1427, "end": 1428}, + {"type": "number", "start": 1428, "end": 1429}, + {"type": "punctuation", "start": 1429, "end": 1430}, + {"type": "punctuation", "start": 1431, "end": 1433}, + {"type": "lang_ts", "start": 1434, "end": 1441}, + {"type": "punctuation", "start": 1434, "end": 1435}, + {"type": "operator", "start": 1435, "end": 1436}, + {"type": "keyword", "start": 1436, "end": 1440}, + {"type": "punctuation", "start": 1440, "end": 1441}, + {"type": "tag", "start": 1443, "end": 1507}, + {"type": "tag", "start": 1443, "end": 1449}, + {"type": "punctuation", "start": 1443, "end": 1444}, + {"type": "attr_name", "start": 1450, "end": 1461}, + {"type": "attr_value", "start": 1461, "end": 1465}, + {"type": "punctuation", "start": 1461, "end": 1462}, + {"type": "punctuation", "start": 1462, "end": 1463}, + {"type": "punctuation", "start": 1464, "end": 1465}, + {"type": "attr_name", "start": 1466, "end": 1478}, + {"type": "lang_ts", "start": 1478, "end": 1481}, + {"type": "punctuation", "start": 1478, "end": 1479}, + {"type": "number", "start": 1479, "end": 1480}, + {"type": "punctuation", "start": 1480, "end": 1481}, + {"type": "attr_name", "start": 1482, "end": 1490}, + {"type": "lang_ts", "start": 1490, "end": 1506}, + {"type": "punctuation", "start": 1490, "end": 1491}, + {"type": "punctuation", "start": 1491, "end": 1492}, + {"type": "punctuation", "start": 1492, "end": 1493}, + {"type": "operator", "start": 1494, "end": 1496}, + {"type": "punctuation", "start": 1497, "end": 1498}, + {"type": "operator", "start": 1500, "end": 1501}, + {"type": "operator", "start": 1502, "end": 1503}, + {"type": "punctuation", "start": 1504, "end": 1505}, + {"type": "punctuation", "start": 1505, "end": 1506}, + {"type": "punctuation", "start": 1506, "end": 1507}, + {"type": "lang_ts", "start": 1510, "end": 1530}, + {"type": "punctuation", "start": 1510, "end": 1511}, + {"type": "decorator", "start": 1511, "end": 1518}, + {"type": "at", "start": 1511, "end": 1512}, + {"type": "function", "start": 1512, "end": 1518}, + {"type": "function", "start": 1519, "end": 1527}, + {"type": "punctuation", "start": 1527, "end": 1528}, + {"type": "punctuation", "start": 1528, "end": 1529}, + {"type": "punctuation", "start": 1529, "end": 1530}, + {"type": "tag", "start": 1532, "end": 1540}, + {"type": "tag", "start": 1532, "end": 1539}, + {"type": "punctuation", "start": 1532, "end": 1534}, + {"type": "punctuation", "start": 1539, "end": 1540}, + {"type": "lang_ts", "start": 1541, "end": 1546}, + {"type": "punctuation", "start": 1541, "end": 1542}, + {"type": "operator", "start": 1542, "end": 1543}, + {"type": "keyword", "start": 1543, "end": 1545}, + {"type": "punctuation", "start": 1545, "end": 1546}, + {"type": "doctype", "start": 1548, "end": 1563}, + {"type": "tag", "start": 1565, "end": 1606}, + {"type": "tag", "start": 1565, "end": 1569}, + {"type": "punctuation", "start": 1565, "end": 1566}, + {"type": "attr_name", "start": 1570, "end": 1575}, + {"type": "attr_value", "start": 1575, "end": 1590}, + {"type": "punctuation", "start": 1575, "end": 1576}, + {"type": "punctuation", "start": 1576, "end": 1577}, + {"type": "punctuation", "start": 1589, "end": 1590}, + {"type": "attr_name", "start": 1591, "end": 1593}, + {"type": "attr_value", "start": 1593, "end": 1605}, + {"type": "punctuation", "start": 1593, "end": 1594}, + {"type": "punctuation", "start": 1594, "end": 1595}, + {"type": "punctuation", "start": 1604, "end": 1605}, + {"type": "punctuation", "start": 1605, "end": 1606}, + {"type": "tag", "start": 1608, "end": 1611}, + {"type": "tag", "start": 1608, "end": 1610}, + {"type": "punctuation", "start": 1608, "end": 1609}, + {"type": "punctuation", "start": 1610, "end": 1611}, + {"type": "tag", "start": 1623, "end": 1627}, + {"type": "tag", "start": 1623, "end": 1626}, + {"type": "punctuation", "start": 1623, "end": 1625}, + {"type": "punctuation", "start": 1626, "end": 1627}, + {"type": "tag", "start": 1628, "end": 1634}, + {"type": "tag", "start": 1628, "end": 1633}, + {"type": "punctuation", "start": 1628, "end": 1630}, + {"type": "punctuation", "start": 1633, "end": 1634}, + {"type": "tag", "start": 1636, "end": 1670}, + {"type": "tag", "start": 1636, "end": 1638}, + {"type": "punctuation", "start": 1636, "end": 1637}, + {"type": "attr_name", "start": 1639, "end": 1644}, + {"type": "attr_value", "start": 1644, "end": 1669}, + {"type": "punctuation", "start": 1644, "end": 1645}, + {"type": "punctuation", "start": 1645, "end": 1646}, + {"type": "punctuation", "start": 1668, "end": 1669}, + {"type": "punctuation", "start": 1669, "end": 1670}, + {"type": "tag", "start": 1677, "end": 1697}, + {"type": "tag", "start": 1677, "end": 1682}, + {"type": "punctuation", "start": 1677, "end": 1678}, + {"type": "attr_name", "start": 1683, "end": 1688}, + {"type": "attr_value", "start": 1688, "end": 1696}, + {"type": "punctuation", "start": 1688, "end": 1689}, + {"type": "punctuation", "start": 1689, "end": 1690}, + {"type": "punctuation", "start": 1695, "end": 1696}, + {"type": "punctuation", "start": 1696, "end": 1697}, + {"type": "tag", "start": 1701, "end": 1708}, + {"type": "tag", "start": 1701, "end": 1707}, + {"type": "punctuation", "start": 1701, "end": 1703}, + {"type": "punctuation", "start": 1707, "end": 1708}, + {"type": "tag", "start": 1709, "end": 1713}, + {"type": "tag", "start": 1709, "end": 1712}, + {"type": "punctuation", "start": 1709, "end": 1711}, + {"type": "punctuation", "start": 1712, "end": 1713}, + {"type": "tag", "start": 1715, "end": 1746}, + {"type": "tag", "start": 1715, "end": 1722}, + {"type": "punctuation", "start": 1715, "end": 1716}, + {"type": "attr_name", "start": 1723, "end": 1727}, + {"type": "attr_value", "start": 1727, "end": 1736}, + {"type": "punctuation", "start": 1727, "end": 1728}, + {"type": "punctuation", "start": 1728, "end": 1729}, + {"type": "punctuation", "start": 1735, "end": 1736}, + {"type": "attr_name", "start": 1737, "end": 1745}, + {"type": "punctuation", "start": 1745, "end": 1746}, + {"type": "tag", "start": 1756, "end": 1765}, + {"type": "tag", "start": 1756, "end": 1764}, + {"type": "punctuation", "start": 1756, "end": 1758}, + {"type": "punctuation", "start": 1764, "end": 1765}, + {"type": "comment", "start": 1767, "end": 1804}, + {"type": "lang_ts", "start": 1805, "end": 1808}, + {"type": "punctuation", "start": 1805, "end": 1806}, + {"type": "punctuation", "start": 1807, "end": 1808}, + {"type": "lang_ts", "start": 1809, "end": 1812}, + {"type": "punctuation", "start": 1809, "end": 1810}, + {"type": "punctuation", "start": 1811, "end": 1812}, + {"type": "lang_ts", "start": 1813, "end": 1820}, + {"type": "punctuation", "start": 1813, "end": 1814}, + {"type": "punctuation", "start": 1819, "end": 1820}, + {"type": "lang_ts", "start": 1821, "end": 1824}, + {"type": "punctuation", "start": 1821, "end": 1822}, + {"type": "constant", "start": 1822, "end": 1823}, + {"type": "punctuation", "start": 1823, "end": 1824}, + {"type": "tag", "start": 1826, "end": 1832}, + {"type": "tag", "start": 1826, "end": 1829}, + {"type": "punctuation", "start": 1826, "end": 1827}, + {"type": "punctuation", "start": 1830, "end": 1832}, + {"type": "tag", "start": 1834, "end": 1840}, + {"type": "tag", "start": 1834, "end": 1837}, + {"type": "punctuation", "start": 1834, "end": 1835}, + {"type": "punctuation", "start": 1838, "end": 1840}, + {"type": "tag", "start": 1842, "end": 1878}, + {"type": "tag", "start": 1842, "end": 1846}, + {"type": "punctuation", "start": 1842, "end": 1843}, + {"type": "attr_name", "start": 1847, "end": 1850}, + {"type": "attr_value", "start": 1850, "end": 1862}, + {"type": "punctuation", "start": 1850, "end": 1851}, + {"type": "punctuation", "start": 1851, "end": 1852}, + {"type": "punctuation", "start": 1861, "end": 1862}, + {"type": "attr_name", "start": 1863, "end": 1866}, + {"type": "attr_value", "start": 1866, "end": 1875}, + {"type": "punctuation", "start": 1866, "end": 1867}, + {"type": "punctuation", "start": 1867, "end": 1868}, + {"type": "punctuation", "start": 1874, "end": 1875}, + {"type": "punctuation", "start": 1876, "end": 1878}, + {"type": "tag", "start": 1880, "end": 1884}, + {"type": "tag", "start": 1880, "end": 1883}, + {"type": "punctuation", "start": 1880, "end": 1881}, + {"type": "punctuation", "start": 1883, "end": 1884}, + {"type": "tag", "start": 1886, "end": 1890}, + {"type": "tag", "start": 1886, "end": 1889}, + {"type": "punctuation", "start": 1886, "end": 1887}, + {"type": "punctuation", "start": 1889, "end": 1890}, + {"type": "tag", "start": 1901, "end": 1906}, + {"type": "tag", "start": 1901, "end": 1905}, + {"type": "punctuation", "start": 1901, "end": 1903}, + {"type": "punctuation", "start": 1905, "end": 1906}, + {"type": "tag", "start": 1908, "end": 1912}, + {"type": "tag", "start": 1908, "end": 1911}, + {"type": "punctuation", "start": 1908, "end": 1909}, + {"type": "punctuation", "start": 1911, "end": 1912}, + {"type": "tag", "start": 1923, "end": 1928}, + {"type": "tag", "start": 1923, "end": 1927}, + {"type": "punctuation", "start": 1923, "end": 1925}, + {"type": "punctuation", "start": 1927, "end": 1928}, + {"type": "tag", "start": 1929, "end": 1934}, + {"type": "tag", "start": 1929, "end": 1933}, + {"type": "punctuation", "start": 1929, "end": 1931}, + {"type": "punctuation", "start": 1933, "end": 1934}, + {"type": "tag", "start": 1936, "end": 1943}, + {"type": "tag", "start": 1936, "end": 1942}, + {"type": "punctuation", "start": 1936, "end": 1937}, + {"type": "punctuation", "start": 1942, "end": 1943}, + {"type": "style", "start": 1943, "end": 2324}, + {"type": "lang_css", "start": 1943, "end": 2324}, + {"type": "selector", "start": 1945, "end": 1956}, + {"type": "punctuation", "start": 1957, "end": 1958}, + {"type": "property", "start": 1961, "end": 1966}, + {"type": "punctuation", "start": 1966, "end": 1967}, + {"type": "punctuation", "start": 1971, "end": 1972}, + {"type": "punctuation", "start": 1974, "end": 1975}, + {"type": "selector", "start": 1978, "end": 1990}, + {"type": "punctuation", "start": 1991, "end": 1992}, + {"type": "property", "start": 1995, "end": 2004}, + {"type": "punctuation", "start": 2004, "end": 2005}, + {"type": "punctuation", "start": 2010, "end": 2011}, + {"type": "punctuation", "start": 2013, "end": 2014}, + {"type": "selector", "start": 2017, "end": 2018}, + {"type": "punctuation", "start": 2019, "end": 2020}, + {"type": "property", "start": 2023, "end": 2033}, + {"type": "punctuation", "start": 2033, "end": 2034}, + {"type": "function", "start": 2044, "end": 2048}, + {"type": "punctuation", "start": 2048, "end": 2049}, + {"type": "punctuation", "start": 2050, "end": 2051}, + {"type": "punctuation", "start": 2053, "end": 2054}, + {"type": "punctuation", "start": 2056, "end": 2057}, + {"type": "punctuation", "start": 2061, "end": 2062}, + {"type": "punctuation", "start": 2062, "end": 2063}, + {"type": "punctuation", "start": 2065, "end": 2066}, + {"type": "comment", "start": 2069, "end": 2082}, + {"type": "comment", "start": 2085, "end": 2117}, + {"type": "selector", "start": 2120, "end": 2130}, + {"type": "punctuation", "start": 2131, "end": 2132}, + {"type": "property", "start": 2135, "end": 2151}, + {"type": "punctuation", "start": 2151, "end": 2152}, + {"type": "punctuation", "start": 2157, "end": 2158}, + {"type": "punctuation", "start": 2160, "end": 2161}, + {"type": "selector", "start": 2164, "end": 2171}, + {"type": "punctuation", "start": 2172, "end": 2173}, + {"type": "property", "start": 2176, "end": 2182}, + {"type": "punctuation", "start": 2182, "end": 2183}, + {"type": "punctuation", "start": 2188, "end": 2189}, + {"type": "punctuation", "start": 2191, "end": 2192}, + {"type": "atrule", "start": 2195, "end": 2220}, + {"type": "rule", "start": 2195, "end": 2201}, + {"type": "punctuation", "start": 2202, "end": 2203}, + {"type": "property", "start": 2203, "end": 2212}, + {"type": "punctuation", "start": 2212, "end": 2213}, + {"type": "punctuation", "start": 2219, "end": 2220}, + {"type": "punctuation", "start": 2221, "end": 2222}, + {"type": "selector", "start": 2225, "end": 2238}, + {"type": "punctuation", "start": 2239, "end": 2240}, + {"type": "property", "start": 2244, "end": 2260}, + {"type": "punctuation", "start": 2260, "end": 2261}, + {"type": "punctuation", "start": 2271, "end": 2272}, + {"type": "punctuation", "start": 2275, "end": 2276}, + {"type": "punctuation", "start": 2278, "end": 2279}, + {"type": "selector", "start": 2282, "end": 2298}, + {"type": "punctuation", "start": 2299, "end": 2300}, + {"type": "property", "start": 2303, "end": 2310}, + {"type": "punctuation", "start": 2310, "end": 2311}, + {"type": "string", "start": 2312, "end": 2319}, + {"type": "punctuation", "start": 2319, "end": 2320}, + {"type": "punctuation", "start": 2322, "end": 2323}, + {"type": "tag", "start": 2324, "end": 2332}, + {"type": "tag", "start": 2324, "end": 2331}, + {"type": "punctuation", "start": 2324, "end": 2326}, + {"type": "punctuation", "start": 2331, "end": 2332} + ], + "html": "<script lang=\"ts\" module>\n\texport const HELLO = 'world';\n</script>\n\n<script lang=\"ts\">\n\t// @ts-expect-error\n\timport Thing from '$lib/Thing.svelte';\n\timport type {Snippet} from 'svelte';\n\n\tconst {\n\t\tthing,\n\t\tbound = $bindable(true),\n\t\tchildren,\n\t}: {\n\t\tthing: Record<string, any>;\n\t\tbound?: boolean;\n\t\tchildren: Snippet;\n\t} = $props();\n\n\tconst thing_keys = $derived(Object.keys(thing));\n\n\tconst a = 1;\n\n\tconst b = 'b';\n\n\tlet c: boolean = $state(true);\n\n\texport type Some_Type = 1 | 'b' | true;\n\n\tclass D {\n\t\td1: string = 'd';\n\t\td2: number;\n\t\td3 = $state(false);\n\n\t\tconstructor(d2: number) {\n\t\t\tthis.d2 = d2;\n\t\t}\n\n\t\tclass_method(): string {\n\t\t\treturn `Hello, ${this.d1}`;\n\t\t}\n\n\t\tinstance_method = () => {\n\t\t\t/* ... */\n\t\t\tthis.#private_method();\n\t\t\t// foo\n\t\t};\n\n\t\t#private_method() {\n\t\t\tthrow new Error(`${this.d1} etc`);\n\t\t}\n\n\t\tprotected protected_method() {\n\t\t\tconsole.log(new Date(123)); // eslint-disable-line no-console\n\t\t}\n\t}\n\n\t// comment\n\n\t/*\n\tother comment\n\n\tconst comment = false;\n\t*/\n\n\t/**\n\t * JSDoc comment\n\t */\n\n\texport interface Some_E {\n\t\tname: string;\n\t\tage: number;\n\t}\n\n\texport const some_e: Some_E = {name: 'A. H.', age: 100};\n\n\texport function add(x: number, y: number): number {\n\t\treturn x + y;\n\t}\n\n\texport const plus = (a: any, b: any): any => a + b;\n</script>\n\n<h1>hello {HELLO}!</h1>\n\n{#each thing_keys as key (key)}\n\t{@const value = thing[key]}\n\t{value}\n{/each}\n\n{#if c}\n\t<Thing string_prop=\"a\" number_prop={1} />\n{:else}\n\t<Thing string_prop=\"b\" number_prop={2} onthing={() => (c = !c)}>\n\t\t{@render children()}\n\t</Thing>\n{/if}\n\n<!DOCTYPE html>\n\n<div class=\"test special\" id=\"unique_id\">\n\t<p>hello world!</p>\n</div>\n\n<p class=\"some_class hypen-class\">\n\tsome <span class=\"a b c\">text</span>\n</p>\n\n<button type=\"button\" disabled> click me </button>\n\n<!-- comment <div>a<br /> b</div> -->\n{a}\n{b}\n{bound}\n{D}\n\n<br />\n\n<hr />\n\n<img src=\"image.jpg\" alt=\"access\" />\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<style>\n\t.some_class {\n\t\tcolor: red;\n\t}\n\n\t.hypen-class {\n\t\tfont-size: 16px;\n\t}\n\n\tp {\n\t\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n\t}\n\n\t/* comment */\n\n\t/*\n\tmulti\n\tline\n\n\t<comment>\n\n\t*/\n\n\t#unique_id {\n\t\tbackground-color: blue;\n\t}\n\n\tdiv > p {\n\t\tmargin: 10px;\n\t}\n\n\t@media (max-width: 600px) {\n\t\t:global(body) {\n\t\t\tbackground-color: lightblue;\n\t\t}\n\t}\n\n\t.special::before {\n\t\tcontent: '< & >';\n\t}\n</style>\n" +} diff --git a/fixtures/generated/svelte/svelte_complex.txt b/fixtures/generated/svelte/svelte_complex.txt new file mode 100644 index 00000000..354d0065 --- /dev/null +++ b/fixtures/generated/svelte/svelte_complex.txt @@ -0,0 +1,614 @@ +=== TOKENS === + 0-25 tag + 57-65 tag + 68-86 tag +1268-1276 tag +1279-1283 tag

+1279-1282 tag

+1289-1296 lang_ts {HELLO} +1289-1290 punctuation { +1290-1295 constant HELLO +1295-1296 punctuation } +1297-1302 tag

+1297-1301 tag +1304-1335 each {#each thing_keys as key (key)} +1304-1305 punctuation { +1305-1310 keyword #each +1311-1322 lang_ts thing_keys +1322-1324 keyword as +1325-1329 lang_ts key +1329-1334 lang_ts (key) +1329-1330 punctuation ( +1333-1334 punctuation ) +1334-1335 punctuation } +1337-1364 lang_ts {@const value = thing[key]} +1337-1338 punctuation { +1339-1344 keyword const +1351-1352 operator = +1358-1359 punctuation [ +1362-1363 punctuation ] +1363-1364 punctuation } +1366-1373 lang_ts {value} +1366-1367 punctuation { +1372-1373 punctuation } +1374-1381 each {/each} +1374-1375 punctuation { +1375-1380 keyword /each +1380-1381 punctuation } +1383-1390 lang_ts {#if c} +1383-1384 punctuation { +1385-1387 keyword if +1389-1390 punctuation } +1392-1433 tag +1392-1398 tag +1434-1441 lang_ts {:else} +1434-1435 punctuation { +1435-1436 operator : +1436-1440 keyword else +1440-1441 punctuation } +1443-1507 tag (c = !c)}> +1443-1449 tag (c = !c)} +1490-1491 punctuation { +1491-1492 punctuation ( +1492-1493 punctuation ) +1494-1496 operator => +1497-1498 punctuation ( +1500-1501 operator = +1502-1503 operator ! +1504-1505 punctuation ) +1505-1506 punctuation } +1506-1507 punctuation > +1510-1530 lang_ts {@render children()} +1510-1511 punctuation { +1511-1518 decorator @render +1511-1512 at @ +1512-1518 function render +1519-1527 function children +1527-1528 punctuation ( +1528-1529 punctuation ) +1529-1530 punctuation } +1532-1540 tag +1532-1539 tag +1541-1546 lang_ts {/if} +1541-1542 punctuation { +1542-1543 operator / +1543-1545 keyword if +1545-1546 punctuation } +1548-1563 doctype +1565-1606 tag
+1565-1569 tag
+1608-1611 tag

+1608-1610 tag

+1623-1627 tag

+1623-1626 tag

+1628-1634 tag
+1628-1633 tag
+1636-1670 tag

+1636-1638 tag

+1677-1697 tag +1677-1682 tag +1701-1708 tag +1701-1707 tag +1709-1713 tag

+1709-1712 tag

+1715-1746 tag +1756-1764 tag +1767-1804 comment +1805-1808 lang_ts {a} +1805-1806 punctuation { +1807-1808 punctuation } +1809-1812 lang_ts {b} +1809-1810 punctuation { +1811-1812 punctuation } +1813-1820 lang_ts {bound} +1813-1814 punctuation { +1819-1820 punctuation } +1821-1824 lang_ts {D} +1821-1822 punctuation { +1822-1823 constant D +1823-1824 punctuation } +1826-1832 tag
+1826-1829 tag
+1834-1840 tag
+1834-1837 tag
+1842-1878 tag access +1842-1846 tag +1880-1884 tag
    +1880-1883 tag
      +1886-1890 tag
    • +1886-1889 tag
    • +1901-1906 tag
      • +1901-1905 tag +1908-1912 tag
    • +1908-1911 tag
    • +1923-1928 tag
      • +1923-1927 tag +1929-1934 tag
    +1929-1933 tag
+1936-1943 tag +2324-2331 tag + +=== STATS === +Total tokens: 576 +Sample length: 2333 characters + +Token types: + punctuation: 270 + tag: 60 + operator: 49 + keyword: 40 + lang_ts: 19 + builtin: 17 + attr_name: 16 + function: 16 + attr_value: 11 + string: 10 + comment: 10 + property: 8 + selector: 7 + number: 6 + class_name: 5 + constant: 4 + boolean: 4 + template_punctuation: 4 + interpolation_punctuation: 4 + script: 2 + template_string: 2 + interpolation: 2 + each: 2 + function_variable: 1 + decorator: 1 + at: 1 + doctype: 1 + style: 1 + lang_css: 1 + atrule: 1 + rule: 1 diff --git a/fixtures/generated/ts/ts_complex.json b/fixtures/generated/ts/ts_complex.json new file mode 100644 index 00000000..e3b567ab --- /dev/null +++ b/fixtures/generated/ts/ts_complex.json @@ -0,0 +1,248 @@ +{ + "sample": { + "lang": "ts", + "variant": "complex", + "content": "const a = 1;\n\nconst b = 'b';\n\nconst c = true;\n\nexport type Some_Type = 1 | 'b' | true;\n\nclass D {\n\td1: string = 'd';\n\td2: number;\n\td3 = $state(false);\n\n\tconstructor(d2: number) {\n\t\tthis.d2 = d2;\n\t}\n\n\tclass_method(): string {\n\t\treturn `Hello, ${this.d1}`;\n\t}\n\n\tinstance_method = (): void => {\n\t\t/* ... */\n\t\tthis.#private_method();\n\t\t// foo\n\t};\n\n\t#private_method() {\n\t\tthrow new Error(`${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t`);\n\t}\n\n\tprotected protected_method(): void {\n\t\tconsole.log(new Date()); // eslint-disable-line no-console\n\t}\n}\n\nexport {a, b, c, D};\n\n// comment\n\n/*\nother comment\n\nconst comment = false;\n*/\n\n/**\n * JSDoc comment\n */\n\nexport interface Some_E {\n\tname: string;\n\tage: number;\n}\n\nexport const some_e: Some_E = {name: 'A. H.', age: 100};\n\nexport function add(x: number, y: number): number {\n\treturn x + y;\n}\n\nexport const plus = (a: any, b: any): any => a + b;\n\n// boundary test cases\nexport const str_with_keywords = 'const class function string';\nexport const str_with_comment = '// this is not a comment';\nexport const template_with_expr = `Value: ${1 + 2}`;\n\n// regex that looks like comment\nexport const regex = /\\/\\/.*/g;\nexport const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;\n\n// string in comment should not be highlighted as string\n// const commented = \"this string is in a comment\";\n", + "filepath": "src/lib/samples/sample_complex.ts" + }, + "tokens": [ + {"type": "keyword", "start": 0, "end": 5}, + {"type": "operator", "start": 8, "end": 9}, + {"type": "number", "start": 10, "end": 11}, + {"type": "punctuation", "start": 11, "end": 12}, + {"type": "keyword", "start": 14, "end": 19}, + {"type": "operator", "start": 22, "end": 23}, + {"type": "string", "start": 24, "end": 27}, + {"type": "punctuation", "start": 27, "end": 28}, + {"type": "keyword", "start": 30, "end": 35}, + {"type": "operator", "start": 38, "end": 39}, + {"type": "boolean", "start": 40, "end": 44}, + {"type": "punctuation", "start": 44, "end": 45}, + {"type": "keyword", "start": 47, "end": 53}, + {"type": "keyword", "start": 54, "end": 58}, + {"type": "class_name", "start": 59, "end": 68}, + {"type": "operator", "start": 69, "end": 70}, + {"type": "number", "start": 71, "end": 72}, + {"type": "operator", "start": 73, "end": 74}, + {"type": "string", "start": 75, "end": 78}, + {"type": "operator", "start": 79, "end": 80}, + {"type": "boolean", "start": 81, "end": 85}, + {"type": "punctuation", "start": 85, "end": 86}, + {"type": "keyword", "start": 88, "end": 93}, + {"type": "class_name", "start": 94, "end": 95}, + {"type": "constant", "start": 94, "end": 95}, + {"type": "punctuation", "start": 96, "end": 97}, + {"type": "operator", "start": 101, "end": 102}, + {"type": "builtin", "start": 103, "end": 109}, + {"type": "operator", "start": 110, "end": 111}, + {"type": "string", "start": 112, "end": 115}, + {"type": "punctuation", "start": 115, "end": 116}, + {"type": "operator", "start": 120, "end": 121}, + {"type": "builtin", "start": 122, "end": 128}, + {"type": "punctuation", "start": 128, "end": 129}, + {"type": "operator", "start": 134, "end": 135}, + {"type": "function", "start": 136, "end": 142}, + {"type": "punctuation", "start": 142, "end": 143}, + {"type": "boolean", "start": 143, "end": 148}, + {"type": "punctuation", "start": 148, "end": 149}, + {"type": "punctuation", "start": 149, "end": 150}, + {"type": "function", "start": 153, "end": 164}, + {"type": "punctuation", "start": 164, "end": 165}, + {"type": "operator", "start": 167, "end": 168}, + {"type": "builtin", "start": 169, "end": 175}, + {"type": "punctuation", "start": 175, "end": 176}, + {"type": "punctuation", "start": 177, "end": 178}, + {"type": "keyword", "start": 181, "end": 185}, + {"type": "punctuation", "start": 185, "end": 186}, + {"type": "operator", "start": 189, "end": 190}, + {"type": "punctuation", "start": 193, "end": 194}, + {"type": "punctuation", "start": 196, "end": 197}, + {"type": "function", "start": 200, "end": 212}, + {"type": "punctuation", "start": 212, "end": 213}, + {"type": "punctuation", "start": 213, "end": 214}, + {"type": "operator", "start": 214, "end": 215}, + {"type": "builtin", "start": 216, "end": 222}, + {"type": "punctuation", "start": 223, "end": 224}, + {"type": "keyword", "start": 227, "end": 233}, + {"type": "template_string", "start": 234, "end": 253}, + {"type": "template_punctuation", "start": 234, "end": 235}, + {"type": "string", "start": 235, "end": 242}, + {"type": "interpolation", "start": 242, "end": 252}, + {"type": "interpolation_punctuation", "start": 242, "end": 244}, + {"type": "keyword", "start": 244, "end": 248}, + {"type": "punctuation", "start": 248, "end": 249}, + {"type": "interpolation_punctuation", "start": 251, "end": 252}, + {"type": "template_punctuation", "start": 252, "end": 253}, + {"type": "punctuation", "start": 253, "end": 254}, + {"type": "punctuation", "start": 256, "end": 257}, + {"type": "operator", "start": 276, "end": 277}, + {"type": "punctuation", "start": 278, "end": 279}, + {"type": "punctuation", "start": 279, "end": 280}, + {"type": "operator", "start": 280, "end": 281}, + {"type": "keyword", "start": 282, "end": 286}, + {"type": "operator", "start": 287, "end": 289}, + {"type": "punctuation", "start": 290, "end": 291}, + {"type": "comment", "start": 294, "end": 303}, + {"type": "keyword", "start": 306, "end": 310}, + {"type": "punctuation", "start": 310, "end": 311}, + {"type": "function", "start": 311, "end": 326}, + {"type": "punctuation", "start": 326, "end": 327}, + {"type": "punctuation", "start": 327, "end": 328}, + {"type": "punctuation", "start": 328, "end": 329}, + {"type": "comment", "start": 332, "end": 338}, + {"type": "punctuation", "start": 340, "end": 341}, + {"type": "punctuation", "start": 341, "end": 342}, + {"type": "function", "start": 345, "end": 360}, + {"type": "punctuation", "start": 360, "end": 361}, + {"type": "punctuation", "start": 361, "end": 362}, + {"type": "punctuation", "start": 363, "end": 364}, + {"type": "keyword", "start": 367, "end": 372}, + {"type": "keyword", "start": 373, "end": 376}, + {"type": "class_name", "start": 377, "end": 382}, + {"type": "punctuation", "start": 382, "end": 383}, + {"type": "template_string", "start": 383, "end": 419}, + {"type": "template_punctuation", "start": 383, "end": 384}, + {"type": "interpolation", "start": 384, "end": 394}, + {"type": "interpolation_punctuation", "start": 384, "end": 386}, + {"type": "keyword", "start": 386, "end": 390}, + {"type": "punctuation", "start": 390, "end": 391}, + {"type": "interpolation_punctuation", "start": 393, "end": 394}, + {"type": "string", "start": 394, "end": 418}, + {"type": "template_punctuation", "start": 418, "end": 419}, + {"type": "punctuation", "start": 419, "end": 420}, + {"type": "punctuation", "start": 420, "end": 421}, + {"type": "punctuation", "start": 423, "end": 424}, + {"type": "keyword", "start": 427, "end": 436}, + {"type": "function", "start": 437, "end": 453}, + {"type": "punctuation", "start": 453, "end": 454}, + {"type": "punctuation", "start": 454, "end": 455}, + {"type": "operator", "start": 455, "end": 456}, + {"type": "keyword", "start": 457, "end": 461}, + {"type": "punctuation", "start": 462, "end": 463}, + {"type": "builtin", "start": 466, "end": 473}, + {"type": "punctuation", "start": 473, "end": 474}, + {"type": "function", "start": 474, "end": 477}, + {"type": "punctuation", "start": 477, "end": 478}, + {"type": "keyword", "start": 478, "end": 481}, + {"type": "class_name", "start": 482, "end": 486}, + {"type": "punctuation", "start": 486, "end": 487}, + {"type": "punctuation", "start": 487, "end": 488}, + {"type": "punctuation", "start": 488, "end": 489}, + {"type": "punctuation", "start": 489, "end": 490}, + {"type": "comment", "start": 491, "end": 524}, + {"type": "punctuation", "start": 526, "end": 527}, + {"type": "punctuation", "start": 528, "end": 529}, + {"type": "keyword", "start": 531, "end": 537}, + {"type": "punctuation", "start": 538, "end": 539}, + {"type": "punctuation", "start": 540, "end": 541}, + {"type": "punctuation", "start": 543, "end": 544}, + {"type": "punctuation", "start": 546, "end": 547}, + {"type": "constant", "start": 548, "end": 549}, + {"type": "punctuation", "start": 549, "end": 550}, + {"type": "punctuation", "start": 550, "end": 551}, + {"type": "comment", "start": 553, "end": 563}, + {"type": "comment", "start": 565, "end": 608}, + {"type": "comment", "start": 610, "end": 634}, + {"type": "keyword", "start": 636, "end": 642}, + {"type": "keyword", "start": 643, "end": 652}, + {"type": "class_name", "start": 653, "end": 659}, + {"type": "punctuation", "start": 660, "end": 661}, + {"type": "operator", "start": 667, "end": 668}, + {"type": "builtin", "start": 669, "end": 675}, + {"type": "punctuation", "start": 675, "end": 676}, + {"type": "operator", "start": 681, "end": 682}, + {"type": "builtin", "start": 683, "end": 689}, + {"type": "punctuation", "start": 689, "end": 690}, + {"type": "punctuation", "start": 691, "end": 692}, + {"type": "keyword", "start": 694, "end": 700}, + {"type": "keyword", "start": 701, "end": 706}, + {"type": "operator", "start": 713, "end": 714}, + {"type": "operator", "start": 722, "end": 723}, + {"type": "punctuation", "start": 724, "end": 725}, + {"type": "operator", "start": 729, "end": 730}, + {"type": "string", "start": 731, "end": 738}, + {"type": "punctuation", "start": 738, "end": 739}, + {"type": "operator", "start": 743, "end": 744}, + {"type": "number", "start": 745, "end": 748}, + {"type": "punctuation", "start": 748, "end": 749}, + {"type": "punctuation", "start": 749, "end": 750}, + {"type": "keyword", "start": 752, "end": 758}, + {"type": "keyword", "start": 759, "end": 767}, + {"type": "function", "start": 768, "end": 771}, + {"type": "punctuation", "start": 771, "end": 772}, + {"type": "operator", "start": 773, "end": 774}, + {"type": "builtin", "start": 775, "end": 781}, + {"type": "punctuation", "start": 781, "end": 782}, + {"type": "operator", "start": 784, "end": 785}, + {"type": "builtin", "start": 786, "end": 792}, + {"type": "punctuation", "start": 792, "end": 793}, + {"type": "operator", "start": 793, "end": 794}, + {"type": "builtin", "start": 795, "end": 801}, + {"type": "punctuation", "start": 802, "end": 803}, + {"type": "keyword", "start": 805, "end": 811}, + {"type": "operator", "start": 814, "end": 815}, + {"type": "punctuation", "start": 817, "end": 818}, + {"type": "punctuation", "start": 819, "end": 820}, + {"type": "keyword", "start": 822, "end": 828}, + {"type": "keyword", "start": 829, "end": 834}, + {"type": "operator", "start": 840, "end": 841}, + {"type": "punctuation", "start": 842, "end": 843}, + {"type": "operator", "start": 844, "end": 845}, + {"type": "builtin", "start": 846, "end": 849}, + {"type": "punctuation", "start": 849, "end": 850}, + {"type": "operator", "start": 852, "end": 853}, + {"type": "builtin", "start": 854, "end": 857}, + {"type": "punctuation", "start": 857, "end": 858}, + {"type": "operator", "start": 858, "end": 859}, + {"type": "builtin", "start": 860, "end": 863}, + {"type": "operator", "start": 864, "end": 866}, + {"type": "operator", "start": 869, "end": 870}, + {"type": "punctuation", "start": 872, "end": 873}, + {"type": "comment", "start": 875, "end": 897}, + {"type": "keyword", "start": 898, "end": 904}, + {"type": "keyword", "start": 905, "end": 910}, + {"type": "operator", "start": 929, "end": 930}, + {"type": "string", "start": 931, "end": 960}, + {"type": "punctuation", "start": 960, "end": 961}, + {"type": "keyword", "start": 962, "end": 968}, + {"type": "keyword", "start": 969, "end": 974}, + {"type": "operator", "start": 992, "end": 993}, + {"type": "string", "start": 994, "end": 1020}, + {"type": "punctuation", "start": 1020, "end": 1021}, + {"type": "keyword", "start": 1022, "end": 1028}, + {"type": "keyword", "start": 1029, "end": 1034}, + {"type": "operator", "start": 1054, "end": 1055}, + {"type": "template_string", "start": 1056, "end": 1073}, + {"type": "template_punctuation", "start": 1056, "end": 1057}, + {"type": "string", "start": 1057, "end": 1064}, + {"type": "interpolation", "start": 1064, "end": 1072}, + {"type": "interpolation_punctuation", "start": 1064, "end": 1066}, + {"type": "number", "start": 1066, "end": 1067}, + {"type": "operator", "start": 1068, "end": 1069}, + {"type": "number", "start": 1070, "end": 1071}, + {"type": "interpolation_punctuation", "start": 1071, "end": 1072}, + {"type": "template_punctuation", "start": 1072, "end": 1073}, + {"type": "punctuation", "start": 1073, "end": 1074}, + {"type": "comment", "start": 1076, "end": 1108}, + {"type": "keyword", "start": 1109, "end": 1115}, + {"type": "keyword", "start": 1116, "end": 1121}, + {"type": "operator", "start": 1128, "end": 1129}, + {"type": "regex", "start": 1130, "end": 1139}, + {"type": "regex_delimiter", "start": 1130, "end": 1131}, + {"type": "regex_source", "start": 1131, "end": 1137}, + {"type": "regex_delimiter", "start": 1137, "end": 1138}, + {"type": "regex_flags", "start": 1138, "end": 1139}, + {"type": "punctuation", "start": 1139, "end": 1140}, + {"type": "keyword", "start": 1141, "end": 1147}, + {"type": "keyword", "start": 1148, "end": 1153}, + {"type": "operator", "start": 1168, "end": 1169}, + {"type": "regex", "start": 1170, "end": 1202}, + {"type": "regex_delimiter", "start": 1170, "end": 1171}, + {"type": "regex_source", "start": 1171, "end": 1201}, + {"type": "regex_delimiter", "start": 1201, "end": 1202}, + {"type": "punctuation", "start": 1202, "end": 1203}, + {"type": "comment", "start": 1205, "end": 1261}, + {"type": "comment", "start": 1262, "end": 1313} + ], + "html": "const a = 1;\n\nconst b = 'b';\n\nconst c = true;\n\nexport type Some_Type = 1 | 'b' | true;\n\nclass D {\n\td1: string = 'd';\n\td2: number;\n\td3 = $state(false);\n\n\tconstructor(d2: number) {\n\t\tthis.d2 = d2;\n\t}\n\n\tclass_method(): string {\n\t\treturn `Hello, ${this.d1}`;\n\t}\n\n\tinstance_method = (): void => {\n\t\t/* ... */\n\t\tthis.#private_method();\n\t\t// foo\n\t};\n\n\t#private_method() {\n\t\tthrow new Error(`${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t`);\n\t}\n\n\tprotected protected_method(): void {\n\t\tconsole.log(new Date()); // eslint-disable-line no-console\n\t}\n}\n\nexport {a, b, c, D};\n\n// comment\n\n/*\nother comment\n\nconst comment = false;\n*/\n\n/**\n * JSDoc comment\n */\n\nexport interface Some_E {\n\tname: string;\n\tage: number;\n}\n\nexport const some_e: Some_E = {name: 'A. H.', age: 100};\n\nexport function add(x: number, y: number): number {\n\treturn x + y;\n}\n\nexport const plus = (a: any, b: any): any => a + b;\n\n// boundary test cases\nexport const str_with_keywords = 'const class function string';\nexport const str_with_comment = '// this is not a comment';\nexport const template_with_expr = `Value: ${1 + 2}`;\n\n// regex that looks like comment\nexport const regex = /\\/\\/.*/g;\nexport const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/;\n\n// string in comment should not be highlighted as string\n// const commented = \"this string is in a comment\";\n" +} diff --git a/fixtures/generated/ts/ts_complex.txt b/fixtures/generated/ts/ts_complex.txt new file mode 100644 index 00000000..9188a0e4 --- /dev/null +++ b/fixtures/generated/ts/ts_complex.txt @@ -0,0 +1,263 @@ +=== TOKENS === + 0-5 keyword const + 8-9 operator = + 10-11 number 1 + 11-12 punctuation ; + 14-19 keyword const + 22-23 operator = + 24-27 string 'b' + 27-28 punctuation ; + 30-35 keyword const + 38-39 operator = + 40-44 boolean true + 44-45 punctuation ; + 47-53 keyword export + 54-58 keyword type + 59-68 class_name Some_Type + 69-70 operator = + 71-72 number 1 + 73-74 operator | + 75-78 string 'b' + 79-80 operator | + 81-85 boolean true + 85-86 punctuation ; + 88-93 keyword class + 94-95 class_name D + 94-95 constant D + 96-97 punctuation { + 101-102 operator : + 103-109 builtin string + 110-111 operator = + 112-115 string 'd' + 115-116 punctuation ; + 120-121 operator : + 122-128 builtin number + 128-129 punctuation ; + 134-135 operator = + 136-142 function $state + 142-143 punctuation ( + 143-148 boolean false + 148-149 punctuation ) + 149-150 punctuation ; + 153-164 function constructor + 164-165 punctuation ( + 167-168 operator : + 169-175 builtin number + 175-176 punctuation ) + 177-178 punctuation { + 181-185 keyword this + 185-186 punctuation . + 189-190 operator = + 193-194 punctuation ; + 196-197 punctuation } + 200-212 function class_method + 212-213 punctuation ( + 213-214 punctuation ) + 214-215 operator : + 216-222 builtin string + 223-224 punctuation { + 227-233 keyword return + 234-253 template_string `Hello, ${this.d1}` + 234-235 template_punctuation ` + 235-242 string Hello, + 242-252 interpolation ${this.d1} + 242-244 interpolation_punctuation ${ + 244-248 keyword this + 248-249 punctuation . + 251-252 interpolation_punctuation } + 252-253 template_punctuation ` + 253-254 punctuation ; + 256-257 punctuation } + 276-277 operator = + 278-279 punctuation ( + 279-280 punctuation ) + 280-281 operator : + 282-286 keyword void + 287-289 operator => + 290-291 punctuation { + 294-303 comment /* ... */ + 306-310 keyword this + 310-311 punctuation . + 311-326 function #private_method + 326-327 punctuation ( + 327-328 punctuation ) + 328-329 punctuation ; + 332-338 comment // foo + 340-341 punctuation } + 341-342 punctuation ; + 345-360 function #private_method + 360-361 punctuation ( + 361-362 punctuation ) + 363-364 punctuation { + 367-372 keyword throw + 373-376 keyword new + 377-382 class_name Error + 382-383 punctuation ( + 383-419 template_string `${this.d1} \n\t\t\tmultiline\n\t\t\tetc\n\t\t` + 383-384 template_punctuation ` + 384-394 interpolation ${this.d1} + 384-386 interpolation_punctuation ${ + 386-390 keyword this + 390-391 punctuation . + 393-394 interpolation_punctuation } + 394-418 string \n\t\t\tmultiline\n\t\t\tetc\n\t\t + 418-419 template_punctuation ` + 419-420 punctuation ) + 420-421 punctuation ; + 423-424 punctuation } + 427-436 keyword protected + 437-453 function protected_method + 453-454 punctuation ( + 454-455 punctuation ) + 455-456 operator : + 457-461 keyword void + 462-463 punctuation { + 466-473 builtin console + 473-474 punctuation . + 474-477 function log + 477-478 punctuation ( + 478-481 keyword new + 482-486 class_name Date + 486-487 punctuation ( + 487-488 punctuation ) + 488-489 punctuation ) + 489-490 punctuation ; + 491-524 comment // eslint-disable-line no-console + 526-527 punctuation } + 528-529 punctuation } + 531-537 keyword export + 538-539 punctuation { + 540-541 punctuation , + 543-544 punctuation , + 546-547 punctuation , + 548-549 constant D + 549-550 punctuation } + 550-551 punctuation ; + 553-563 comment // comment + 565-608 comment /*\nother comment\n\nconst comment = false;\n*/ + 610-634 comment /**\n * JSDoc comment\n */ + 636-642 keyword export + 643-652 keyword interface + 653-659 class_name Some_E + 660-661 punctuation { + 667-668 operator : + 669-675 builtin string + 675-676 punctuation ; + 681-682 operator : + 683-689 builtin number + 689-690 punctuation ; + 691-692 punctuation } + 694-700 keyword export + 701-706 keyword const + 713-714 operator : + 722-723 operator = + 724-725 punctuation { + 729-730 operator : + 731-738 string 'A. H.' + 738-739 punctuation , + 743-744 operator : + 745-748 number 100 + 748-749 punctuation } + 749-750 punctuation ; + 752-758 keyword export + 759-767 keyword function + 768-771 function add + 771-772 punctuation ( + 773-774 operator : + 775-781 builtin number + 781-782 punctuation , + 784-785 operator : + 786-792 builtin number + 792-793 punctuation ) + 793-794 operator : + 795-801 builtin number + 802-803 punctuation { + 805-811 keyword return + 814-815 operator + + 817-818 punctuation ; + 819-820 punctuation } + 822-828 keyword export + 829-834 keyword const + 840-841 operator = + 842-843 punctuation ( + 844-845 operator : + 846-849 builtin any + 849-850 punctuation , + 852-853 operator : + 854-857 builtin any + 857-858 punctuation ) + 858-859 operator : + 860-863 builtin any + 864-866 operator => + 869-870 operator + + 872-873 punctuation ; + 875-897 comment // boundary test cases + 898-904 keyword export + 905-910 keyword const + 929-930 operator = + 931-960 string 'const class function string' + 960-961 punctuation ; + 962-968 keyword export + 969-974 keyword const + 992-993 operator = + 994-1020 string '// this is not a comment' +1020-1021 punctuation ; +1022-1028 keyword export +1029-1034 keyword const +1054-1055 operator = +1056-1073 template_string `Value: ${1 + 2}` +1056-1057 template_punctuation ` +1057-1064 string Value: +1064-1072 interpolation ${1 + 2} +1064-1066 interpolation_punctuation ${ +1066-1067 number 1 +1068-1069 operator + +1070-1071 number 2 +1071-1072 interpolation_punctuation } +1072-1073 template_punctuation ` +1073-1074 punctuation ; +1076-1108 comment // regex that looks like comment +1109-1115 keyword export +1116-1121 keyword const +1128-1129 operator = +1130-1139 regex /\/\/.*/g +1130-1131 regex_delimiter / +1131-1137 regex_source \/\/.* +1137-1138 regex_delimiter / +1138-1139 regex_flags g +1139-1140 punctuation ; +1141-1147 keyword export +1148-1153 keyword const +1168-1169 operator = +1170-1202 regex /^(?:\/\*.*?\*\/|\/\/.*|[^/])+$/ +1170-1171 regex_delimiter / +1171-1201 regex_source ^(?:\/\*.*?\*\/|\/\/.*|[^/])+$ +1201-1202 regex_delimiter / +1202-1203 punctuation ; +1205-1261 comment // string in comment should not be highlighted as string +1262-1313 comment // const commented = "this string is in a comment"; + +=== STATS === +Total tokens: 237 +Sample length: 1314 characters + +Token types: + punctuation: 79 + operator: 39 + keyword: 37 + builtin: 13 + comment: 10 + string: 9 + function: 8 + template_punctuation: 6 + interpolation_punctuation: 6 + number: 5 + class_name: 5 + regex_delimiter: 4 + boolean: 3 + template_string: 3 + interpolation: 3 + constant: 2 + regex: 2 + regex_source: 2 + regex_flags: 1 diff --git a/package-lock.json b/package-lock.json index b7e69e6f..9588a795 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@ryanatkn/belt": "^0.34.1", "@ryanatkn/eslint-config": "^0.8.0", "@ryanatkn/fuz": "^0.145.0", - "@ryanatkn/gro": "^0.164.1", + "@ryanatkn/gro": "^0.165.0", "@ryanatkn/moss": "^0.33.0", "@sveltejs/adapter-static": "^3.0.9", "@sveltejs/kit": "^2.37.1", @@ -22,10 +22,12 @@ "@types/node": "^24.3.1", "eslint": "^9.35.0", "eslint-plugin-svelte": "^3.12.1", + "esm-env": "^1.2.2", "prettier": "^3.6.2", "prettier-plugin-svelte": "^3.4.0", "svelte": "^5.38.7", "svelte-check": "^4.3.1", + "tinybench": "^5.0.1", "tslib": "^2.8.1", "typescript": "^5.9.2", "typescript-eslint": "^8.42.0", @@ -1475,9 +1477,9 @@ } }, "node_modules/@ryanatkn/gro": { - "version": "0.164.1", - "resolved": "https://registry.npmjs.org/@ryanatkn/gro/-/gro-0.164.1.tgz", - "integrity": "sha512-bUtHPqDEgL6Ehy+5XN7XlU6Ng7EqIorXSQmGRnmwZkqPgukgUtP7fVPw4+O+cOco8zCTsRCl/YRdKaHamyPMXA==", + "version": "0.165.0", + "resolved": "https://registry.npmjs.org/@ryanatkn/gro/-/gro-0.165.0.tgz", + "integrity": "sha512-FrDuq5+mzRMr3+h5Ooj3QFJmuLsKFkJlkt6Ukbn4f6c2AYwAL9SDHT4bIFwelAhzK63HkJIfYf70Zo2KyipGAA==", "dev": true, "license": "MIT", "dependencies": { @@ -3988,11 +3990,14 @@ } }, "node_modules/tinybench": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", - "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-5.0.1.tgz", + "integrity": "sha512-aNVgWQZY4veCZLQJRftDA1X9OoLUIjDWNfC90nledkX7Lx205IpSEFYnsu4slyofoPGpJ+NIQj+BNSt4U5edMg==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } }, "node_modules/tinyexec": { "version": "0.3.2", @@ -4370,6 +4375,13 @@ } } }, + "node_modules/vitest/node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 37125119..88a5a282 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,10 @@ "check": "gro check", "test": "gro test", "preview": "vite preview", - "deploy": "gro deploy" + "deploy": "gro deploy", + "benchmark": "gro run benchmark/run_benchmarks.ts", + "benchmark-compare": "gro run benchmark/compare/run_compare.ts", + "update-generated-fixtures": "gro src/fixtures/update" }, "type": "module", "engines": { @@ -46,7 +49,7 @@ "@ryanatkn/belt": "^0.34.1", "@ryanatkn/eslint-config": "^0.8.0", "@ryanatkn/fuz": "^0.145.0", - "@ryanatkn/gro": "^0.164.1", + "@ryanatkn/gro": "^0.165.0", "@ryanatkn/moss": "^0.33.0", "@sveltejs/adapter-static": "^3.0.9", "@sveltejs/kit": "^2.37.1", @@ -55,10 +58,12 @@ "@types/node": "^24.3.1", "eslint": "^9.35.0", "eslint-plugin-svelte": "^3.12.1", + "esm-env": "^1.2.2", "prettier": "^3.6.2", "prettier-plugin-svelte": "^3.4.0", "svelte": "^5.38.7", "svelte-check": "^4.3.1", + "tinybench": "^5.0.1", "tslib": "^2.8.1", "typescript": "^5.9.2", "typescript-eslint": "^8.42.0", @@ -91,61 +96,22 @@ "!dist/**/*.test.*" ], "exports": { - ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - }, "./package.json": "./package.json", - "./code_sample_inputs.js": { - "types": "./dist/code_sample_inputs.d.ts", - "default": "./dist/code_sample_inputs.js" - }, - "./code_sample_outputs.js": { - "types": "./dist/code_sample_outputs.d.ts", - "default": "./dist/code_sample_outputs.js" - }, - "./Code.svelte": { - "types": "./dist/Code.svelte.d.ts", - "svelte": "./dist/Code.svelte", - "default": "./dist/Code.svelte" - }, - "./grammar_clike.js": { - "types": "./dist/grammar_clike.d.ts", - "default": "./dist/grammar_clike.js" - }, - "./grammar_css.js": { - "types": "./dist/grammar_css.d.ts", - "default": "./dist/grammar_css.js" - }, - "./grammar_js.js": { - "types": "./dist/grammar_js.d.ts", - "default": "./dist/grammar_js.js" - }, - "./grammar_json.js": { - "types": "./dist/grammar_json.d.ts", - "default": "./dist/grammar_json.js" - }, - "./grammar_markup.js": { - "types": "./dist/grammar_markup.d.ts", - "default": "./dist/grammar_markup.js" - }, - "./grammar_svelte.js": { - "types": "./dist/grammar_svelte.d.ts", - "default": "./dist/grammar_svelte.js" - }, - "./grammar_ts.js": { - "types": "./dist/grammar_ts.d.ts", - "default": "./dist/grammar_ts.js" + "./*.js": { + "types": "./dist/*.d.ts", + "default": "./dist/*.js" }, - "./syntax_styler.js": { - "types": "./dist/syntax_styler.d.ts", - "default": "./dist/syntax_styler.js" + "./*.svelte": { + "types": "./dist/*.svelte.d.ts", + "svelte": "./dist/*.svelte", + "default": "./dist/*.svelte" }, - "./theme_standalone.css": { - "default": "./dist/theme_standalone.css" + "./*.json": { + "types": "./dist/*.json.d.ts", + "default": "./dist/*.json" }, - "./theme.css": { - "default": "./dist/theme.css" + "./*.css": { + "default": "./dist/*.css" } } } diff --git a/src/fixtures/check.test.ts b/src/fixtures/check.test.ts new file mode 100644 index 00000000..93c73bbf --- /dev/null +++ b/src/fixtures/check.test.ts @@ -0,0 +1,202 @@ +import {test, assert, describe} from 'vitest'; +import {readFileSync, existsSync} from 'node:fs'; +import { + discover_samples, + process_sample, + get_fixture_path, + type Generated_Output, +} from './helpers.js'; +import {sample_langs} from '$lib/code_sample.js'; + +/** + * Test Architecture Overview + * ========================= + * + * This test suite verifies that our syntax highlighters produce consistent, + * correct output by comparing runtime behavior against generated fixtures. + * + * Tests generated fixture files to ensure: + * - HTML rendering matches generated fixtures + * - Token ranges are properly nested (no invalid overlaps) + * - All expected tokens are present + */ + +describe('generated fixtures match runtime', () => { + // Discover all sample files using helper + const samples = discover_samples(); + + // Generate test for each sample + for (const sample of samples) { + describe(`${sample.lang}_${sample.variant}`, () => { + const fixture_path = get_fixture_path(sample.lang, sample.variant, 'json'); + + test('fixture file exists', () => { + // Basic sanity check - fixtures must be generated before tests can run + assert.ok( + existsSync(fixture_path), + `Fixture file missing: ${fixture_path}. Run 'npm run task src/fixtures/update' to generate.`, + ); + }); + + test('syntax styler output matches fixture', () => { + /** + * Current: Tests exact HTML string match + * + * Ideal: Should test: + * - All code is highlighted (no plain text except whitespace) + * - Token boundaries are correct + * - Token types are semantically correct + * - No overlapping spans + * - Performance metrics (time, memory) + */ + if (!existsSync(fixture_path)) { + console.warn(`Skipping test - fixture missing: ${fixture_path}`); // eslint-disable-line no-console + return; + } + + const fixture: Generated_Output = JSON.parse(readFileSync(fixture_path, 'utf-8')); + const runtime_output = process_sample(sample); + + assert.strictEqual( + runtime_output.html, + fixture.html, + `HTML output mismatch for ${sample.lang}_${sample.variant}`, + ); + + // TODO: Additional assertions + // assert(calculate_coverage(sample.content, runtime_output.html) > 0.95); + // assert(validate_no_overlaps(extract_tokens(runtime_output.html))); + }); + + test('token positions are valid', () => { + /** + * Validates that token positions are correctly calculated + * from the DOM styler token tree + * + * Should verify: + * - No overlapping tokens + * - All positions within bounds + * - Tokens match expected patterns + */ + const runtime_output = process_sample(sample); + + // Verify tokens are properly nested (overlapping is ok if fully contained) + const tokensByStart = [...runtime_output.tokens].sort((a, b) => + a.start !== b.start ? a.start - b.start : b.end - a.end, + ); + + for (let i = 0; i < tokensByStart.length; i++) { + const token = tokensByStart[i]; + + // Check bounds + assert.ok( + token.start >= 0 && token.end <= sample.content.length, + `Token ${token.type} extends beyond content at position ${token.end} (content length: ${sample.content.length})`, + ); + + // Check that any overlapping tokens are properly nested + for (let j = i + 1; j < tokensByStart.length; j++) { + const other = tokensByStart[j]; + if (other.start >= token.end) break; // No more overlaps possible + + // If tokens overlap, one must fully contain the other + if (other.start < token.end) { + const properlyNested = + (token.start <= other.start && token.end >= other.end) || // token contains other + (other.start <= token.start && other.end >= token.end); // other contains token + + assert.ok( + properlyNested, + `Invalid overlap: token ${token.type} [${token.start}-${token.end}] partially overlaps with ${other.type} [${other.start}-${other.end}]`, + ); + } + } + } + }); + + test('token data matches fixture', () => { + /** + * Ensures token positions are consistent between runs + * + * Should verify: + * - Token positions match expected values + * - Token types are correctly identified + * - Tokenization is deterministic + */ + if (!existsSync(fixture_path)) { + console.warn(`Skipping test - fixture missing: ${fixture_path}`); // eslint-disable-line no-console + return; + } + + const fixture: Generated_Output = JSON.parse(readFileSync(fixture_path, 'utf-8')); + const runtime_output = process_sample(sample); + + assert.deepEqual( + runtime_output.tokens, + fixture.tokens, + `Token data mismatch for ${sample.lang}_${sample.variant}`, + ); + }); + }); + } +}); + +describe('all expected languages are tested', () => { + test('sample files exist for all supported languages', () => { + const found_languages: Set = new Set(); + + const samples = discover_samples(); + for (const sample of samples) { + found_languages.add(sample.lang); + } + + for (const lang of sample_langs) { + assert.ok(found_languages.has(lang), `Missing sample files for language: ${lang}`); + } + }); +}); + +/** + * Future Test Improvements + * ======================= + * + * describe('semantic equivalence', () => { + * test('all tokens have proper ranges', () => { + * // Compare token positions are valid + * }); + * + * test('no code is left unhighlighted', () => { + * // Verify 95%+ coverage (allowing for whitespace) + * }); + * }); + * + * describe('performance benchmarks', () => { + * test('highlighting completes within time budget', () => { + * // Track time per KB of code + * }); + * + * test('memory usage is reasonable', () => { + * // Track memory per KB of code + * }); + * }); + * + * describe('edge cases', () => { + * test('handles malformed code gracefully', () => { + * // Should not crash, should highlight what it can + * }); + * + * test('handles extremely long lines', () => { + * // Performance should degrade gracefully + * }); + * + * test('handles nested language boundaries', () => { + * // JS in HTML in Svelte, etc. + * }); + * }); + * + * describe('visual regression', () => { + * test('highlighted code screenshots match', () => { + * // Render to canvas, compare pixels + * }); + * }); + */ diff --git a/src/fixtures/helpers.ts b/src/fixtures/helpers.ts new file mode 100644 index 00000000..8989af90 --- /dev/null +++ b/src/fixtures/helpers.ts @@ -0,0 +1,184 @@ +import {readFileSync} from 'node:fs'; +import {search_fs} from '@ryanatkn/gro/search_fs.js'; +import {basename, join, relative} from 'node:path'; +import {syntax_styler_global} from '$lib/syntax_styler_global.js'; +import {tokenize_syntax, type Syntax_Token_Stream, Syntax_Token} from '$lib/syntax_styler.js'; + +export interface Sample_Spec { + lang: string; + variant: string; + content: string; + filepath: string; +} + +export interface Generated_Output { + sample: Sample_Spec; + tokens: Array; + html: string; +} + +/** + * Discover all sample files in src/lib/samples + */ +export const discover_samples = (): Array => { + const sample_files = search_fs('src/lib/samples', { + file_filter: (path) => /sample_[^/]+\.(ts|css|html|json|svelte)$/.test(path), + }); + + const samples: Array = []; + + for (const file of sample_files) { + const filename = basename(file.id); + const match = /sample_([^.]+)\.(.+)$/.exec(filename); + if (!match) continue; + + const [, variant, lang] = match; + const content = readFileSync(file.id, 'utf-8'); + + samples.push({ + lang, + variant, + content, + filepath: relative(process.cwd(), file.id), + }); + } + + return samples; +}; + +/** + * Get the fixture path for a given language and variant + */ +export const get_fixture_path = (lang: string, variant: string, ext: 'json' | 'txt'): string => { + return join('fixtures/generated', lang, `${lang}_${variant}.${ext}`); +}; + +/** + * Generate syntax HTML output for a sample + */ +export const generate_syntax_output = (sample: Sample_Spec): string => { + return syntax_styler_global.stylize(sample.content, sample.lang); +}; + +/** + * Extract all tokens with positions for fixture generation + */ +const extract_all_tokens = ( + tokens: Syntax_Token_Stream, + offset: number = 0, +): Array<{type: string; start: number; end: number}> => { + const result: Array<{type: string; start: number; end: number}> = []; + let pos = offset; + + for (const token of tokens) { + if (typeof token === 'string') { + // Plain text, advance position + pos += token.length; + } else if (token instanceof Syntax_Token) { + const start = pos; + const length = get_token_length(token); + const end = start + length; + + // Add this token + result.push({ + type: token.type, + start, + end, + }); + + // Process nested tokens + if (Array.isArray(token.content)) { + const nested = extract_all_tokens(token.content, start); + result.push(...nested); + } + + pos = end; + } + } + + return result; +}; + +/** + * Calculate the total text length of a token + */ +const get_token_length = (token: Syntax_Token): number => { + if (typeof token.content === 'string') { + return token.content.length; + } + + let length = 0; + for (const item of token.content) { + if (typeof item === 'string') { + length += item.length; + } else { + length += get_token_length(item); + } + } + return length; +}; + +/** + * Generate token data from syntax styler + */ +export const generate_token_data = (sample: Sample_Spec): Array => { + // Get tokens from syntax styler and extract all with positions + const grammar = syntax_styler_global.get_lang(sample.lang); + const tokens = tokenize_syntax(sample.content, grammar); + const flat_tokens = extract_all_tokens(tokens); + + return flat_tokens; +}; + +/** + * Process a sample to generate all outputs + */ +export const process_sample = (sample: Sample_Spec): Generated_Output => { + const html = generate_syntax_output(sample); + const tokens = generate_token_data(sample); + + return { + sample, + tokens, + html, + }; +}; + +/** + * Generate debug text output for a sample + */ +export const generate_debug_text = (output: Generated_Output): string => { + const {sample, tokens} = output; + + let debug = '=== TOKENS ===\n'; + // Show all tokens, no elision + for (const t of tokens) { + const text = sample.content + .substring(t.start, t.end) + .replace(/\n/g, '\\n') + .replace(/\t/g, '\\t'); + // Format: start-end type text + const start = String(t.start).padStart(4); + const end = String(t.end).padEnd(4); + const position = `${start}-${end}`; + debug += `${position.padEnd(10)} ${t.type.padEnd(25)} ${text}\n`; + } + + // Add token statistics + debug += '\n=== STATS ===\n'; + debug += `Total tokens: ${tokens.length}\n`; + debug += `Sample length: ${sample.content.length} characters\n`; + + // Count token types + const tokenTypes: Record = {}; + for (const token of tokens) { + const {type} = token; + tokenTypes[type] = (tokenTypes[type] || 0) + 1; + } + debug += '\nToken types:\n'; + for (const [type, count] of Object.entries(tokenTypes).sort((a, b) => b[1] - a[1])) { + debug += ` ${type}: ${count}\n`; + } + + return debug; +}; diff --git a/src/fixtures/update.task.ts b/src/fixtures/update.task.ts new file mode 100644 index 00000000..5f904f6d --- /dev/null +++ b/src/fixtures/update.task.ts @@ -0,0 +1,59 @@ +import type {Task} from '@ryanatkn/gro'; +import {writeFileSync, mkdirSync, rmSync, existsSync} from 'node:fs'; +import {join} from 'node:path'; +import {format_file} from '@ryanatkn/gro/format_file.js'; +import { + discover_samples, + process_sample, + generate_debug_text, + get_fixture_path, +} from './helpers.js'; + +export const task: Task = { + summary: 'update all test fixtures from sample files', + run: async ({invoke_task}) => { + await invoke_task('gen'); + + // Discover all sample files + const samples = discover_samples(); + + // Get unique languages to clean directories + const languages = new Set(samples.map((s) => s.lang)); + + // Remove existing language directories + for (const lang of languages) { + const dir = join('fixtures/generated', lang); + if (existsSync(dir)) { + rmSync(dir, {recursive: true, force: true}); + console.log(`Removed existing directory: ${dir}`); // eslint-disable-line no-console + } + } + + // Process each sample + for (const sample of samples) { + console.log(`Processing ${sample.lang}_${sample.variant}...`); // eslint-disable-line no-console + + // Process sample using helper + const output = process_sample(sample); + + // Ensure directory exists + const dir = join('fixtures/generated', sample.lang); + mkdirSync(dir, {recursive: true}); + + // Write JSON file + const json_path = get_fixture_path(sample.lang, sample.variant, 'json'); + const json_content = JSON.stringify(output); + const formatted_json = await format_file(json_content, {filepath: json_path}); // eslint-disable-line no-await-in-loop + writeFileSync(json_path, formatted_json); + console.log(` → ${json_path}`); // eslint-disable-line no-console + + // Generate and write debug text file + const debug_text = generate_debug_text(output); + const txt_path = get_fixture_path(sample.lang, sample.variant, 'txt'); + writeFileSync(txt_path, debug_text); + console.log(` → ${txt_path}`); // eslint-disable-line no-console + } + + console.log(`\n✓ Updated ${samples.length} samples`); // eslint-disable-line no-console + }, +}; diff --git a/src/lib/Code.svelte b/src/lib/Code.svelte index 1bb58976..ca85fc65 100644 --- a/src/lib/Code.svelte +++ b/src/lib/Code.svelte @@ -1,48 +1,114 @@ + + -{#if children}{@render children(markup)}{:else}{@html markup}{/if}{#if use_ranges && children}{@render children( + content, + )}{:else if use_ranges}{content}{:else if children}{@render children( + html_content, + )}{:else}{@html html_content}{/if} diff --git a/src/lib/code_sample.ts b/src/lib/code_sample.ts new file mode 100644 index 00000000..77b7da24 --- /dev/null +++ b/src/lib/code_sample.ts @@ -0,0 +1,10 @@ +export interface Code_Sample { + name: string; + lang: string; + content: string; +} + +// Languages ordered from simple to complex +export const sample_langs = ['json', 'css', 'ts', 'html', 'svelte'] as const; + +export type Sample_Lang = (typeof sample_langs)[number]; diff --git a/src/lib/code_sample_inputs.ts b/src/lib/code_sample_inputs.ts deleted file mode 100644 index dda665e1..00000000 --- a/src/lib/code_sample_inputs.ts +++ /dev/null @@ -1,239 +0,0 @@ -// see `styled_json_code` in `./code_sample_outputs.js` -// for the result of `stylize(sample_json_code, Syntax_Styler.langs.json, 'json')` -export const sample_json_code = ` - -{ - "string": "a string", - "number": 12345, - "boolean": true, - "null": null, - "object": { - "array": [1, "b", false] - } // comments :D -} - -`.trim(); - -// see `styled_html_code` in `./code_sample_outputs.js` -// for the result of `stylize(sample_html_code, Syntax_Styler.langs.markup, 'html')` -export const sample_html_code = ` - - - -
-

hello world!

-
- -

- some text -

- - - - - -
- -
- -access - -
    -
  • list item 1
    • -
  • list item 2
    • -
- - - - - - -]]> -`.trim(); - -// see `styled_css_code` in `./code_sample_outputs.js` -// for the result of `stylize(sample_css_code, Syntax_Styler.langs.css, 'css')` -export const sample_css_code = ` - -.some_class { - color: red; -} - -.dash-class { - font-size: 16px; -} - -p { - box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); -} - -/* comment */ - -/* -multi -line - - - -*/ - -#unique_id { - background-color: blue; -} - -div > p { - margin: 10px; -} - -@media (max-width: 600px) { - body { - background-color: lightblue; - } -} -`.trim(); - -// see `styled_ts_code` in `./code_sample_outputs.js` -// for the result of `stylize(sample_ts_code, Syntax_Styler.langs.ts, 'ts')` -export const sample_ts_code = ` - -const a = 1; - -const b = 'b'; - -const c = true; - -export type Some_Type = 1 | 'b' | true; - -class D { - d1: string = 'd'; - d2: number; - d3 = $state(false); - - constructor(d2: number) { - this.d2 = d2; - } - - class_method(): string { - return \`Hello, \${this.d1}\`; - } - - instance_method = () => { /* ... */ }; - - #private_method() { - throw new Error(\`\${this.d1} etc\`); - } - - protected protected_method() { - console.log(new RegExp('protected')); - } -} - -// comment - -/* -other comment - -const comment = false; -*/ - -/** - * JSDoc comment - */ - -export interface Some_E { - name: string; - age: number; -} - -export const some_e: Some_E = {name: 'A. H.', age: 100}; - -export function add(x: number, y: number): number { - return x + y; -} - -export const plus = (a: any, b: any): any => a + b; -`.trim(); - -// see `styled_css_code` in `./code_sample_outputs.js` -// for the result of `stylize(sample_svelte_code, Syntax_Styler.langs.svelte, 'svelte')` -export const sample_svelte_code = ` - - - - - -

hello {HELLO}!

- -{#each thing_keys as key (key)} - {@const value = thing[key]} - {value} -{/each} - -{#if true} - -{:else} - - {@render children()} - -{/if} - -${sample_html_code} - - - -`.trim(); - -export const samples = [ - { - content: sample_json_code, - lang: 'json', - }, - { - content: sample_html_code, - lang: 'html', - }, - { - content: sample_css_code, - lang: 'css', - }, - { - content: sample_ts_code, - lang: 'ts', - }, - { - content: sample_svelte_code, - lang: 'svelte', - }, -]; diff --git a/src/lib/code_sample_outputs.ts b/src/lib/code_sample_outputs.ts deleted file mode 100644 index 328a2be6..00000000 --- a/src/lib/code_sample_outputs.ts +++ /dev/null @@ -1,14 +0,0 @@ -export const styled_json_code = - '{\n\t"string": "a string",\n\t"number": 12345,\n\t"boolean": true,\n\t"null": null,\n\t"object": {\n\t\t"array": [1, "b", false]\n\t} // comments :D\n}'; - -export const styled_html_code = - '<!DOCTYPE html>\n\n<div class="test">\n\t<p>hello world!</p>\n</div>\n\n<p class="some_class dash-class">\n\tsome <span class="a b c">text</span>\n</p>\n\n<button type="button" disabled>\n\tclick me\n</button>\n\n<!-- comment <div>a<br /> b</div> -->\n\n<br>\n\n<hr />\n\n<img src="image.jpg" alt="access">\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<script type="text/javascript">\n\tconst ok = \'yes\';\n</script>\n\n<style type="text/css">\n\t.special::before {\n\t\tcontent: "< & >";\n\t}\n</style>\n\n<![CDATA[\n\tif (a < 0) alert("b");\n\t<not-a-tag>\n]]>'; - -export const styled_css_code = - '.some_class {\n\tcolor: red;\n}\n\n.dash-class {\n\tfont-size: 16px;\n}\n\np {\n\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n}\n\n/* comment */\n\n/*\nmulti\nline\n\n<comment>\n\n*/\n\n#unique_id {\n\tbackground-color: blue;\n}\n\ndiv > p {\n\tmargin: 10px;\n}\n\n@media (max-width: 600px) {\n\tbody {\n\t\tbackground-color: lightblue;\n\t}\n}'; - -export const styled_ts_code = - 'const a = 1;\n\nconst b = \'b\';\n\nconst c = true;\n\nexport type Some_Type = 1 | \'b\' | true;\n\nclass D {\n\td1: string = \'d\';\n\td2: number;\n\td3 = $state(false);\n\n\tconstructor(d2: number) {\n\t\tthis.d2 = d2;\n\t}\n\n\tclass_method(): string {\n\t\treturn `Hello, ${this.d1}`;\n\t}\n\n\tinstance_method = () => { /* ... */ };\n\n\t#private_method() {\n\t\tthrow new Error(`${this.d1} etc`);\n\t}\n\n\tprotected protected_method() {\n\t\tconsole.log(new RegExp(\'protected\'));\n\t}\n}\n\n// comment\n\n/*\nother comment\n\nconst comment = false;\n*/\n\n/**\n * JSDoc comment\n */\n\nexport interface Some_E {\n\tname: string;\n\tage: number;\n}\n\nexport const some_e: Some_E = {name: \'A. H.\', age: 100};\n\nexport function add(x: number, y: number): number {\n\treturn x + y;\n}\n\nexport const plus = (a: any, b: any): any => a + b;'; - -export const styled_svelte_code = - '<script lang="ts" module>\n\texport const HELLO = \'world\';\n</script>\n\n<script lang="ts">\n\timport Thing from \'$lib/Thing.svelte\';\n\n\tinterface Props {\n\t\tthing: Record<string, any>;\n\t}\n\t\n\tconst {thing}: Props = $props();\n\n\tconst thing_keys = $derived(Object.keys(thing));\n\n\tconst a = 1;\n\t\n\tconst b = \'b\';\n\t\n\tconst c = true;\n\t\n\texport type Some_Type = 1 | \'b\' | true;\n\t\n\tclass D {\n\t\td1: string = \'d\';\n\t\td2: number;\n\t\td3 = $state(false);\n\t\n\t\tconstructor(d2: number) {\n\t\t\tthis.d2 = d2;\n\t\t}\n\t\n\t\tclass_method(): string {\n\t\t\treturn `Hello, ${this.d1}`;\n\t\t}\n\t\n\t\tinstance_method = () => { /* ... */ };\n\t\n\t\t#private_method() {\n\t\t\tthrow new Error(`${this.d1} etc`);\n\t\t}\n\t\n\t\tprotected protected_method() {\n\t\t\tconsole.log(new RegExp(\'protected\'));\n\t\t}\n\t}\n\t\n\t// comment\n\t\n\t/*\n\tother comment\n\t\n\tconst comment = false;\n\t*/\n\t\n\t/**\n\t * JSDoc comment\n\t */\n\t\n\texport interface Some_E {\n\t\tname: string;\n\t\tage: number;\n\t}\n\t\n\texport const some_e: Some_E = {name: \'A. H.\', age: 100};\n\t\n\texport function add(x: number, y: number): number {\n\t\treturn x + y;\n\t}\n\t\n\texport const plus = (a: any, b: any): any => a + b;\n</script>\n\n<h1>hello {HELLO}!</h1>\n\n{#each thing_keys as key (key)}\n\t{@const value = thing[key]}\n\t{value}\n{/each}\n\n{#if true}\n\t<Thing string_prop="a" number_prop={1} />\n{:else}\n\t<Thing string_prop="b" number_prop={2}>\n\t\t{@render children()}\n\t</Thing>\n{/if}\n\n<!DOCTYPE html>\n\n<div class="test">\n\t<p>hello world!</p>\n</div>\n\n<p class="some_class dash-class">\n\tsome <span class="a b c">text</span>\n</p>\n\n<button type="button" disabled>\n\tclick me\n</button>\n\n<!-- comment <div>a<br /> b</div> -->\n\n<br>\n\n<hr />\n\n<img src="image.jpg" alt="access">\n\n<ul>\n\t<li>list item 1</li>\n\t<li>list item 2</li>\n</ul>\n\n<script type="text/javascript">\n\tconst ok = \'yes\';\n</script>\n\n<style type="text/css">\n\t.special::before {\n\t\tcontent: "< & >";\n\t}\n</style>\n\n<![CDATA[\n\tif (a < 0) alert("b");\n\t<not-a-tag>\n]]>\n\n<style>\n\t.some_class {\n\t\tcolor: red;\n\t}\n\t\n\t.dash-class {\n\t\tfont-size: 16px;\n\t}\n\t\n\tp {\n\t\tbox-shadow: 0 0 10px rgba(0, 0, 0, 0.1);\n\t}\n\t\n\t/* comment */\n\t\n\t/*\n\tmulti\n\tline\n\t\n\t<comment>\n\t\n\t*/\n\t\n\t#unique_id {\n\t\tbackground-color: blue;\n\t}\n\t\n\tdiv > p {\n\t\tmargin: 10px;\n\t}\n\t\n\t@media (max-width: 600px) {\n\t\tbody {\n\t\t\tbackground-color: lightblue;\n\t\t}\n\t}\n</style>'; diff --git a/src/lib/grammar_markup.ts b/src/lib/grammar_markup.ts index a1996332..8458dc53 100644 --- a/src/lib/grammar_markup.ts +++ b/src/lib/grammar_markup.ts @@ -1,4 +1,4 @@ -import type {Add_Grammar, Syntax_Styler, Grammar, Grammar_Token} from '$lib/syntax_styler.js'; +import type {Syntax_Styler, Add_Grammar, Grammar, Grammar_Token} from '$lib/syntax_styler.js'; /** * Based on Prism (https://github.com/PrismJS/prism) @@ -14,13 +14,14 @@ export const add_grammar_markup: Add_Grammar = (syntax_styler) => { pattern: //, greedy: true, }, - prolog: { + processing_instruction: { pattern: /<\?[\s\S]+?\?>/, greedy: true, }, // https://www.w3.org/TR/xml/#NT-doctypedecl doctype: { - pattern: //i, // vastly simplified compared to the original implementation, but may be lacking in some cases + pattern: /]*>/i, + greedy: true, }, cdata: { pattern: //i, @@ -87,8 +88,6 @@ export const add_grammar_markup: Add_Grammar = (syntax_styler) => { * @param tag_name - The name of the tag that contains the inlined language. This name will be treated as * case insensitive. * @param lang - The language key. - * @example - * grammar_markup_add_inlined(syntax_styler, 'style', 'css'); */ export const grammar_markup_add_inlined = ( syntax_styler: Syntax_Styler, @@ -140,8 +139,6 @@ export const grammar_markup_add_inlined = ( * @param attr_name - The name of the tag that contains the inlined language. This name will be treated as * case insensitive. * @param lang - The language key. - * @example - * grammar_markup_add_attribute(syntax_styler, 'style', 'css'); */ export const grammar_markup_add_attribute = ( syntax_styler: Syntax_Styler, diff --git a/src/lib/highlight_manager.ts b/src/lib/highlight_manager.ts new file mode 100644 index 00000000..5421dc24 --- /dev/null +++ b/src/lib/highlight_manager.ts @@ -0,0 +1,165 @@ +import {Syntax_Token, type Syntax_Token_Stream} from './syntax_styler.js'; + +export type Highlight_Mode = 'auto' | 'ranges' | 'html'; + +/** + * Check for CSS Highlights API support. + */ +export const supports_css_highlight_api = (): boolean => + !!(globalThis.CSS?.highlights && globalThis.Highlight); // eslint-disable-line @typescript-eslint/no-unnecessary-condition + +/** + * Manages highlights for a single element. + * Tracks ranges per element and only removes its own ranges when clearing. + */ +export class Highlight_Manager { + element_ranges: Map>; + + constructor() { + if (!supports_css_highlight_api()) { + throw Error('CSS Highlights API not supported'); + } + this.element_ranges = new Map(); + } + + /** + * Create ranges for all tokens in the tree + */ + #create_all_ranges( + tokens: Syntax_Token_Stream, + text_node: Node, + ranges_by_type: Map>, + offset: number, + ): number { + let pos = offset; + + for (const token of tokens) { + if (typeof token === 'string') { + // Plain text, just advance position + pos += token.length; + continue; + } + + const start = pos; + const length = this.#get_token_length(token); + const end = start + length; + + // Create range for EVERY token - no complex logic + try { + const range = new Range(); + range.setStart(text_node, start); + range.setEnd(text_node, end); + + const type = token.type; + if (!ranges_by_type.has(type)) { + ranges_by_type.set(type, []); + } + ranges_by_type.get(type)!.push(range); + } catch (e) { + throw new Error(`Failed to create range for ${token.type}: ${e}`); + } + + // Process nested tokens + if (Array.isArray(token.content)) { + this.#create_all_ranges(token.content, text_node, ranges_by_type, start); + } + + pos = end; + } + + return pos; + } + + /** + * Calculate the total text length of a token + */ + #get_token_length(token: Syntax_Token): number { + if (typeof token.content === 'string') { + return token.content.length; + } + + let length = 0; + for (const item of token.content) { + if (typeof item === 'string') { + length += item.length; + } else { + length += this.#get_token_length(item); + } + } + return length; + } + + /** + * Highlight from syntax styler token stream + */ + highlight_from_syntax_tokens(element: Element, tokens: Syntax_Token_Stream): void { + // Find the text node (it might not be firstChild due to Svelte comment nodes) + let text_node: Node | null = null; + for (const node of element.childNodes) { + if (node.nodeType === Node.TEXT_NODE) { + text_node = node; + break; + } + } + + if (!text_node) { + throw new Error('no text node to highlight'); + } + + // Clear existing highlights + this.clear_element_ranges(); + + // Create ranges for all tokens - simple direct traversal + const ranges_by_type: Map> = new Map(); + this.#create_all_ranges(tokens, text_node, ranges_by_type, 0); + + // Apply highlights + for (const [type, ranges] of ranges_by_type) { + // Track ranges for this element + this.element_ranges.set(type, ranges); + + // Get or create the shared highlight + let highlight = CSS.highlights.get(type); + if (!highlight) { + highlight = new globalThis.Highlight(); + CSS.highlights.set(type, highlight); + } + + // Add all ranges to the highlight + for (const range of ranges) { + highlight.add(range); + } + } + } + + /** + * Clear only this element's ranges from highlights + */ + clear_element_ranges(): void { + for (const [name, ranges] of this.element_ranges) { + const highlight = CSS.highlights.get(name); + if (!highlight) { + throw new Error('Expected to find CSS highlight: ' + name); + } + // Remove only this element's ranges + for (const range of ranges) { + highlight.delete(range); + } + + // If highlight is now empty, remove it from registry + if (highlight.size === 0) { + CSS.highlights.delete(name); + } + } + + // Clear our tracking + this.element_ranges.clear(); + } + + /** + * Destroy this manager and clean up + */ + destroy(): void { + this.clear_element_ranges(); + } +} diff --git a/src/lib/samples/all.gen.ts b/src/lib/samples/all.gen.ts new file mode 100644 index 00000000..9f0d683f --- /dev/null +++ b/src/lib/samples/all.gen.ts @@ -0,0 +1,78 @@ +import type {Gen} from '@ryanatkn/gro'; +import {readFileSync} from 'node:fs'; +import {search_fs} from '@ryanatkn/gro/search_fs.js'; +import {basename} from 'node:path'; +import {sample_langs} from '$lib/code_sample.js'; + +export const gen: Gen = ({origin_path}) => { + // Discover all sample files dynamically + const sample_files = search_fs('src/lib/samples', { + file_filter: (path) => /sample_[^/]+\.(ts|css|html|json|svelte)$/.test(path), + }); + + // Create flat structure with lang_variant keys + const samples: Array<{key: string; name: string; lang: string; content: string}> = []; + + for (const file of sample_files) { + // Parse filename: sample_complex.ts → {variant: 'complex', lang: 'ts'} + const filename = basename(file.id); + const match = /sample_([^.]+)\.(.+)$/.exec(filename); + if (!match) continue; + + const [, variant, lang] = match; + const content = readFileSync(file.id, 'utf-8'); + + samples.push({ + key: `${lang}_${variant}`, + name: `${lang}_${variant}`, + lang, + content: escape_string(content), + }); + } + + // Sort using sample_langs order (simple to complex) + samples.sort((a, b) => { + const a_lang_index = sample_langs.indexOf(a.lang as any); + const b_lang_index = sample_langs.indexOf(b.lang as any); + if (a_lang_index !== b_lang_index) { + return a_lang_index - b_lang_index; + } + // If same language, sort by variant name + return a.key.localeCompare(b.key); + }); + + const banner = `// generated by ${origin_path} - DO NOT EDIT OR RISK LOST DATA`; + + // Generate the single export object with all samples + const sample_entries = samples + .map(({key, name, lang, content}) => { + return `\t${key}: { + name: '${name}', + lang: '${lang}', + content: \`${content}\`, + }`; + }) + .join(',\n'); + + // Also generate a type for the samples object + const type_keys = samples.map((s) => `'${s.key}'`).join(' | '); + + return `${banner} + + import {sample_langs, type Code_Sample} from '$lib/code_sample.js'; + + export type Sample_Key = ${type_keys}; + + export const samples: Record = { + ${sample_entries}, + }; + + export {sample_langs}; + + ${banner} + `; +}; + +const escape_string = (str: string): string => { + return str.replace(/\\/g, '\\\\').replace(/`/g, '\\`').replace(/\${/g, '\\${'); +}; diff --git a/src/lib/samples/all.ts b/src/lib/samples/all.ts new file mode 100644 index 00000000..cf83fd57 --- /dev/null +++ b/src/lib/samples/all.ts @@ -0,0 +1,384 @@ +// generated by src/lib/samples/all.gen.ts - DO NOT EDIT OR RISK LOST DATA + +import {sample_langs, type Code_Sample} from '$lib/code_sample.js'; + +export type Sample_Key = + | 'json_complex' + | 'css_complex' + | 'ts_complex' + | 'html_complex' + | 'svelte_complex'; + +export const samples: Record = { + json_complex: { + name: 'json_complex', + lang: 'json', + content: `{ + "string": "a string", + "number": 12345, + "boolean": true, + "null": null, + "empty": "", + "escaped": "quote: \\"test\\" and backslash: \\\\", + "object": { + "array": [1, "b", false], + "strings": ["1", "2", "3"], + "mixed": ["start", 123, true, "middle", null, "end"], + "nested": [["a", "str", ""], {"key": "nested value"}] + } +} +`, + }, + css_complex: { + name: 'css_complex', + lang: 'css', + content: `.some_class { + color: red; +} + +.hypen-class { + font-size: 16px; +} + +p { + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); +} + +/* comment */ + +/* +multi +.line { + i: 100px + + + +@media*/ + +#id { + background-color: blue; +} + +div > p { + margin: 10px; +} + +@media (max-width: 600px) { + body { + background-color: lightblue; + } +} + +/* patterns in strings are not falsely detected */ +.content::before { + content: ' /* not a comment */'; +} + +.attr[title='Click: here'] { + cursor: pointer; +} + +.background /* comment*/ { + background-image: url('data:image/svg+xml...'); +} +`, + }, + ts_complex: { + name: 'ts_complex', + lang: 'ts', + content: `const a = 1; + +const b = 'b'; + +const c = true; + +export type Some_Type = 1 | 'b' | true; + +class D { + d1: string = 'd'; + d2: number; + d3 = $state(false); + + constructor(d2: number) { + this.d2 = d2; + } + + class_method(): string { + return \`Hello, \${this.d1}\`; + } + + instance_method = (): void => { + /* ... */ + this.#private_method(); + // foo + }; + + #private_method() { + throw new Error(\`\${this.d1} + multiline + etc + \`); + } + + protected protected_method(): void { + console.log(new Date()); // eslint-disable-line no-console + } +} + +export {a, b, c, D}; + +// comment + +/* +other comment + +const comment = false; +*/ + +/** + * JSDoc comment + */ + +export interface Some_E { + name: string; + age: number; +} + +export const some_e: Some_E = {name: 'A. H.', age: 100}; + +export function add(x: number, y: number): number { + return x + y; +} + +export const plus = (a: any, b: any): any => a + b; + +// boundary test cases +export const str_with_keywords = 'const class function string'; +export const str_with_comment = '// this is not a comment'; +export const template_with_expr = \`Value: \${1 + 2}\`; + +// regex that looks like comment +export const regex = /\\/\\/.*/g; +export const complex_regex = /^(?:\\/\\*.*?\\*\\/|\\/\\/.*|[^/])+$/; + +// string in comment should not be highlighted as string +// const commented = "this string is in a comment"; +`, + }, + html_complex: { + name: 'html_complex', + lang: 'html', + content: ` + +
+

hello world!

+
+ +

some text

+ + + + + +
+ +
+ +access + +
    +
  • list item 1
    • +
  • list item 2
    • +
+ + + + + + ]]> +`, + }, + svelte_complex: { + name: 'svelte_complex', + lang: 'svelte', + content: ` + + + +

hello {HELLO}!

+ +{#each thing_keys as key (key)} + {@const value = thing[key]} + {value} +{/each} + +{#if c} + +{:else} + (c = !c)}> + {@render children()} + +{/if} + + + +
+

hello world!

+
+ +

+ some text +

+ + + + +{a} +{b} +{bound} +{D} + +
+ +
+ +access + +
    +
  • list item 1
    • +
  • list item 2
    • +
+ + +`, + }, +}; + +export {sample_langs}; + +// generated by src/lib/samples/all.gen.ts - DO NOT EDIT OR RISK LOST DATA diff --git a/src/lib/samples/sample_complex.css b/src/lib/samples/sample_complex.css new file mode 100644 index 00000000..6c19b1a2 --- /dev/null +++ b/src/lib/samples/sample_complex.css @@ -0,0 +1,49 @@ +.some_class { + color: red; +} + +.hypen-class { + font-size: 16px; +} + +p { + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); +} + +/* comment */ + +/* +multi +.line { + i: 100px + + + +@media*/ + +#id { + background-color: blue; +} + +div > p { + margin: 10px; +} + +@media (max-width: 600px) { + body { + background-color: lightblue; + } +} + +/* patterns in strings are not falsely detected */ +.content::before { + content: ' /* not a comment */'; +} + +.attr[title='Click: here'] { + cursor: pointer; +} + +.background /* comment*/ { + background-image: url('data:image/svg+xml...'); +} diff --git a/src/lib/samples/sample_complex.html b/src/lib/samples/sample_complex.html new file mode 100644 index 00000000..c79c452d --- /dev/null +++ b/src/lib/samples/sample_complex.html @@ -0,0 +1,34 @@ + + +
+

hello world!

+
+ +

some text

+ + + + + +
+ +
+ +access + +
    +
  • list item 1
    • +
  • list item 2
    • +
+ + + + + + ]]> diff --git a/src/lib/samples/sample_complex.json b/src/lib/samples/sample_complex.json new file mode 100644 index 00000000..c5039c4a --- /dev/null +++ b/src/lib/samples/sample_complex.json @@ -0,0 +1,14 @@ +{ + "string": "a string", + "number": 12345, + "boolean": true, + "null": null, + "empty": "", + "escaped": "quote: \"test\" and backslash: \\", + "object": { + "array": [1, "b", false], + "strings": ["1", "2", "3"], + "mixed": ["start", 123, true, "middle", null, "end"], + "nested": [["a", "str", ""], {"key": "nested value"}] + } +} diff --git a/src/lib/samples/sample_complex.svelte b/src/lib/samples/sample_complex.svelte new file mode 100644 index 00000000..b2b5692c --- /dev/null +++ b/src/lib/samples/sample_complex.svelte @@ -0,0 +1,168 @@ + + + + +

hello {HELLO}!

+ +{#each thing_keys as key (key)} + {@const value = thing[key]} + {value} +{/each} + +{#if c} + +{:else} + (c = !c)}> + {@render children()} + +{/if} + + + +
+

hello world!

+
+ +

+ some text +

+ + + + +{a} +{b} +{bound} +{D} + +
+ +
+ +access + +
    +
  • list item 1
    • +
  • list item 2
    • +
+ + diff --git a/src/lib/samples/sample_complex.ts b/src/lib/samples/sample_complex.ts new file mode 100644 index 00000000..a8175af3 --- /dev/null +++ b/src/lib/samples/sample_complex.ts @@ -0,0 +1,77 @@ +const a = 1; + +const b = 'b'; + +const c = true; + +export type Some_Type = 1 | 'b' | true; + +class D { + d1: string = 'd'; + d2: number; + d3 = $state(false); + + constructor(d2: number) { + this.d2 = d2; + } + + class_method(): string { + return `Hello, ${this.d1}`; + } + + instance_method = (): void => { + /* ... */ + this.#private_method(); + // foo + }; + + #private_method() { + throw new Error(`${this.d1} + multiline + etc + `); + } + + protected protected_method(): void { + console.log(new Date()); // eslint-disable-line no-console + } +} + +export {a, b, c, D}; + +// comment + +/* +other comment + +const comment = false; +*/ + +/** + * JSDoc comment + */ + +export interface Some_E { + name: string; + age: number; +} + +export const some_e: Some_E = {name: 'A. H.', age: 100}; + +export function add(x: number, y: number): number { + return x + y; +} + +export const plus = (a: any, b: any): any => a + b; + +// boundary test cases +export const str_with_keywords = 'const class function string'; +export const str_with_comment = '// this is not a comment'; +export const template_with_expr = `Value: ${1 + 2}`; + +// regex that looks like comment +export const regex = /\/\/.*/g; +export const complex_regex = /^(?:\/\*.*?\*\/|\/\/.*|[^/])+$/; + +// string in comment should not be highlighted as string +// const commented = "this string is in a comment"; diff --git a/src/lib/syntax_styler.test.ts b/src/lib/syntax_styler.test.ts index e2137649..739cf100 100644 --- a/src/lib/syntax_styler.test.ts +++ b/src/lib/syntax_styler.test.ts @@ -1,42 +1,560 @@ -import {test, assert} from 'vitest'; +import {test, assert, describe} from 'vitest'; +import {Syntax_Styler, tokenize_syntax} from './syntax_styler.js'; +import {add_grammar_js} from './grammar_js.js'; +import {add_grammar_ts} from './grammar_ts.js'; +import {add_grammar_css} from './grammar_css.js'; +import {add_grammar_markup} from './grammar_markup.js'; +import {add_grammar_json} from './grammar_json.js'; +import {add_grammar_svelte} from './grammar_svelte.js'; +import {add_grammar_clike} from './grammar_clike.js'; +import {syntax_styler_global} from './syntax_styler_global.js'; +import {samples} from './samples/all.js'; -import {syntax_styler} from '$lib/index.js'; -import { - sample_html_code, - sample_css_code, - sample_ts_code, - sample_svelte_code, - sample_json_code, -} from '$lib/code_sample_inputs.js'; -import { - styled_html_code, - styled_css_code, - styled_ts_code, - styled_svelte_code, - styled_json_code, -} from '$lib/code_sample_outputs.js'; +// Helper to create a properly initialized syntax styler +const create_styler_with_grammars = (): Syntax_Styler => { + const s = new Syntax_Styler(); + // Load in dependency order + add_grammar_markup(s); + add_grammar_clike(s); + add_grammar_css(s); + add_grammar_js(s); + add_grammar_ts(s); + add_grammar_json(s); + add_grammar_svelte(s); + return s; +}; -test('styles HTML', () => { - const styled = syntax_styler.stylize(sample_html_code, 'html'); - assert.strictEqual(styled, styled_html_code); +describe('grammar mutation behavior', () => { + test('greedy patterns gain global flag on first use', () => { + const syntax_styler = create_styler_with_grammars(); + + // Track patterns before tokenization + const before_flags = new Map(); + for (const lang of ['js', 'ts', 'css']) { + const grammar = syntax_styler.get_lang(lang); + if ((grammar.string as any)?.pattern) { + before_flags.set(lang, (grammar.string as any).pattern.flags); + } + } + + // Tokenize + for (const lang of before_flags.keys()) { + syntax_styler.stylize('test "string"', lang); + } + + // Verify mutation happened (documenting expected behavior) + for (const lang of before_flags.keys()) { + const grammar = syntax_styler.get_lang(lang); + const new_flags = (grammar.string as any).pattern.flags; + assert.ok( + new_flags.includes('g'), + `${lang} string pattern should have global flag after use`, + ); + } + }); + + test('track greedy patterns across all languages', () => { + const syntax_styler = create_styler_with_grammars(); + + // Collect all greedy patterns + const greedy_patterns: Map< + string, + Array<{path: string; pattern: RegExp; source: string; flags: string}> + > = new Map(); + const languages = ['js', 'ts', 'css', 'html', 'json', 'svelte']; + + for (const lang of languages) { + const grammar = syntax_styler.get_lang(lang); + const patterns: Array<{path: string; pattern: RegExp; source: string; flags: string}> = []; + + // Recursively find greedy patterns + const find_greedy = (obj: any, path = ''): void => { + for (const key in obj) { + const val = obj[key]; + if (val && typeof val === 'object' && val.greedy && val.pattern instanceof RegExp) { + patterns.push({ + path: path + key, + pattern: val.pattern, + source: val.pattern.source, + flags: val.pattern.flags, + }); + } else if ( + val && + typeof val === 'object' && + !(val instanceof RegExp) && + !Array.isArray(val) + ) { + // Don't recurse into arrays or regex objects + if (key !== 'rest' && key !== 'inside') { + // Avoid circular refs + find_greedy(val, path + key + '.'); + } + } + } + }; + + find_greedy(grammar); + greedy_patterns.set(lang, patterns); + } + + // Tokenize samples from each language with code that exercises the patterns + for (const lang of languages) { + if (lang === 'js' || lang === 'ts') { + // Include template strings for JS/TS + syntax_styler.stylize('test "string" `template` /* comment */ 123', lang); + } else { + syntax_styler.stylize('test "string" content /* comment */ 123', lang); + } + } + + // Document mutation behavior - patterns gain 'g' flag when needed + for (const [lang, patterns] of greedy_patterns) { + for (const info of patterns) { + // Source remains the same + assert.equal( + info.pattern.source, + info.source, + `${lang}.${info.path} source should not change`, + ); + // But flags may have gained 'g' if they didn't have it + // Exception: patterns that are anchored (^) may not get mutated since they can only match once + // Also: patterns only get mutated when they actually match something during tokenization + if (!info.flags.includes('g')) { + if (info.source.startsWith('^')) { + // Anchored patterns may or may not get g flag - both are valid + // since they can only match once anyway + continue; + } + // Patterns only get mutated when they actually match during tokenization + // If pattern didn't match anything in our test input, it won't be mutated + if (!info.pattern.flags.includes('g')) { + // This is acceptable - pattern wasn't used so wasn't mutated + // Common for patterns like template_string that need specific syntax + continue; + } + assert.ok( + info.pattern.flags.includes('g'), + `${lang}.${info.path} should gain global flag`, + ); + } else { + assert.equal( + info.pattern.flags, + info.flags, + `${lang}.${info.path} flags unchanged (already had g)`, + ); + } + } + } + }); }); -test('styles CSS', () => { - const styled = syntax_styler.stylize(sample_css_code, 'css'); - assert.strictEqual(styled, styled_css_code); +describe('concurrent tokenization safety', () => { + test('multiple tokenizations produce consistent results', () => { + const code1 = 'const a = "test"; function foo() { return /regex/g; }'; + const code2 = 'let b = 123; /* comment */ class Bar {}'; + + // Tokenize in different orders + const result1a = syntax_styler_global.stylize(code1, 'js'); + const result2a = syntax_styler_global.stylize(code2, 'js'); + const result1b = syntax_styler_global.stylize(code1, 'js'); + const result2b = syntax_styler_global.stylize(code2, 'js'); + + // Results should be identical + assert.equal(result1a, result1b, 'Same code should produce same result'); + assert.equal(result2a, result2b, 'Same code should produce same result'); + }); + + test('greedy pattern lastIndex does not leak between tokenizations', () => { + // Code with multiple strings to test greedy pattern + const code = '"string1" + "string2" + "string3"'; + + // Tokenize multiple times + const results: Array = []; + for (let i = 0; i < 10; i++) { + results.push(syntax_styler_global.stylize(code, 'js')); + } + + // All results should be identical + for (let i = 1; i < results.length; i++) { + assert.equal(results[i], results[0], `Result ${i} should match first result`); + } + }); + + test('tokenization with interleaved languages is safe', () => { + const js_code = 'const x = "test";'; + const css_code = '.class { color: red; }'; + const json_code = '{"key": "value"}'; + + // Interleave tokenization of different languages + const js1 = syntax_styler_global.stylize(js_code, 'js'); + const css1 = syntax_styler_global.stylize(css_code, 'css'); + const js2 = syntax_styler_global.stylize(js_code, 'js'); + const json1 = syntax_styler_global.stylize(json_code, 'json'); + const css2 = syntax_styler_global.stylize(css_code, 'css'); + const js3 = syntax_styler_global.stylize(js_code, 'js'); + const json2 = syntax_styler_global.stylize(json_code, 'json'); + + // All results for same input should be identical + assert.equal(js1, js2); + assert.equal(js2, js3); + assert.equal(css1, css2); + assert.equal(json1, json2); + }); +}); + +describe('tokenization correctness', () => { + test('produces consistent output for common patterns', () => { + const samples_to_test = [ + {code: 'const a = 1;', lang: 'js'}, + {code: 'function test() { return "string"; }', lang: 'js'}, + {code: 'class Foo { constructor() {} }', lang: 'js'}, + {code: '// comment\n/* block comment */', lang: 'js'}, + {code: 'const regex = /test.*pattern/gi;', lang: 'js'}, + {code: '`template ${expression} string`', lang: 'js'}, + {code: '.class { color: red; }', lang: 'css'}, + {code: '{"key": "value", "number": 123}', lang: 'json'}, + {code: '
Hello
', lang: 'html'}, + ]; + + for (const sample of samples_to_test) { + const result = syntax_styler_global.stylize(sample.code, sample.lang); + + // Basic checks for valid output + assert.ok( + result.includes('= sample.code.length, + `Output for ${sample.lang} should not lose content`, + ); + } + }); + + test('handles all language samples correctly', () => { + const sample_values = Object.values(samples); + + for (const sample of sample_values) { + // Should not throw and produce valid HTML + const result = syntax_styler_global.stylize(sample.content, sample.lang); + assert.ok(result.includes(' { + const code = 'const x = "test";'; + const grammar = syntax_styler_global.get_lang('js'); + const tokens = tokenize_syntax(code, grammar); + + // Should produce array of tokens + assert.ok(Array.isArray(tokens), 'Should produce array'); + assert.ok(tokens.length > 0, 'Should have tokens'); + + // Verify token structure + let has_keyword = false; + let has_string = false; + for (const token of tokens) { + if (typeof token === 'object' && token.type === 'keyword') has_keyword = true; + if (typeof token === 'object' && token.type === 'string') has_string = true; + } + assert.ok(has_keyword, 'Should have keyword token'); + assert.ok(has_string, 'Should have string token'); + }); }); -test('styles TypeScript', () => { - const styled = syntax_styler.stylize(sample_ts_code, 'ts'); - assert.strictEqual(styled, styled_ts_code); +describe('edge cases', () => { + test('handles empty input', () => { + const result = syntax_styler_global.stylize('', 'js'); + assert.equal(result, ''); + }); + + test('handles whitespace only', () => { + const result = syntax_styler_global.stylize(' \n\t ', 'js'); + assert.equal(result, ' \n\t '); + }); + + test('handles very large files', () => { + // Generate large file + const lines: Array = []; + for (let i = 0; i < 1000; i++) { + lines.push(`const var${i} = "string${i}"; // comment ${i}`); + } + const large_code = lines.join('\n'); + + // Should complete without error + const result = syntax_styler_global.stylize(large_code, 'js'); + assert.ok(result.length > large_code.length, 'Should have added markup'); + assert.ok(result.includes(' { + const nested = '`a${`b${`c${d}`}`}`'; + const result = syntax_styler_global.stylize(nested, 'js'); + assert.ok(result.includes('template'), 'Should recognize template strings'); + }); + + test('handles special regex patterns', () => { + const code = 'const r = /[\\]{}()*+?.\\\\^$|#\\s]/g;'; + const result = syntax_styler_global.stylize(code, 'js'); + assert.ok(result.includes('regex'), 'Should recognize regex'); + }); + + test('handles malformed input gracefully', () => { + // Unclosed string + const malformed1 = 'const x = "unclosed'; + const result1 = syntax_styler_global.stylize(malformed1, 'js'); + assert.ok(result1.length > 0, 'Should handle unclosed string'); + + // Unclosed comment + const malformed2 = '/* unclosed comment'; + const result2 = syntax_styler_global.stylize(malformed2, 'js'); + assert.ok(result2.length > 0, 'Should handle unclosed comment'); + }); }); -test('styles Svelte', () => { - const styled = syntax_styler.stylize(sample_svelte_code, 'svelte'); - assert.strictEqual(styled, styled_svelte_code); +describe('specific grammar patterns', () => { + test('JS template literals with nested expressions', () => { + const code = '`Hello ${name}, you have ${count} ${count === 1 ? "item" : "items"}`'; + const result = syntax_styler_global.stylize(code, 'js'); + assert.ok(result.includes('template'), 'Should recognize template literal'); + assert.ok(result.includes('interpolation'), 'Should recognize interpolation'); + }); + + test('CSS with complex selectors', () => { + const code = '.class:hover > div[data-attr="value"] + p::before { content: "test"; }'; + const result = syntax_styler_global.stylize(code, 'css'); + assert.ok(result.includes('selector'), 'Should recognize selector'); + assert.ok(result.includes('property'), 'Should recognize property'); + }); + + test('TypeScript with generics', () => { + const code = 'function test(x: T): Promise { }'; + const result = syntax_styler_global.stylize(code, 'ts'); + assert.ok(result.includes('function'), 'Should recognize function'); + assert.ok(result.includes('class_name'), 'Should recognize type annotations'); + }); + + test('Svelte with mixed languages', () => { + const code = '\n\n
{x}
'; + const result = syntax_styler_global.stylize(code, 'svelte'); + assert.ok(result.includes('tag'), 'Should recognize HTML tags'); + // Note: embedded language support depends on implementation + }); }); -test('styles JSON', () => { - const styled = syntax_styler.stylize(sample_json_code, 'json'); - assert.strictEqual(styled, styled_json_code); +describe('lastIndex and position management', () => { + test('lastIndex properly reset for greedy patterns', () => { + const syntax_styler = create_styler_with_grammars(); + + // Test with code that has multiple strings at different positions + const code1 = ' "first" "second"'; + const code2 = '"immediate" "next"'; + + const result1 = syntax_styler.stylize(code1, 'js'); + const result2 = syntax_styler.stylize(code2, 'js'); + + // Both should tokenize correctly despite different starting positions + assert.ok(result1.includes('"first"')); + assert.ok(result1.includes('"second"')); + assert.ok(result2.includes('"immediate"')); + assert.ok(result2.includes('"next"')); + }); + + test('greedy patterns work with strings at various positions', () => { + const syntax_styler = create_styler_with_grammars(); + + // Strings at different positions in the text + const code = ` + "start" + "indented" + const x = "inline"; + "far right" + `; + + const result = syntax_styler.stylize(code, 'js'); + + // All strings should be tokenized regardless of position + const string_count = (result.match(//g) || []).length; + assert.equal(string_count, 4, 'Should tokenize all 4 strings'); + }); +}); + +describe('pattern flag edge cases', () => { + test('patterns with existing global flag not double-processed', () => { + const syntax_styler = new Syntax_Styler(); + + // Add a test grammar with patterns that already have flags + syntax_styler.add_lang('test', { + already_global: { + pattern: /test/g, // Already has g flag + greedy: true, + }, + case_insensitive: { + pattern: /test/i, // Has i flag but not g + greedy: true, + }, + multi_flag: { + pattern: /test/gim, // Multiple flags including g + greedy: true, + }, + }); + + const grammar = syntax_styler.get_lang('test'); + const original_global = (grammar.already_global as any).pattern; + const original_insensitive = (grammar.case_insensitive as any).pattern; + const original_multi = (grammar.multi_flag as any).pattern; + + syntax_styler.stylize('test TEST tEsT', 'test'); + + // Patterns with g flag should remain the same object (no mutation needed) + assert.equal( + (grammar.already_global as any).pattern, + original_global, + 'Already global pattern should not be replaced', + ); + assert.equal( + (grammar.multi_flag as any).pattern, + original_multi, + 'Multi-flag pattern should not be replaced', + ); + + // Pattern without g WILL be mutated (expected behavior now) + assert.notEqual( + (grammar.case_insensitive as any).pattern, + original_insensitive, + 'Pattern without g gets replaced with global version', + ); + assert.equal( + (grammar.case_insensitive as any).pattern.flags, + 'gi', // Now has both flags + 'Pattern gains global flag', + ); + }); + + test('non-greedy patterns remain untouched', () => { + const syntax_styler = create_styler_with_grammars(); + const grammar = syntax_styler.get_lang('js'); + + // Get non-greedy patterns (keyword is an array of patterns) + const keyword_patterns = grammar.keyword as Array; + const original_first = keyword_patterns[0]; + const original_second = keyword_patterns[1]; + + syntax_styler.stylize('const let var function class extends', 'js'); + + // Non-greedy patterns should remain exactly the same + assert.equal(keyword_patterns[0], original_first, 'First keyword pattern unchanged'); + assert.equal(keyword_patterns[1], original_second, 'Second keyword pattern unchanged'); + }); +}); + +describe('multiple Syntax_Styler instances', () => { + test('separate instances have independent grammars', () => { + const styler1 = new Syntax_Styler(); + const styler2 = new Syntax_Styler(); + + // Load grammars into both + add_grammar_markup(styler1); + add_grammar_clike(styler1); + add_grammar_js(styler1); + + add_grammar_markup(styler2); + add_grammar_clike(styler2); + add_grammar_js(styler2); + + const grammar1 = styler1.get_lang('js'); + const grammar2 = styler2.get_lang('js'); + + // Store original patterns + const pattern1_original = (grammar1.string as any).pattern; + const pattern2_original = (grammar2.string as any).pattern; + + // Patterns should be different objects (each instance has its own) + assert.notEqual( + pattern1_original, + pattern2_original, + 'Each instance has separate pattern objects', + ); + + // Tokenizing with one shouldn't affect the other + styler1.stylize('"test"', 'js'); + + // Mutation is expected for styler1 + assert.notEqual((grammar1.string as any).pattern, pattern1_original, 'styler1 pattern mutated'); + assert.equal((grammar1.string as any).pattern.flags, 'g', 'styler1 pattern has g flag'); + + // But styler2 should still be untouched (instance isolation) + assert.equal((grammar2.string as any).pattern, pattern2_original, 'styler2 pattern unchanged'); + assert.equal((grammar2.string as any).pattern.flags, '', 'styler2 pattern has no flags'); + }); +}); + +describe('nested tokenization', () => { + test('template strings with nested interpolation', () => { + const syntax_styler = create_styler_with_grammars(); + + // Deeply nested template strings + const code = '`outer ${`inner ${`deepest ${x}`} middle`} end`'; + const result = syntax_styler.stylize(code, 'js'); + + // Should handle all levels of nesting + assert.ok(result.includes('template'), 'Should recognize template strings'); + assert.ok(result.includes('interpolation'), 'Should recognize interpolations'); + + // Count interpolations (should be 3) + const interpolation_count = (result.match(/interpolation/g) || []).length; + assert.ok(interpolation_count >= 3, 'Should handle all nested interpolations'); + }); + + test('regex pattern with special characters', () => { + const syntax_styler = create_styler_with_grammars(); + + // Complex regex that might break if caching is wrong + const code = 'const r = /\\$\\{[^}]+\\}/g;'; // Regex that looks like template syntax + const result = syntax_styler.stylize(code, 'js'); + + assert.ok(result.includes('regex'), 'Should recognize as regex'); + assert.ok(!result.includes('template'), 'Should not confuse with template string'); + }); +}); + +describe('tokenization consistency', () => { + test('same pattern used multiple times in one tokenization', () => { + const syntax_styler = create_styler_with_grammars(); + + // Many strings to ensure pattern is reused + const code = '"a" + "b" + "c" + "d" + "e" + "f" + "g" + "h"'; + const result = syntax_styler.stylize(code, 'js'); + + // All strings should be tokenized + const string_count = (result.match(//g) || []).length; + assert.equal(string_count, 8, 'Should tokenize all 8 strings'); + }); + + test('empty strings handled correctly', () => { + const syntax_styler = create_styler_with_grammars(); + + const code = 'const empty = "";'; + const result = syntax_styler.stylize(code, 'js'); + + assert.ok(result.includes('string'), 'Should tokenize empty string'); + assert.ok(result.includes('""'), 'Should preserve empty string content'); + }); + + test('very long strings handled correctly', () => { + const syntax_styler = create_styler_with_grammars(); + + // Create a very long string to test lastIndex with large values + const long_content = 'x'.repeat(10000); + const code = `const long = "${long_content}";`; + const result = syntax_styler.stylize(code, 'js'); + + assert.ok(result.includes('string'), 'Should tokenize long string'); + assert.ok(result.includes(long_content), 'Should preserve long string content'); + }); }); diff --git a/src/lib/index.ts b/src/lib/syntax_styler_global.ts similarity index 56% rename from src/lib/index.ts rename to src/lib/syntax_styler_global.ts index 0651764e..ac090784 100644 --- a/src/lib/index.ts +++ b/src/lib/syntax_styler_global.ts @@ -7,12 +7,12 @@ import {add_grammar_ts} from '$lib/grammar_ts.js'; import {add_grammar_svelte} from '$lib/grammar_svelte.js'; import {add_grammar_json} from '$lib/grammar_json.js'; -export const syntax_styler = new Syntax_Styler(); +export const syntax_styler_global = new Syntax_Styler(); -add_grammar_markup(syntax_styler); -add_grammar_css(syntax_styler); -add_grammar_clike(syntax_styler); -add_grammar_js(syntax_styler); -add_grammar_ts(syntax_styler); -add_grammar_svelte(syntax_styler); -add_grammar_json(syntax_styler); +add_grammar_markup(syntax_styler_global); +add_grammar_css(syntax_styler_global); +add_grammar_clike(syntax_styler_global); +add_grammar_js(syntax_styler_global); +add_grammar_ts(syntax_styler_global); +add_grammar_svelte(syntax_styler_global); +add_grammar_json(syntax_styler_global); diff --git a/src/lib/theme.css b/src/lib/theme.css index e76f79e7..d39ee2e1 100644 --- a/src/lib/theme.css +++ b/src/lib/theme.css @@ -1,69 +1,246 @@ -/* - -@ryanatkn/fuz_code/theme.css - -This is the default CSS file that styles the output HTML of `stylize`. - -*/ - +/* Comments, punctuation, and processing */ .token.comment, -.token.prolog, +::highlight(comment), +.token.attr_equals, +::highlight(attr_equals), +.token.processing_instruction, +::highlight(processing_instruction), .token.doctype, +::highlight(doctype), .token.cdata, -.token.punctuation { +::highlight(cdata), +.token.punctuation, +::highlight(punctuation) { color: var(--text_color_5); } -.token.namespace { +/* Namespaces */ +.token.namespace, +::highlight(namespace) { color: var(--color_d_5); } +/* Properties, tags, and constants */ .token.property, +::highlight(property), .token.tag, +::highlight(tag), .token.constant, +::highlight(constant), .token.symbol, -.token.deleted { +::highlight(symbol), +.token.deleted, +::highlight(deleted) { color: var(--color_a_5); } -.token.number { +/* Numbers */ +.token.number, +::highlight(number) { color: var(--color_e_5); } -.token.boolean { +/* Booleans */ +.token.boolean, +::highlight(boolean) { color: var(--color_i_5); } +/* Strings, selectors, and values */ .token.selector, -.token.attr_name, +::highlight(selector), .token.string, +::highlight(string), +.token.attr_value, +::highlight(attr_value), .token.char, +::highlight(char), .token.builtin, -.token.inserted { +::highlight(builtin), +.token.inserted, +::highlight(inserted) { color: var(--color_b_5); } +/* Keywords and at-rules */ .token.atrule, -.token.attr_value, -.token.keyword { +::highlight(atrule), +.token.attr_name, +::highlight(attr_name), +.token.keyword, +::highlight(keyword), +.token.null, +::highlight(null) { color: var(--color_f_5); } +/* Functions and classes */ .token.function, -.token.class_name { +::highlight(function), +.token.class_name, +::highlight(class_name) { color: var(--color_d_5); } +/* Regular expressions and variables */ .token.regex, +::highlight(regex), .token.important, -.token.variable { +::highlight(important), +.token.variable, +::highlight(variable) { color: var(--color_e_5); } +/* Font weight modifiers */ .token.important, -.token.bold { +::highlight(important), +.token.bold, +::highlight(bold) { font-weight: bold; } -.token.italic { + +/* Font style modifiers */ +.token.italic, +::highlight(italic) { font-style: italic; } + +/* TODO some of these should probably be used, others are unnecessary */ + +/* TypeScript/JS specific */ +/* .token.template_string, +::highlight(template_string) { + color: var(--color_b_5); +} + +.token.template_punctuation, +::highlight(template_punctuation) { + color: var(--color_a_5); +} + +.token.parameter, +::highlight(parameter) { + color: var(--color_a_5); +} */ + +/* Regex-specific tokens */ +/* .token.regex_delimiter, +::highlight(regex_delimiter) { + color: var(--text_color_5); +} + +.token.regex_flags, +::highlight(regex_flags) { + color: var(--color_e_5); +} + +.token.regex_source, +::highlight(regex_source) { + color: var(--color_e_5); +} + +.token.operator, +::highlight(operator) { + color: var(--text_color_5); +} */ + +/* CSS specific */ +/* .token.color, +::highlight(color), +.token.hexcode, +::highlight(hexcode) { + color: var(--color_e_5); +} + +.token.unit, +::highlight(unit) { + color: var(--color_e_5); +} + +.token.url, +::highlight(url) { + color: var(--color_b_5); +} */ + +/* HTML/XML specific */ +/* .token.entity, +::highlight(entity) { + color: var(--color_e_5); +} + +.token.attr, +::highlight(attr) { + color: var(--color_f_5); +} */ + +/* JSON specific */ +/* .token.property_string, +::highlight(property_string) { + color: var(--color_a_5); +} + +.token.function_variable, +::highlight(function_variable) { + color: var(--color_d_5); +} */ + +/* Additional tokens */ +/* .token.interpolation_punctuation, +::highlight(interpolation_punctuation) { + color: var(--color_a_5); +} + +.token.at, +::highlight(at) { + color: var(--color_f_5); +} + +.token.rule, +::highlight(rule) { + color: var(--color_f_5); +} */ + +/* + * TODO: Additional tokens found in current grammars that need .token rules: + * + * These tokens exist in the grammars but aren't in theme.css. They should be added + * with appropriate colors: + * + * 11. Template strings (JS) + * - .token.template_string / ::highlight(template_string) - same as string: var(--color_b_5) + * - .token.template_punctuation / ::highlight(template_punctuation) - same as string: var(--color_b_5) + * + * 12. Regex components (JS) + * - .token.regex_delimiter / ::highlight(regex_delimiter) - same as punctuation: var(--text_color_5) + * - .token.regex_flags / ::highlight(regex_flags) - same as regex: var(--color_e_5) + * - .token.regex_source / ::highlight(regex_source) - same as regex: var(--color_e_5) + * + * 13. Operators + * - .token.operator / ::highlight(operator) - same as punctuation: var(--text_color_5) + * + * 14. Function variable (JS) + * - .token.function_variable / ::highlight(function_variable) - same as function: var(--color_d_5) + * + * 15. Interpolation punctuation (JS template strings) + * - .token.interpolation_punctuation / ::highlight(interpolation_punctuation) - same as punctuation: var(--text_color_5) + * + * 16. At symbol (TS decorators) + * - .token.at / ::highlight(at) - same as operator/punctuation: var(--text_color_5) + * + * 17. CSS rule + * - .token.rule / ::highlight(rule) - same as atrule: var(--color_f_5) + * + * 18. URL (CSS) + * - .token.url / ::highlight(url) - same as string: var(--color_b_5) + * + * 19. Entity (HTML) + * - .token.entity / ::highlight(entity) - same as number: var(--color_e_5) + * + * Tokens to NOT include: + * - parameter (deleted in grammar_ts) + * - color, hexcode, unit (not in grammars) + * - attr (we use attr_name, attr_value, attr_equals) + * - property_string (doesn't exist) + * - interpolation (container only, no styling needed) + */ diff --git a/src/lib/theme_standalone.css b/src/lib/theme_standalone.css deleted file mode 100644 index d950db3b..00000000 --- a/src/lib/theme_standalone.css +++ /dev/null @@ -1,70 +0,0 @@ -/* - -@ryanatkn/fuz_code/standalone_theme.css - -Same as the default theme but without the dependency on Moss. -Unlike `theme.css`, these colors do not adapt to the `color-scheme`. - -*/ - -.token.comment, -.token.prolog, -.token.doctype, -.token.cdata, -.token.punctuation { - color: #8e7e71; -} - -.token.namespace { - color: light-dark(#6a40bf, #a68cd9); -} - -.token.property, -.token.tag, -.token.constant, -.token.symbol, -.token.deleted { - color: light-dark(#397fc6, #88b2dd); -} - -.token.number { - color: light-dark(#ad9625, #e2cb5a); -} - -.token.boolean { - color: light-dark(#19b3b3, #79ecec); -} - -.token.selector, -.token.attr_name, -.token.string, -.token.char, -.token.builtin, -.token.inserted { - color: light-dark(#298e29, #66c266); -} - -.token.atrule, -.token.attr_value, -.token.keyword { - color: light-dark(#6a3e1b, #b08b6d); -} - -.token.function, -.token.class_name { - color: light-dark(#6a40bf, #a68cd9); -} - -.token.regex, -.token.important, -.token.variable { - color: light-dark(#ad9625, #e2cb5a); -} - -.token.important, -.token.bold { - font-weight: bold; -} -.token.italic { - font-style: italic; -} diff --git a/src/lib/theme_variables.css b/src/lib/theme_variables.css new file mode 100644 index 00000000..2dece325 --- /dev/null +++ b/src/lib/theme_variables.css @@ -0,0 +1,19 @@ +/* + +CSS variables for syntax highlighting when not using Moss. +Import this alongside theme.css if you're not using Moss. + +*/ + +:root { + --text_color_5: light-dark(#8e7e71, #8e7e71); + --color_a_5: light-dark(#397fc6, #88b2dd); + --color_b_5: light-dark(#298e29, #66c266); + --color_c_5: light-dark(#d22d2d, #dd4040); + --color_d_5: light-dark(#6a40bf, #a68cd9); + --color_e_5: light-dark(#ad9625, #e2cb5a); + --color_f_5: light-dark(#6a3e1b, #b08b6d); + --color_g_5: light-dark(#e03e81, #ea7ba9); + --color_h_5: light-dark(#f24e0d, #f67c4c); + --color_i_5: light-dark(#19b3b3, #79ecec); +} diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 5a20b7ad..04e63282 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -7,11 +7,11 @@ import Themed from '@ryanatkn/fuz/Themed.svelte'; import type {Snippet} from 'svelte'; - interface Props { + const { + children, + }: { children: Snippet; - } - - const {children}: Props = $props(); + } = $props(); diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 017c5084..89161041 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -24,16 +24,19 @@ docs. +
+
+ + samples + benchmark + about +
+
-
+
-
- - samples - about -
diff --git a/src/routes/Code_Tome.svelte b/src/routes/Code_Tome.svelte index 0d166b3a..c0091f25 100644 --- a/src/routes/Code_Tome.svelte +++ b/src/routes/Code_Tome.svelte @@ -14,6 +14,7 @@
+

Usage

The @@ -35,6 +36,19 @@ import '@ryanatkn/fuz_code/theme.css'; // add this"

outputs:

+
+

Dependencies

+

+ By default fuz_code depends on my CSS framework Moss. If + you're not using it, import theme_variables.css: +

+ +
+

Svelte support

styles Svelte @@ -69,6 +84,7 @@ import '@ryanatkn/fuz_code/theme.css'; // add this"

+

TypeScript support

supports TypeScript with lang="ts":

@@ -80,6 +96,7 @@ import '@ryanatkn/fuz_code/theme.css'; // add this"
+

Fallback to no styling

Passing lang={'{'}null} disables syntax styling:

@@ -89,28 +106,15 @@ import '@ryanatkn/fuz_code/theme.css'; // add this" all is gray`} />
+

Layout

is a block by default:

ab
ab'} /> -
-

- can be inlined with `} - /> + It can be inlined with `} />

- - diff --git a/src/routes/Footer.svelte b/src/routes/Footer.svelte new file mode 100644 index 00000000..15d195f2 --- /dev/null +++ b/src/routes/Footer.svelte @@ -0,0 +1,7 @@ + + +
+ 🎨 +
diff --git a/src/routes/Tome_Link.svelte b/src/routes/Tome_Link.svelte index 7134a5bb..18290977 100644 --- a/src/routes/Tome_Link.svelte +++ b/src/routes/Tome_Link.svelte @@ -1,10 +1,11 @@ diff --git a/src/routes/benchmark/+page.svelte b/src/routes/benchmark/+page.svelte new file mode 100644 index 00000000..837fc6d3 --- /dev/null +++ b/src/routes/benchmark/+page.svelte @@ -0,0 +1,144 @@ + + +
+

benchmark

+

+ chromium --js-flags="--expose-gc" +

+ +
+

config

+ + + + + + + + + + {#if running} + + {/if} +
+ + {#if running} +
+

Progress

+
+
Testing: {current_test}
+
Progress: {progress} / {total_tests}
+
+ +
+ {/if} + + {#if benchmark_state} + + {/if} + +
+
+ + diff --git a/src/routes/benchmark/Benchmark_Harness.svelte b/src/routes/benchmark/Benchmark_Harness.svelte new file mode 100644 index 00000000..85b119cf --- /dev/null +++ b/src/routes/benchmark/Benchmark_Harness.svelte @@ -0,0 +1,91 @@ + + +{#if current_component && current_props} + {#key iteration_key} + + {/key} +{/if} diff --git a/src/routes/benchmark/Benchmark_Instance.svelte b/src/routes/benchmark/Benchmark_Instance.svelte new file mode 100644 index 00000000..115d63d9 --- /dev/null +++ b/src/routes/benchmark/Benchmark_Instance.svelte @@ -0,0 +1,43 @@ + + +
+ {#if Benchmarked_Component && props} + + {/if} +
diff --git a/src/routes/benchmark/Benchmark_Results.svelte b/src/routes/benchmark/Benchmark_Results.svelte new file mode 100644 index 00000000..30e86c3d --- /dev/null +++ b/src/routes/benchmark/Benchmark_Results.svelte @@ -0,0 +1,107 @@ + + +{#if warnings.length > 0} +
+

⚠️ Warnings

+
    + {#each warnings as warning (warning)} +
  • {warning}
    • + {/each} +
+
+{/if} + +{#if summary} +
+

Summary

+
+ {#each Object.entries(summary) as entry (entry)} + {@const [impl, stats] = entry} +
+

{impl}

+
+ {fmt(stats.avg_mean)}ms + avg time +
+
+ {fmt(stats.avg_ops, 0)} + ops/sec +
+
+ {fmt(stats.avg_cv * 100, 1)}% + CV +
+ {#if impl !== 'syntax_html' && stats.improvement !== undefined} +
+ 0} class:negative={stats.improvement < 0}> + {stats.improvement > 0 ? '+' : ''}{fmt(stats.improvement, 1)}% + + vs baseline +
+ {/if} +
+ {/each} +
+
+{/if} + +{#if results.length > 0} +
+

Results

+ + + + {#each RESULT_COLUMNS as column (column)} + + {/each} + + + + {#each results as result (result)} + + {#each RESULT_COLUMNS as column (column)} + + {/each} + + {/each} + +
{column.header}
+ {column.format(result[column.key], result)} +
+ +
+ + copy results as markdown + +
+ +
+

Legend

+
    +
  • + CV: Coefficient of Variation (std_dev/mean) - lower is better, <15% is + good +
    • +
  • P95: 95th percentile - 95% of measurements were faster than this
    • +
  • Ops/sec: Operations per second (throughput)
    • +
  • Per Item: Time per individual component in batch
    • +
  • Stability: Percentage of iterations with stable system metrics
    • +
+
+
+{/if} diff --git a/src/routes/benchmark/benchmark_dom.ts b/src/routes/benchmark/benchmark_dom.ts new file mode 100644 index 00000000..a1d4154e --- /dev/null +++ b/src/routes/benchmark/benchmark_dom.ts @@ -0,0 +1,20 @@ +// Ensure browser paint has completed +export const ensure_paint = (): Promise => { + return new Promise((resolve) => { + requestAnimationFrame(() => { + requestAnimationFrame(() => { + resolve(); + }); + }); + }); +}; + +// Inter-test cooldown with randomization +export const inter_test_cooldown = async (cooldown_ms: number): Promise => { + // Randomized cooldown to avoid rhythmic patterns + const actual_cooldown = cooldown_ms + Math.random() * cooldown_ms; + await new Promise((resolve) => setTimeout(resolve, actual_cooldown)); + + await ensure_paint(); + await ensure_paint(); +}; diff --git a/src/routes/benchmark/benchmark_fixtures.ts b/src/routes/benchmark/benchmark_fixtures.ts new file mode 100644 index 00000000..612d0e09 --- /dev/null +++ b/src/routes/benchmark/benchmark_fixtures.ts @@ -0,0 +1,44 @@ +import Code from '$lib/Code.svelte'; +import type {Benchmarked_Implementation} from './benchmark_types.js'; +import type {Code_Sample} from '$lib/code_sample.js'; + +/* eslint-disable no-console */ + +// Find appropriate sample for a language +export const find_sample = ( + samples: Record, + lang: string, +): Code_Sample | undefined => + Object.values(samples).find((s) => s.lang === lang && s.name.includes('complex')); + +// Simple pre-generation using content.repeat() +export const pre_generate_large_contents = ( + samples: Record, + langs: ReadonlyArray, + multiplier: number, +): Map => { + const large_contents: Map = new Map(); + + for (const lang of langs) { + const sample = find_sample(samples, lang); + if (!sample) continue; + + const large_content = sample.content.repeat(multiplier); + large_contents.set(lang, large_content); + + // Log the size for debugging + const size_kb = (new TextEncoder().encode(large_content).length / 1024).toFixed(1); + console.log(`[Fixtures] Pre-generated ${lang}: ${size_kb}KB (${multiplier}x)`); + } + + return large_contents; +}; + +export const implementations: Array = [ + {name: 'html', component: Code, mode: 'html'}, + {name: 'ranges', component: Code, mode: 'ranges'}, +]; + +// Languages to test (excluding Svelte for now) +export const languages = ['ts', 'css', 'html', 'json', 'svelte'] as const; +export type Language = (typeof languages)[number]; diff --git a/src/routes/benchmark/benchmark_results.ts b/src/routes/benchmark/benchmark_results.ts new file mode 100644 index 00000000..d7fee6cc --- /dev/null +++ b/src/routes/benchmark/benchmark_results.ts @@ -0,0 +1,94 @@ +import {fmt} from './benchmark_stats.js'; +import type {Benchmark_Result} from './benchmark_types.js'; + +/** + * Column definitions for benchmark results table + */ +export interface Result_Column { + header: string; + key: keyof Benchmark_Result; + format: (value: any, result: Benchmark_Result) => string; + class?: (value: any, result: Benchmark_Result) => string; +} + +export const RESULT_COLUMNS: Array = [ + { + header: 'Language', + key: 'language', + format: (v) => v, + }, + { + header: 'Implementation', + key: 'implementation', + format: (v) => v, + }, + { + header: 'Mean (ms)', + key: 'mean', + format: (v) => fmt(v), + }, + { + header: 'Median (ms)', + key: 'median', + format: (v) => fmt(v), + }, + { + header: 'Std Dev', + key: 'std_dev', + format: (v) => fmt(v), + }, + { + header: 'CV', + key: 'cv', + format: (v) => `${fmt(v * 100, 1)}%`, + class: (v) => (v > 0.15 ? 'warning' : ''), + }, + { + header: 'P95 (ms)', + key: 'p95', + format: (v) => fmt(v), + }, + { + header: 'Ops/sec', + key: 'ops_per_second', + format: (v) => fmt(v, 0), + }, + { + header: 'Outliers', + key: 'outliers', + format: (v, result) => `${v}/${result.raw_sample_size}`, + class: (_v, result) => (result.outlier_ratio > 0.1 ? 'warning' : ''), + }, + { + header: 'Failed', + key: 'failed_iterations', + format: (v) => v.toString(), + class: (v) => (v > 0 ? 'warning' : ''), + }, + { + header: 'Stability', + key: 'stability_ratio', + format: (v) => `${fmt(v * 100, 0)}%`, + class: (v) => (v > 0.9 ? 'good' : ''), + }, +]; + +/** + * Convert benchmark results to a markdown table + */ +export const results_to_markdown = (results: Array): string => { + if (results.length === 0) return ''; + + // Header + const headers = RESULT_COLUMNS.map((col) => col.header); + let markdown = '| ' + headers.join(' | ') + ' |\n'; + markdown += '| ' + headers.map(() => '---').join(' | ') + ' |\n'; + + // Add rows + for (const result of results) { + const row = RESULT_COLUMNS.map((col) => col.format(result[col.key], result)); + markdown += '| ' + row.join(' | ') + ' |\n'; + } + + return markdown; +}; diff --git a/src/routes/benchmark/benchmark_runner.ts b/src/routes/benchmark/benchmark_runner.ts new file mode 100644 index 00000000..01f98af8 --- /dev/null +++ b/src/routes/benchmark/benchmark_runner.ts @@ -0,0 +1,263 @@ +import type { + Benchmarked_Implementation, + Measurement_Data, + Benchmark_Config, + Benchmark_Result, + Progress_Callbacks, + Benchmark_State, + Benchmark_Harness_Controller, + Benchmark_Component_Props, +} from './benchmark_types.js'; +import type {Code_Sample} from '$lib/code_sample.js'; +import {inter_test_cooldown} from './benchmark_dom.js'; +import { + check_system_stability, + get_instability_reason, + extended_cooldown, +} from './benchmark_stability.js'; +import {implementations, languages, pre_generate_large_contents} from './benchmark_fixtures.js'; +import {analyze_results, calculate_summary, check_high_variance} from './benchmark_stats.js'; + +/* eslint-disable no-console */ +/* eslint-disable no-await-in-loop */ + +// Timing validation constants +const MIN_VALID_TIMING_MS = 0.01; +const MAX_VALID_TIMING_MS = 60000; +const SUSPICIOUS_MEAN_MS = 0.1; +const HIGH_OUTLIER_RATIO = 0.2; + +// Warmup phase using harness with large content +export const warmup_phase = async ( + impl: Benchmarked_Implementation, + content: string, + lang: string, + warmup_count: number, + cooldown_ms: number, + harness: Benchmark_Harness_Controller, +): Promise => { + for (let i = 0; i < warmup_count; i++) { + const props: Benchmark_Component_Props = {content, lang}; + if (impl.mode !== null) { + props.mode = impl.mode; + } + + await harness.run_iteration(impl.component, props); + + await inter_test_cooldown(cooldown_ms); + } +}; + +// Run measurements for a single implementation using harness with large content +export const measurement_phase = async ( + impl: Benchmarked_Implementation, + content: string, + lang: string, + config: Benchmark_Config, + recent_timings: Array, + harness: Benchmark_Harness_Controller, + on_progress?: () => void, + should_stop?: () => boolean, +): Promise => { + const times: Array = []; + const stability_checks = []; + const timestamps = []; + + for (let i = 0; i < config.iterations; i++) { + // Check for stop signal + if (should_stop?.()) { + console.log('[Measurement] Stopped by user'); + break; + } + + console.log(`[Measurement] Iteration ${i + 1}/${config.iterations}`); + + // Cooldown between tests + await inter_test_cooldown(config.cooldown_ms); + + const stability = await check_system_stability(recent_timings); + stability_checks.push(stability); + + if (!stability.is_stable) { + const reason = get_instability_reason(stability); + console.log(`[Measurement] System unstable: ${reason}, extended cooldown...`); + await extended_cooldown(reason); + } + + const props: Benchmark_Component_Props = {content, lang}; + if (impl.mode !== null) { + props.mode = impl.mode; + } + + console.log(`[Measurement] Running iteration ${i + 1}...`); + + try { + const elapsed = await harness.run_iteration(impl.component, props); + + // Validate the timing + if (elapsed <= 0) { + console.warn(`[Measurement] Suspicious timing (${elapsed}ms) - marking as failed`); + times.push(NaN); + timestamps.push(Date.now()); + } else if (elapsed < MIN_VALID_TIMING_MS) { + console.warn( + `[Measurement] Suspiciously fast timing (${elapsed}ms) - possible no-op render`, + ); + times.push(elapsed); + timestamps.push(Date.now()); + recent_timings.push(elapsed); + } else if (elapsed > MAX_VALID_TIMING_MS) { + console.warn(`[Measurement] Extremely slow timing (${elapsed}ms) - possible hang`); + times.push(NaN); + timestamps.push(Date.now()); + } else { + console.log(`[Measurement] Iteration ${i + 1} complete: ${elapsed.toFixed(2)}ms`); + times.push(elapsed); + timestamps.push(Date.now()); + recent_timings.push(elapsed); + } + } catch (error) { + console.error(`[Measurement] Iteration ${i + 1} failed:`, error); + // Continue with next iteration instead of failing entire test + // Record a null/NaN to indicate failure + times.push(NaN); + timestamps.push(Date.now()); + } + + if (on_progress) { + on_progress(); + } + + if (globalThis.gc) { + globalThis.gc(); + } + } + + return {times, stability_checks, timestamps}; +}; + +// Run complete benchmark suite +export const run_all_benchmarks = async ( + samples: Record, + config: Benchmark_Config, + harness: Benchmark_Harness_Controller, + callbacks?: Progress_Callbacks, + custom_implementations?: Array, + custom_languages?: Array, +): Promise => { + const impls = custom_implementations || implementations; + const langs = custom_languages || languages; + const results: Array = []; + const warnings: Array = []; + const recent_timings: Array = []; + + const total_tests = impls.length * langs.length; + const total_iterations = total_tests * config.iterations; + let current_progress = 0; + + // Pre-generate all large content before benchmarking + console.log(`[Runner] Pre-generating large content (${config.content_multiplier}x)...`); + const contents = pre_generate_large_contents(samples, langs, config.content_multiplier); + console.log('[Runner] Pre-generation complete'); + + for (const lang of langs) { + // Check for stop signal + if (callbacks?.should_stop?.()) { + console.log('[Runner] Benchmark stopped by user'); + break; + } + + // Get pre-generated large content for this language + const content = contents.get(lang); + + if (!content) { + warnings.push(`No complex sample found for ${lang}`); + continue; + } + + for (const impl of impls) { + // Check for stop signal + if (callbacks?.should_stop?.()) { + console.log('[Runner] Benchmark stopped by user'); + break; + } + const test_name = `${impl.name} / ${lang}`; + console.log(`\n[Runner] Starting test: ${test_name}`); + + if (callbacks?.on_test_start) { + callbacks.on_test_start(test_name); + } + + try { + // Warmup with large content + await warmup_phase(impl, content, lang, config.warmup_count, config.cooldown_ms, harness); + + // Measurement with progress tracking and large content + const measurement_data = await measurement_phase( + impl, + content, + lang, + config, + recent_timings, + harness, + () => { + current_progress++; + if (callbacks?.on_progress) { + callbacks.on_progress(current_progress, total_iterations); + } + }, + callbacks?.should_stop, + ); + + // Analysis + const stats = analyze_results(measurement_data); + console.log( + `[Runner] ${test_name} complete - mean: ${stats.mean.toFixed(2)}ms, ops/sec: ${stats.ops_per_second.toFixed(2)}`, + ); + + // Check for suspicious results + if (stats.failed_iterations > 0) { + warnings.push(`${test_name}: ${stats.failed_iterations} failed iterations`); + } + if (stats.mean < SUSPICIOUS_MEAN_MS) { + warnings.push(`${test_name}: Suspiciously fast mean time (${stats.mean.toFixed(3)}ms)`); + } + if (stats.outlier_ratio > HIGH_OUTLIER_RATIO) { + warnings.push( + `${test_name}: High outlier ratio (${(stats.outlier_ratio * 100).toFixed(1)}%)`, + ); + } + + results.push({ + implementation: impl.name, + language: lang, + ...stats, + }); + + if (callbacks?.on_test_complete) { + callbacks.on_test_complete(); + } + } catch (error) { + console.error(`Error testing ${impl.name}/${lang}:`, error); + warnings.push(`Failed: ${impl.name}/${lang}`); + } + } + } + + // Calculate summary statistics + const summary = calculate_summary(results); + + // Check for high variance + const variance_warnings = check_high_variance(results); + warnings.push(...variance_warnings); + + // Add stopped message if applicable + if (callbacks?.should_stop?.()) { + warnings.push('Benchmark was stopped by user'); + } + + // Clean up harness + await harness.cleanup(); + + return {results, warnings, summary}; +}; diff --git a/src/routes/benchmark/benchmark_stability.ts b/src/routes/benchmark/benchmark_stability.ts new file mode 100644 index 00000000..0e7a4acf --- /dev/null +++ b/src/routes/benchmark/benchmark_stability.ts @@ -0,0 +1,59 @@ +import type {Stability_Check} from './benchmark_types.js'; +const MIN_SAMPLES_FOR_JITTER = 5; +const RECENT_SAMPLE_COUNT = 10; +const MAX_ACCEPTABLE_LAG_MS = 5; +const MAX_MEMORY_PRESSURE = 0.9; +const MAX_JITTER_RATIO = 2; + +const COOLDOWN_TIMES: Record = { + high_lag: 200, + memory_pressure: 500, + high_jitter: 300, + unknown: 300, +}; +export const calculate_timing_jitter = (recent_timings: Array): number => { + if (recent_timings.length < MIN_SAMPLES_FOR_JITTER) return 0; + + const recent = recent_timings.slice(-RECENT_SAMPLE_COUNT); + const mean = recent.reduce((a, b) => a + b, 0) / recent.length; + const variance = recent.reduce((sum, val) => sum + (val - mean) ** 2, 0) / recent.length; + const std_dev = Math.sqrt(variance); + + return std_dev / mean; +}; +export const check_system_stability = async ( + recent_timings: Array, +): Promise => { + const lag_start = performance.now(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const lag = performance.now() - lag_start; + + let memory_pressure = 0; + // @ts-ignore + if (typeof performance !== 'undefined' && performance.memory) { + // @ts-ignore + memory_pressure = performance.memory.usedJSHeapSize / performance.memory.jsHeapSizeLimit; + } + + const jitter = calculate_timing_jitter(recent_timings); + + const is_stable = + lag < MAX_ACCEPTABLE_LAG_MS && + memory_pressure < MAX_MEMORY_PRESSURE && + jitter < MAX_JITTER_RATIO; + + return {is_stable, lag, memory_pressure, jitter}; +}; +export const get_instability_reason = (stability: Stability_Check): string => { + if (stability.lag > MAX_ACCEPTABLE_LAG_MS) return 'high_lag'; + if (stability.jitter > MAX_JITTER_RATIO) return 'high_jitter'; + if (stability.memory_pressure && stability.memory_pressure > MAX_MEMORY_PRESSURE) + return 'memory_pressure'; + return 'unknown'; +}; +export const extended_cooldown = async (reason: string): Promise => { + console.log(`System instability: ${reason}, waiting...`); // eslint-disable-line no-console + await new Promise((resolve) => + setTimeout(resolve, COOLDOWN_TIMES[reason] || COOLDOWN_TIMES.unknown), + ); +}; diff --git a/src/routes/benchmark/benchmark_stats.ts b/src/routes/benchmark/benchmark_stats.ts new file mode 100644 index 00000000..bf6e369e --- /dev/null +++ b/src/routes/benchmark/benchmark_stats.ts @@ -0,0 +1,281 @@ +// Statistical analysis functions for benchmark results + +import type { + Benchmark_Result, + Benchmark_Stats, + Measurement_Data, + Summary_Stats, +} from './benchmark_types.js'; + +// Statistical constants +const QUARTILE_Q1 = 0.25; +const QUARTILE_Q3 = 0.75; +const IQR_MULTIPLIER = 1.5; +const MAD_Z_SCORE_THRESHOLD = 3.5; +const MAD_Z_SCORE_EXTREME = 5.0; +const MAD_CONSTANT = 0.6745; // For normal distribution approximation +const OUTLIER_RATIO_HIGH = 0.3; +const OUTLIER_RATIO_EXTREME = 0.4; +const OUTLIER_KEEP_RATIO = 0.8; +const PERCENTILE_95 = 0.95; +const PERCENTILE_99 = 0.99; +const CONFIDENCE_INTERVAL_Z = 1.96; +const MS_PER_SECOND = 1000; +const MIN_SAMPLE_SIZE = 3; + +// Calculate median +const calculate_median = (sorted_array: Array): number => { + const mid = Math.floor(sorted_array.length / 2); + return sorted_array.length % 2 === 0 + ? (sorted_array[mid - 1] + sorted_array[mid]) / 2 + : sorted_array[mid]; +}; + +// Outlier detection using MAD (Median Absolute Deviation) method +export const detect_outliers = ( + times: Array, +): { + cleaned_times: Array; + outliers: Array; +} => { + if (times.length < MIN_SAMPLE_SIZE) { + return {cleaned_times: times, outliers: []}; + } + + const sorted = [...times].sort((a, b) => a - b); + const median = calculate_median(sorted); + + // Calculate MAD (Median Absolute Deviation) + const deviations = times.map((t) => Math.abs(t - median)); + const sorted_deviations = [...deviations].sort((a, b) => a - b); + const mad = calculate_median(sorted_deviations); + + if (mad === 0) { + const q1 = sorted[Math.floor(sorted.length * QUARTILE_Q1)]; + const q3 = sorted[Math.floor(sorted.length * QUARTILE_Q3)]; + const iqr = q3 - q1; + + if (iqr === 0) { + return {cleaned_times: times, outliers: []}; + } + + const lower_bound = q1 - IQR_MULTIPLIER * iqr; + const upper_bound = q3 + IQR_MULTIPLIER * iqr; + + const cleaned_times: Array = []; + const outliers: Array = []; + + for (const time of times) { + if (time < lower_bound || time > upper_bound) { + outliers.push(time); + } else { + cleaned_times.push(time); + } + } + + return {cleaned_times, outliers}; + } + + // Use modified Z-score with MAD + const cleaned_times: Array = []; + const outliers: Array = []; + + for (const time of times) { + const modified_z_score = (MAD_CONSTANT * (time - median)) / mad; + if (Math.abs(modified_z_score) > MAD_Z_SCORE_THRESHOLD) { + outliers.push(time); + } else { + cleaned_times.push(time); + } + } + + // If too many outliers, increase threshold and try again + if (outliers.length > times.length * OUTLIER_RATIO_HIGH) { + cleaned_times.length = 0; + outliers.length = 0; + + for (const time of times) { + const modified_z_score = (MAD_CONSTANT * (time - median)) / mad; + if (Math.abs(modified_z_score) > MAD_Z_SCORE_EXTREME) { + outliers.push(time); + } else { + cleaned_times.push(time); + } + } + + if (outliers.length > times.length * OUTLIER_RATIO_EXTREME) { + // Sort by distance from median and keep closest values + const with_distances = times.map((t) => ({ + time: t, + distance: Math.abs(t - median), + })); + with_distances.sort((a, b) => a.distance - b.distance); + + const keep_count = Math.floor(times.length * OUTLIER_KEEP_RATIO); + cleaned_times.length = 0; + outliers.length = 0; + + for (let i = 0; i < with_distances.length; i++) { + if (i < keep_count) { + cleaned_times.push(with_distances[i].time); + } else { + outliers.push(with_distances[i].time); + } + } + } + } + + return {cleaned_times, outliers}; +}; + +// Statistical analysis +export const analyze_results = (data: Measurement_Data): Benchmark_Stats => { + // Filter out invalid values (failed iterations) + const valid_times: Array = []; + let failed_count = 0; + + for (const t of data.times) { + if (!isNaN(t) && isFinite(t) && t > 0) { + valid_times.push(t); + } else { + failed_count++; + } + } + + // If no valid times, return empty stats + if (valid_times.length === 0) { + return { + mean: NaN, + median: NaN, + std_dev: NaN, + min: NaN, + max: NaN, + p95: NaN, + p99: NaN, + cv: NaN, + confidence_interval: [NaN, NaN], + outliers: 0, + outlier_ratio: 0, + sample_size: 0, + raw_sample_size: data.times.length, + stability_ratio: 0, + unstable_iterations: data.times.length, + ops_per_second: 0, + failed_iterations: failed_count, + }; + } + + const {cleaned_times, outliers} = detect_outliers(valid_times); + const final_sorted = [...cleaned_times].sort((a, b) => a - b); + + const mean = cleaned_times.reduce((a, b) => a + b, 0) / cleaned_times.length; + const median = calculate_median(final_sorted); + + const variance = + cleaned_times.reduce((sum, val) => sum + (val - mean) ** 2, 0) / cleaned_times.length; + const std_dev = Math.sqrt(variance); + + const min = final_sorted[0]; + const max = final_sorted[final_sorted.length - 1]; + const p95 = final_sorted[Math.floor(final_sorted.length * PERCENTILE_95)]; + const p99 = final_sorted[Math.floor(final_sorted.length * PERCENTILE_99)]; + + const cv = std_dev / mean; + + const se = std_dev / Math.sqrt(cleaned_times.length); + const ci_margin = CONFIDENCE_INTERVAL_Z * se; + const ci_lower = mean - ci_margin; + const ci_upper = mean + ci_margin; + + // Stability analysis + const unstable_count = data.stability_checks.filter((s) => !s.is_stable).length; + const stability_ratio = 1 - unstable_count / data.stability_checks.length; + + // Throughput calculation (operations per second) + const ops_per_second = mean > 0 ? MS_PER_SECOND / mean : 0; + + return { + mean, + median, + std_dev, + min, + max, + p95, + p99, + cv, + confidence_interval: [ci_lower, ci_upper], + outliers: outliers.length, + outlier_ratio: outliers.length / valid_times.length, + sample_size: cleaned_times.length, + raw_sample_size: data.times.length, + stability_ratio, + unstable_iterations: unstable_count, + ops_per_second, + failed_iterations: failed_count, + }; +}; + +// Calculate summary across all tests +export const calculate_summary = ( + results: Array, +): Record => { + const by_impl: Record> = {}; + + // Group results by implementation + for (const result of results) { + if (!(by_impl as Record | undefined>)[result.implementation]) { + by_impl[result.implementation] = []; + } + by_impl[result.implementation].push(result); + } + + const summary: Record = {}; + + // Calculate averages for each implementation + for (const [impl, impl_results] of Object.entries(by_impl)) { + const mean_times = impl_results.map((r) => r.mean); + const avg_mean = mean_times.reduce((a, b) => a + b, 0) / mean_times.length; + const avg_ops = + impl_results.map((r) => r.ops_per_second).reduce((a, b) => a + b, 0) / impl_results.length; + const avg_cv = impl_results.map((r) => r.cv).reduce((a, b) => a + b, 0) / impl_results.length; + + summary[impl] = { + avg_mean, + avg_ops, + avg_cv, + languages: impl_results.length, + }; + } + + // Calculate relative performance + const baseline_mean = summary.syntax_html.avg_mean || 1; + for (const impl of Object.keys(summary)) { + summary[impl].relative_speed = baseline_mean / summary[impl].avg_mean; + summary[impl].improvement = (summary[impl].relative_speed - 1) * 100; + } + + return summary; +}; + +// Check for high variance +export const check_high_variance = ( + results: Array, + threshold = 0.15, +): Array => { + const warnings: Array = []; + + for (const result of results) { + if (result.cv > threshold) { + warnings.push( + `High variance for ${result.implementation}/${result.language}: CV=${(result.cv * 100).toFixed(1)}%`, + ); + } + } + + return warnings; +}; + +// Format number for display +export const fmt = (n: number, decimals = 2): string => { + return n.toFixed(decimals); +}; diff --git a/src/routes/benchmark/benchmark_types.ts b/src/routes/benchmark/benchmark_types.ts new file mode 100644 index 00000000..0bbcf0dc --- /dev/null +++ b/src/routes/benchmark/benchmark_types.ts @@ -0,0 +1,93 @@ +// Type definitions for benchmarking system + +export interface Benchmark_Component_Props { + content: string; + lang: string; + mode?: 'html' | 'ranges' | 'auto'; +} + +export interface Benchmark_Config { + iterations: number; + warmup_count: number; + cooldown_ms: number; + content_multiplier: number; +} + +import type {Component} from 'svelte'; + +export interface Benchmarked_Implementation { + name: string; + component: Component; + mode: 'html' | 'ranges' | 'auto' | null; +} + +export interface Stability_Check { + is_stable: boolean; + lag: number; + memory_pressure?: number; + jitter: number; +} + +export interface Measurement_Data { + times: Array; + stability_checks: Array; + timestamps: Array; +} + +export interface Benchmark_Stats { + mean: number; + median: number; + std_dev: number; + min: number; + max: number; + p95: number; + p99: number; + cv: number; + confidence_interval: [number, number]; + outliers: number; + outlier_ratio: number; + sample_size: number; + raw_sample_size: number; + stability_ratio: number; + unstable_iterations: number; + ops_per_second: number; + failed_iterations: number; +} + +export interface Benchmark_Result extends Benchmark_Stats { + implementation: string; + language: string; +} + +export interface Summary_Stats { + avg_mean: number; + avg_ops: number; + avg_cv: number; + languages: number; + relative_speed?: number; + improvement?: number; +} + +// Benchmark harness controller interface +export interface Benchmark_Harness_Controller { + run_iteration: ( + component: Component, + props: Benchmark_Component_Props, + ) => Promise; + cleanup: () => Promise; +} + +// Progress tracking callbacks +export interface Progress_Callbacks { + on_progress?: (current: number, total: number) => void; + on_test_start?: (test: string) => void; + on_test_complete?: () => void; + should_stop?: () => boolean; +} + +// Benchmark runner state +export interface Benchmark_State { + results: Array; + warnings: Array; + summary: Record | null; +} diff --git a/src/routes/moss.css b/src/routes/moss.css index 722146ee..ed974e45 100644 --- a/src/routes/moss.css +++ b/src/routes/moss.css @@ -2,11 +2,11 @@ /* * * File statistics: - * - Total files in filer: 262 - * - External dependencies: 234 - * - Internal project files: 28 - * - Files processed (passed filter): 232 - * - Files with CSS classes: 19 + * - Total files in filer: 285 + * - External dependencies: 235 + * - Internal project files: 50 + * - Files processed (passed filter): 250 + * - Files with CSS classes: 23 * - Unique classes found: 41 */ @@ -34,12 +34,31 @@ max-width: var(--distance_md); } +.width_lg { + width: 100%; + max-width: var(--distance_lg); +} + /* A panel is a box embedded into the page, useful for visually isolating content. */ .panel { border-radius: var(--border_radius_xs); background-color: var(--panel_bg, var(--fg_1)); } +/* TODO other button variants? */ +/* TODO this is slightly strange that it doesn't use --icon_size */ +/* These are used as modifiers to buttons, and so they use `:where` so they cascade. */ +.icon_button { + width: var(--input_height); + height: var(--input_height); + min-width: var(--input_height); + min-height: var(--input_height); + flex-shrink: 0; + line-height: 1; + font-weight: 900; + padding: 0; +} + .chip { font-weight: 600; padding-left: var(--space_xs); @@ -84,8 +103,12 @@ a.chip { .flex_wrap { flex-wrap: wrap; } -.justify_content_space_around { - justify-content: space-around; +.justify_content_center { + justify-content: center; +} +.font_size_sm { + font-size: var(--font_size_sm); + --font_size: var(--font_size_sm); } .font_size_xl2 { font-size: var(--font_size_xl2); @@ -94,6 +117,10 @@ a.chip { .text_align_center { text-align: center; } +.font_weight_400 { + font-weight: 400; + --font_weight: 400; +} .bg { background-color: var(--bg); } @@ -120,6 +147,9 @@ a.chip { .p_lg { padding: var(--space_lg); } +.p_xl2 { + padding: var(--space_xl2); +} .px_xl { padding-left: var(--space_xl); padding-right: var(--space_xl); @@ -128,18 +158,38 @@ a.chip { padding-top: var(--space_sm); padding-bottom: var(--space_sm); } +.py_xl5 { + padding-top: var(--space_xl5); + padding-bottom: var(--space_xl5); +} .mt_0 { margin-top: 0; } +.mt_md { + margin-top: var(--space_md); +} .mt_xl5 { margin-top: var(--space_xl5); } .mr_xs { margin-right: var(--space_xs); } +.mb_0 { + margin-bottom: 0; +} .mb_lg { margin-bottom: var(--space_lg); } +.mb_xl5 { + margin-bottom: var(--space_xl5); +} +.mx_auto { + margin-left: auto; + margin-right: auto; +} +.gap_sm { + gap: var(--space_sm); +} .gap_xl3 { gap: var(--space_xl3); } diff --git a/src/routes/package.ts b/src/routes/package.ts index 6dd2b4cf..9c0b8e86 100644 --- a/src/routes/package.ts +++ b/src/routes/package.ts @@ -25,6 +25,9 @@ export const package_json: Package_Json = { test: 'gro test', preview: 'vite preview', deploy: 'gro deploy', + benchmark: 'gro run benchmark/run_benchmarks.ts', + 'benchmark-compare': 'gro run benchmark/compare/run_compare.ts', + 'update-generated-fixtures': 'gro src/fixtures/update', }, type: 'module', engines: {node: '>=22.15'}, @@ -35,7 +38,7 @@ export const package_json: Package_Json = { '@ryanatkn/belt': '^0.34.1', '@ryanatkn/eslint-config': '^0.8.0', '@ryanatkn/fuz': '^0.145.0', - '@ryanatkn/gro': '^0.164.1', + '@ryanatkn/gro': '^0.165.0', '@ryanatkn/moss': '^0.33.0', '@sveltejs/adapter-static': '^3.0.9', '@sveltejs/kit': '^2.37.1', @@ -44,10 +47,12 @@ export const package_json: Package_Json = { '@types/node': '^24.3.1', eslint: '^9.35.0', 'eslint-plugin-svelte': '^3.12.1', + 'esm-env': '^1.2.2', prettier: '^3.6.2', 'prettier-plugin-svelte': '^3.4.0', svelte: '^5.38.7', 'svelte-check': '^4.3.1', + tinybench: '^5.0.1', tslib: '^2.8.1', typescript: '^5.9.2', 'typescript-eslint': '^8.42.0', @@ -64,37 +69,15 @@ export const package_json: Package_Json = { sideEffects: ['**/*.css'], files: ['dist', 'src/lib/**/*.ts', '!src/lib/**/*.test.*', '!dist/**/*.test.*'], exports: { - '.': {types: './dist/index.d.ts', default: './dist/index.js'}, './package.json': './package.json', - './code_sample_inputs.js': { - types: './dist/code_sample_inputs.d.ts', - default: './dist/code_sample_inputs.js', - }, - './code_sample_outputs.js': { - types: './dist/code_sample_outputs.d.ts', - default: './dist/code_sample_outputs.js', - }, - './Code.svelte': { - types: './dist/Code.svelte.d.ts', - svelte: './dist/Code.svelte', - default: './dist/Code.svelte', - }, - './grammar_clike.js': {types: './dist/grammar_clike.d.ts', default: './dist/grammar_clike.js'}, - './grammar_css.js': {types: './dist/grammar_css.d.ts', default: './dist/grammar_css.js'}, - './grammar_js.js': {types: './dist/grammar_js.d.ts', default: './dist/grammar_js.js'}, - './grammar_json.js': {types: './dist/grammar_json.d.ts', default: './dist/grammar_json.js'}, - './grammar_markup.js': { - types: './dist/grammar_markup.d.ts', - default: './dist/grammar_markup.js', - }, - './grammar_svelte.js': { - types: './dist/grammar_svelte.d.ts', - default: './dist/grammar_svelte.js', - }, - './grammar_ts.js': {types: './dist/grammar_ts.d.ts', default: './dist/grammar_ts.js'}, - './syntax_styler.js': {types: './dist/syntax_styler.d.ts', default: './dist/syntax_styler.js'}, - './theme_standalone.css': {default: './dist/theme_standalone.css'}, - './theme.css': {default: './dist/theme.css'}, + './*.js': {types: './dist/*.d.ts', default: './dist/*.js'}, + './*.svelte': { + types: './dist/*.svelte.d.ts', + svelte: './dist/*.svelte', + default: './dist/*.svelte', + }, + './*.json': {types: './dist/*.json.d.ts', default: './dist/*.json'}, + './*.css': {default: './dist/*.css'}, }, } as any; @@ -102,30 +85,14 @@ export const src_json: Src_Json = { name: '@ryanatkn/fuz_code', version: '0.24.0', modules: { - '.': {path: 'index.ts', declarations: [{name: 'syntax_styler', kind: 'variable'}]}, - './package.json': {path: 'package.json', declarations: [{name: 'default', kind: 'json'}]}, - './code_sample_inputs.js': { - path: 'code_sample_inputs.ts', - declarations: [ - {name: 'sample_json_code', kind: 'variable'}, - {name: 'sample_html_code', kind: 'variable'}, - {name: 'sample_css_code', kind: 'variable'}, - {name: 'sample_ts_code', kind: 'variable'}, - {name: 'sample_svelte_code', kind: 'variable'}, - {name: 'samples', kind: 'variable'}, - ], - }, - './code_sample_outputs.js': { - path: 'code_sample_outputs.ts', + './code_sample.js': { + path: 'code_sample.ts', declarations: [ - {name: 'styled_json_code', kind: 'variable'}, - {name: 'styled_html_code', kind: 'variable'}, - {name: 'styled_css_code', kind: 'variable'}, - {name: 'styled_ts_code', kind: 'variable'}, - {name: 'styled_svelte_code', kind: 'variable'}, + {name: 'Code_Sample', kind: 'type'}, + {name: 'sample_langs', kind: 'variable'}, + {name: 'Sample_Lang', kind: 'type'}, ], }, - './Code.svelte': {path: 'Code.svelte', declarations: [{name: 'default', kind: 'component'}]}, './grammar_clike.js': { path: 'grammar_clike.ts', declarations: [{name: 'add_grammar_clike', kind: 'function'}], @@ -161,6 +128,18 @@ export const src_json: Src_Json = { path: 'grammar_ts.ts', declarations: [{name: 'add_grammar_ts', kind: 'function'}], }, + './highlight_manager.js': { + path: 'highlight_manager.ts', + declarations: [ + {name: 'Highlight_Mode', kind: 'type'}, + {name: 'supports_css_highlight_api', kind: 'function'}, + {name: 'Highlight_Manager', kind: 'class'}, + ], + }, + './syntax_styler_global.js': { + path: 'syntax_styler_global.ts', + declarations: [{name: 'syntax_styler_global', kind: 'variable'}], + }, './syntax_styler.js': { path: 'syntax_styler.ts', declarations: [ @@ -180,8 +159,9 @@ export const src_json: Src_Json = { {name: 'Hook_Wrap_Callback_Context', kind: 'type'}, ], }, - './theme_standalone.css': { - path: 'theme_standalone.css', + './Code.svelte': {path: 'Code.svelte', declarations: [{name: 'default', kind: 'component'}]}, + './theme_variables.css': { + path: 'theme_variables.css', declarations: [{name: 'default', kind: 'css'}], }, './theme.css': {path: 'theme.css', declarations: [{name: 'default', kind: 'css'}]}, diff --git a/src/routes/samples/+page.svelte b/src/routes/samples/+page.svelte index e0abff52..406eec06 100644 --- a/src/routes/samples/+page.svelte +++ b/src/routes/samples/+page.svelte @@ -2,25 +2,27 @@ import Breadcrumb from '@ryanatkn/fuz/Breadcrumb.svelte'; import Code from '$lib/Code.svelte'; - import {samples} from '$lib/code_sample_inputs.js'; + import {samples} from '$lib/samples/all.js'; + import Footer from '$routes/Footer.svelte'; -
+
🎨 -
- {#each samples as { content, lang } (lang)} -
-

{lang}

-
+ {#each Object.values(samples) as sample (sample.name)} +
+

{sample.lang}

+
+
+

HTML

+ +
+
+

Highlight API

+ +
- {/each} -
+ +
+ {/each} +
- - diff --git a/src/update_tests.task.ts b/src/update_tests.task.ts deleted file mode 100644 index 78823136..00000000 --- a/src/update_tests.task.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type {Task} from '@ryanatkn/gro'; -import {format_file} from '@ryanatkn/gro/format_file.js'; -import {writeFileSync} from 'node:fs'; - -import {samples} from '$lib/code_sample_inputs.js'; -import {syntax_styler} from '$lib/index.js'; - -// TODO better way to do this? can't use gen because we want it to be opt-in, unless a new feature is added - -export const task: Task = { - summary: 'update tests with current behavior', - run: async () => { - const path = 'src/lib/code_sample_outputs.ts'; - - const contents = `// code_sample_outputs.ts - -${samples.map(({content, lang}) => `export const styled_${lang}_code = ${JSON.stringify(syntax_styler.stylize(content, lang))};`).join('\n\n')} -`; - - writeFileSync(path, await format_file(contents, {filepath: path})); - }, -}; diff --git a/tsconfig.json b/tsconfig.json index 57c075d5..52e51f1f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,6 +33,7 @@ ".svelte-kit/types/**/$types.d.ts", "src/**/*.js", "src/**/*.ts", - "src/**/*.svelte" + "src/**/*.svelte", + "benchmark/**/*.ts" ] }