Add boilerplate for automated testing of syntax grammar

Author: stanislaw

.github/workflows/ci-linux-ubuntu-latest.yml

@@ -0,0 +1,38 @@
+name: "StrictDoc.tmLanguage on Linux"
+
+on:
+  pull_request:
+    branches: [ "**" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.12
+
+    - name: Upgrade pip
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Install minimal Python packages
+      run: |
+        pip install -r requirements.txt
+
+    - name: Install Node packages
+      run: |
+        npm install
+
+    - name: Run tests
+      run: |
+        invoke test

.gitignore

@@ -1,4 +1,10 @@
 .idea
 node_modules
+package-lock.json
 *.vsix
 /_*
+
+# tests/integration
+.lit_test_times.txt
+**/Output/**
+

package.json

@@ -33,10 +33,17 @@
         "configuration": "./language-configuration.json"
       }
     ],
-    "grammars": [{
-      "language": "sdoc",
-      "scopeName": "source.sdoc",
-      "path": "./syntaxes/sdoc.tmLanguage.json"
-    }]
+    "grammars": [
+      {
+        "language": "sdoc",
+        "scopeName": "source.sdoc",
+        "path": "./syntaxes/sdoc.tmLanguage.json"
+      }
+    ]
+  },
+  "dependencies": {
+    "onigasm": "^2.2.5",
+    "vscode-textmate": "^9.2.0",
+    "vscode-oniguruma": "^1.5.1"
   }
 }

parse_syntax.js

@@ -0,0 +1,46 @@
+const fs = require('fs');
+const path = require('path');
+const vsctm = require('vscode-textmate');
+const oniguruma = require('vscode-oniguruma');
+
+const wasmBin = fs.readFileSync(path.join(__dirname, './node_modules/vscode-oniguruma/release/onig.wasm')).buffer;
+const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin).then(() => {
+    return {
+        createOnigScanner(patterns) { return new oniguruma.OnigScanner(patterns); },
+        createOnigString(s) { return new oniguruma.OnigString(s); }
+    };
+});
+
+const scopeName = "source.sdoc";
+const grammarPath = path.join(__dirname, "syntaxes/sdoc.tmLanguage.json");
+const filePath = process.argv[2];
+if (!fs.existsSync(filePath)) {
+  throw('File does NOT exist');
+}
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+  onigLib: vscodeOnigurumaLib,
+  loadGrammar: (scope) => {
+    if (scope === scopeName) {
+      const grammarData = fs.readFileSync(grammarPath, 'utf-8');
+      return Promise.resolve(vsctm.parseRawGrammar(grammarData, grammarPath));
+    }
+    return null;
+  }
+});
+
+registry.loadGrammar(scopeName).then(grammar => {
+    const lines = fs.readFileSync(filePath, 'utf-8').split(/\r?\n/);
+    let ruleStack = vsctm.INITIAL;
+
+    lines.forEach((line, lineIndex) => {
+      const lineTokens = grammar.tokenizeLine(line, ruleStack);
+      ruleStack = lineTokens.ruleStack;
+
+      lineTokens.tokens.forEach(token => {
+        const tokenText = line.slice(token.startIndex, token.endIndex);
+        console.log(`[${lineIndex + 1}:${token.startIndex}-${token.endIndex}] "${tokenText}" → ${token.scopes.join(' ')}`);
+      });
+    });
+  });

requirements.txt

@@ -0,0 +1,3 @@
+invoke
+lit
+filecheck>=0.0.20,<1.0.0

tasks.py

@@ -0,0 +1,95 @@
+# Invoke is broken on Python 3.11
+# https://github.com/pyinvoke/invoke/issues/833#issuecomment-1293148106
+import inspect
+import os
+import re
+import shutil
+import sys
+import tempfile
+from enum import Enum
+from pathlib import Path
+from typing import Dict, Optional
+
+if not hasattr(inspect, "getargspec"):
+    inspect.getargspec = inspect.getfullargspec
+
+import invoke
+from invoke import task
+
+# Specifying encoding because Windows crashes otherwise when running Invoke
+# tasks below:
+# UnicodeEncodeError: 'charmap' codec can't encode character '\ufffd'
+# in position 16: character maps to <undefined>
+# People say, it might also be possible to export PYTHONIOENCODING=utf8 but this
+# seems to work.
+# FIXME: If you are a Windows user and expert, please advise on how to do this
+# properly.
+sys.stdout = open(1, "w", encoding="utf-8", closefd=False, buffering=1)
+
+
+def run_invoke(
+    context,
+    cmd,
+    environment: Optional[dict] = None,
+    pty: bool = False,
+    warn: bool = False,
+) -> invoke.runners.Result:
+    def one_line_command(string):
+        return re.sub("\\s+", " ", string).strip()
+
+    return context.run(
+        one_line_command(cmd),
+        env=environment,
+        hide=False,
+        warn=warn,
+        pty=pty,
+        echo=True,
+    )
+
+
+@task(aliases=["ti"])
+def test_integration(
+    context,
+    focus=None,
+    debug=False,
+    no_parallelization=False,
+    fail_first=False,
+):
+    clean_itest_artifacts(context)
+
+    cwd = os.getcwd()
+
+    parse_syntax_script = f'node \\"{cwd}/parse_syntax.js\\"'
+
+    debug_opts = "-vv --show-all" if debug else ""
+    focus_or_none = f"--filter {focus}" if focus else ""
+    fail_first_argument = "--max-failures 1" if fail_first else ""
+    parallelize_opts = "" if not no_parallelization else "--threads 1"
+    test_folder = f"{cwd}/tests/integration"
+
+    itest_command = f"""
+        lit
+        --param PARSE_SYNTAX_EXEC="{parse_syntax_script}"
+        -v
+        {debug_opts}
+        {focus_or_none}
+        {fail_first_argument}
+        {parallelize_opts}
+        {test_folder}
+    """
+    run_invoke(
+        context,
+        itest_command,
+    )
+
+@task
+def clean_itest_artifacts(context):
+    # The command sometimes exits with 1 even if the files are deleted.
+    # warn=True ensures that the execution continues.
+    run_invoke(
+        context,
+        """
+        git clean -dX --force --quiet tests/integration/
+        """,
+        warn=True,
+    )

tests/integration/lit.cfg.py

@@ -0,0 +1,24 @@
+# ruff: noqa: F821
+
+import os
+import sys
+from typing import Any
+
+import lit.formats
+
+config: Any
+lit_config: Any
+
+config.name = "StrictDoc integration tests"
+config.test_format = lit.formats.ShTest("0")
+config.suffixes = [".itest"]
+
+current_dir = os.getcwd()
+
+parse_syntax_exec = lit_config.params["PARSE_SYNTAX_EXEC"]
+
+# NOTE: All substitutions work for the RUN: statements but they don't for CHECK:.
+#       That's how LLVM LIT works.
+config.substitutions.append(("%THIS_TEST_FOLDER", '$(basename "%S")'))
+
+config.substitutions.append(("%parse_syntax", parse_syntax_exec))

tests/integration/syntax/01_basic_document_node/sample.sdoc

@@ -0,0 +1,2 @@
+[DOCUMENT]
+TITLE: Document Title

tests/integration/syntax/01_basic_document_node/test.itest

@@ -0,0 +1,7 @@
+RUN: %parse_syntax %S/sample.sdoc | filecheck %s
+
+CHECK: [1:0-10] "[DOCUMENT]" → source.sdoc keyword.sdoc
+CHECK: [2:0-5] "TITLE" → source.sdoc keyword.control.sdoc keyword.control.sdoc
+CHECK: [2:5-7] ": " → source.sdoc keyword.control.sdoc
+CHECK: [2:7-22] "Document Title" → source.sdoc keyword.control.sdoc string.sdoc
+CHECK: [3:0-1] "" → source.sdoc