diff --git a/.dockerignore b/.dockerignore index 008e65c..dd97eb3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,39 @@ -target/ -.git/ -.ruff_cache/ -.vscode/ -services/ws-server/storage/ +# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit. +# Docker reads only this file; patterns are **/-prefixed to match at any depth +# like .gitignore. Edit .gitignore and regenerate. + +**/.claude/ +**/*.wasm +**/*.onnx +**/target/ **/.DS_Store +services/ws-wasm-agent/pkg/ +services/ws-server/static/models/ +**/.zig-cache/ +**/zig-out/ +**/*.o +**/*.pem +**/mprocs.log +**/__pycache__/ +**/.pytest_cache/ +**/.python-version +**/uv.lock +**/node_modules/ +**/pnpm-lock.yaml +**/.venv/ +# .NET build output. `obj/` is safe globally (nothing tracked is named obj/), +# but `bin/` is scoped to the module so it never matches a Rust crate's +# `src/bin/` (e.g. utilities/int-gen/src/bin/). +**/obj/ +services/ws-modules/dotnet-data1/bin/ +# Editor dir (but keep the shared recommended-extensions list), tool caches, and +# the ws-server's runtime file storage. +.vscode/* +!.vscode/extensions.json +**/.ruff_cache/ +**/.lycheecache +services/ws-server/storage/ +**/.git/ +**/Dockerfile* +README.md +**/.dockerignore diff --git a/.dprint.jsonc b/.dprint.jsonc index 790dece..41d6eee 100644 --- a/.dprint.jsonc +++ b/.dprint.jsonc @@ -1,36 +1,6 @@ +// dprint anchors its base directory (the tree it formats) to the directory of +// the config file it discovers. This stub keeps that base at the repo root while +// the real config lives in config/ alongside the other linter configs. { - "java": { - }, - "json": { - }, - // Match the repo-wide 120 line-length set in .editorconfig and ruff.toml, - // otherwise dprint's bundled ruff would reformat Python files to its - // default and fight with `mise run ruff-fmt`. - "ruff": { - "lineLength": 120, - }, - "malva": { - }, - "markdown": { - }, - "markup": { - }, - "typescript": { - }, - "yaml": { - }, - "excludes": [ - "**/node_modules", - "**/*-lock.json", - ], - "plugins": [ - "https://github.com/speakeasy-api/dprint-plugin-java/releases/latest/download/dprint_plugin_java.wasm", - "https://plugins.dprint.dev/g-plane/malva-v0.15.2.wasm", - "https://plugins.dprint.dev/g-plane/markup_fmt-v0.27.0.wasm", - "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", - "https://plugins.dprint.dev/json-0.21.3.wasm", - "https://plugins.dprint.dev/markdown-0.21.1.wasm", - "https://plugins.dprint.dev/ruff-0.7.10.wasm", - "https://plugins.dprint.dev/typescript-0.95.15.wasm", - ], + "extends": "config/dprint.jsonc", } diff --git a/.editorconfig b/.editorconfig index 2f0e8b4..5276445 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,6 +12,12 @@ trim_trailing_whitespace = true [*.md] indent_size = unset +# OPA/Rego: `conftest fmt` (opa fmt) indents with tabs and isn't configurable, so +# its canonical formatting needs tabs, not the repo's space default. +[*.rego] +indent_style = tab +indent_size = unset + # License files use the canonical upstream formatting (centred headers, odd # indent widths, etc.) — leave them alone. [LICENSE-*] diff --git a/.github/workflows/check.yml b/.github/workflows/check.yaml similarity index 71% rename from .github/workflows/check.yml rename to .github/workflows/check.yaml index 6013369..c471796 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yaml @@ -27,19 +27,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Install mise uses: taiki-e/install-action@v2 with: - tool: cargo-binstall,mise + tool: cargo-binstall,mise@2026.6.5 - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - name: Install pipx (Windows only — aqua has no Windows build) - if: runner.os == 'Windows' - run: python -m pip install pipx - # Optional npm backend, installed before the main `mise install`. # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - name: Install aube (optional npm backend, allowed to fail) @@ -53,14 +50,7 @@ jobs: - name: Install mise tools run: | - mise settings add idiomatic_version_file_enable_tools "[]" - mise settings experimental=true - mise settings set cargo.binstall true - # See test.yml for notes on why conda:openssl is installed up front. - mise install conda:openssl - # On macOS, lld is needed to compile Rust binary tools from source - # (e.g. `cargo:taplo-cli`, see CARGO_TARGET_*_APPLE_DARWIN_RUSTFLAGS). - mise install conda:lld + mise run preinstall mise install env: GITHUB_TOKEN: ${{ github.token }} @@ -69,6 +59,12 @@ jobs: # doesn't fail the whole `mise install` step. MISE_HTTP_TIMEOUT: "120" + - name: Prefetch Rust dependencies + run: mise run prefetch:rust + env: + GITHUB_TOKEN: ${{ github.token }} + CARGO_NET_RETRY: "5" + - name: Run checkers run: | mise run check diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yaml similarity index 72% rename from .github/workflows/dependencies.yml rename to .github/workflows/dependencies.yaml index d86a56b..ed2baa8 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yaml @@ -7,8 +7,9 @@ name: dependencies - Cargo.lock - Cargo.toml - "**/Cargo.toml" - - deny.toml - - .github/workflows/dependencies.yml + - config/deny.toml + - config/osv-scanner.toml + - .github/workflows/dependencies.yaml workflow_dispatch: permissions: @@ -22,11 +23,6 @@ defaults: run: shell: bash -# Deliberately mise-free: the only tools this job needs are the three -# dep-audit binaries, all of which taiki-e/install-action ships -# prebuilt. Skipping mise also skips the conda:openssl + workspace -# tool install path that the main CI flows take ~3 min on, keeping -# this check fast (~30 s typical). jobs: dependencies: runs-on: ubuntu-latest @@ -34,17 +30,25 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false - - name: Install dep-audit tools + - name: Install tools uses: taiki-e/install-action@v2 with: - tool: cargo-deny,cargo-unmaintained,osv-scanner + tool: cargo-deny,cargo-unmaintained,mise@2026.6.5,osv-scanner + + - name: Trust mise config + run: mise trust + + - name: Generate config/osv-scanner.toml from config/deny.toml + run: mise run gen:osv-scanner - name: cargo deny check - run: cargo deny check + run: mise run cargo-deny-check - name: osv-scanner - run: osv-scanner --lockfile Cargo.lock + run: mise run osv-scanner # `cargo unmaintained` persists per-repository archival/last-commit # lookups under `$XDG_CACHE_HOME/cargo-unmaintained` (default @@ -68,4 +72,7 @@ jobs: - name: cargo unmaintained env: GITHUB_TOKEN: ${{ github.token }} - run: cargo unmaintained + run: mise run cargo-unmaintained-check + + - name: Check config/osv-scanner.toml is committed + run: git diff --exit-code -- config/osv-scanner.toml diff --git a/.github/workflows/docker-linux.yaml b/.github/workflows/docker-linux.yaml new file mode 100644 index 0000000..416e4fe --- /dev/null +++ b/.github/workflows/docker-linux.yaml @@ -0,0 +1,59 @@ +--- +name: docker-linux + +"on": + pull_request: + paths: + - .github/workflows/docker-linux.yaml + - Dockerfile + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + # The image is huge (every language toolchain + prefetched models + a full + # debug build, incl. aws-lc-sys's large C objects), and its peak `target/` + # overruns a single runner disk. Reclaim the unused preinstalled SDKs and + # concatenate the freed root space with /mnt into one LVM volume mounted at + # Docker's data dir, then restart Docker so the build uses the combined space. + - name: Maximize build space (combine root + /mnt for Docker) + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 4096 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + remove-docker-images: "true" + build-mount-path: /var/lib/docker + build-mount-path-ownership: "root:root" + + - name: Restart Docker on the maximized volume + run: sudo systemctl restart docker + + - name: Build stage test + env: + GITHUB_TOKEN: ${{ github.token }} + run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . + + - name: Run the test suite + run: docker run --rm edge-toolkit-test diff --git a/.github/workflows/docker-windows.yaml b/.github/workflows/docker-windows.yaml new file mode 100644 index 0000000..7c3185a --- /dev/null +++ b/.github/workflows/docker-windows.yaml @@ -0,0 +1,52 @@ +--- +name: docker-windows + +"on": + pull_request: + paths: + - .github/workflows/docker-windows.yaml + - Dockerfile.nanoserver + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: windows-2022 + timeout-minutes: 120 + env: + # The classic Windows builder can't substitute build-args into the Dockerfile's RUN, + # and mise's prebuilt "latest" zip is stale (2026.3.0, too old for the config). + # 2026.6.5 is the first release with auto_env (loads .mise/config.windows.toml). + MISE_VERSION: "2026.6.5" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + # Hosted Windows runners don't reliably leave the Docker daemon running, so + # the build can fail connecting to the docker_engine pipe. Start it (no-op + # if already running) and confirm connectivity before building. + - name: Start the Docker daemon + run: | + sc query docker | grep -q RUNNING || net start docker + docker version + + - name: Prepare mise and Github token for the build context + run: | + v="${{ env.MISE_VERSION }}" + curl -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" + printf '%s' "${{ github.token }}" > gh_token + + - name: Build stage precompile + run: docker build -f Dockerfile.nanoserver --target precompile -t edge-toolkit-windows . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yaml similarity index 77% rename from .github/workflows/test.yml rename to .github/workflows/test.yaml index acd1d03..5d03a72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yaml @@ -17,6 +17,9 @@ name: test - macos-26-intel - windows-latest +permissions: + contents: read + concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} @@ -45,13 +48,13 @@ jobs: github.event.inputs.os == 'all' && format( '[{0},{1},{2},{3},{4}]', '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":25}', + '{"os":"ubuntu-24.04-arm","timeout":30}', '{"os":"macos-latest","timeout":45}', '{"os":"macos-26-intel","timeout":60}', '{"os":"windows-latest","timeout":60}' ) || github.event.inputs.os == 'ubuntu-latest' && '[{"os":"ubuntu-latest","timeout":30}]' - || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":25}]' + || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":30}]' || github.event.inputs.os == 'macos-latest' && '[{"os":"macos-latest","timeout":45}]' || github.event.inputs.os == 'macos-26-intel' && '[{"os":"macos-26-intel","timeout":60}]' || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":60}]' @@ -59,13 +62,15 @@ jobs: || format( '[{0},{1},{2}]', '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":25}', + '{"os":"ubuntu-24.04-arm","timeout":30}', '{"os":"macos-latest","timeout":45}' ) ) }} steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false # Reclaim the large preinstalled toolchains we never use (~30 GB). # `|| true` so a missing dir (e.g. on the arm image) is not fatal. @@ -85,15 +90,11 @@ jobs: - name: Install mise uses: taiki-e/install-action@v2 with: - tool: cargo-binstall,mise + tool: cargo-binstall,mise@2026.6.5 - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - name: Install pipx (Windows only — aqua has no Windows build) - if: runner.os == 'Windows' - run: python -m pip install pipx - # Optional npm backend, installed before the main `mise install`. # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - name: Install aube (optional npm backend, allowed to fail) @@ -107,27 +108,11 @@ jobs: - name: Install mise tools run: | - mise settings add idiomatic_version_file_enable_tools "[]" - mise settings experimental=true - mise settings set cargo.binstall true - - # Install conda:openssl up front — the project's OPENSSL_DIR points - # at its install dir, and any `cargo:*` tool that pulls openssl-sys - # reads that during its own `cargo install`. With the default - # parallel `mise install`, openssl-sys can panic with "OpenSSL - # include directory does not exist" if conda:openssl hasn't - # finished populating `include/` yet. - - mise install conda:openssl - - # And on macos, lld is needed to compile Rust binary tools from source. - - mise install conda:lld - + mise run preinstall mise install env: GITHUB_TOKEN: ${{ github.token }} - # Match check.yml: bump mise's 30s HTTP timeout so GitHub-release + # Match check.yaml: bump mise's 30s HTTP timeout so GitHub-release # downloads don't fail the install on transient slowness. MISE_HTTP_TIMEOUT: "120" diff --git a/.gitignore b/.gitignore index 067250e..9284041 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,16 @@ __pycache__/ uv.lock node_modules/ pnpm-lock.yaml +.venv/ +# .NET build output. `obj/` is safe globally (nothing tracked is named obj/), +# but `bin/` is scoped to the module so it never matches a Rust crate's +# `src/bin/` (e.g. utilities/int-gen/src/bin/). +obj/ +services/ws-modules/dotnet-data1/bin/ +# Editor dir (but keep the shared recommended-extensions list), tool caches, and +# the ws-server's runtime file storage. +.vscode/* +!.vscode/extensions.json +.ruff_cache/ +.lycheecache +services/ws-server/storage/ diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 69c2c54..536c5b4 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -7,6 +7,7 @@ dart_release = "https://storage.googleapis.com/dart-archive/channels/stable/rele dart_bucket = "https://storage.googleapis.com/storage/v1/b/dart-archive/o" [tools] +# cargo: backend -- dart-typegen has no prebuilt binary (crates.io only). "cargo:dart-typegen" = "latest" # dart isn't an aqua/registry tool — install it straight from Google's dart @@ -75,14 +76,14 @@ run = "dart pub get --directory services/ws-modules/dart-comm1" [tasks."gen:dart-ws"] depends = ["gen:ws-spec"] description = "Emit the Dart sealed-class WS client via dart-typegen (consumes generated/specs/ws.kdl from gen:ws-spec)" -shell = "bash -euo pipefail -c" # dart-typegen exits 0 on Windows but writes an empty .dart file, which # would clobber the committed source and fail gen-specs-check. Skip the # regen there; the committed file (regenerated on Linux/macOS) stays # canonical. run = """ -[[ "${OS:-}" == "Windows_NT" ]] && { echo "gen:dart-ws: skipped on Windows (dart-typegen empty output)"; exit 0; } +[ "${OS:-}" = "Windows_NT" ] && { echo "gen:dart-ws: skipped on Windows (dart-typegen empty output)"; exit 0; } mkdir -p generated/dart-ws/lib dart-typegen generate -i generated/specs/ws.kdl -o generated/dart-ws/lib/ws_messages.dart dart format generated/dart-ws/lib/ws_messages.dart """ +shell = "bash -euo pipefail -c" diff --git a/.mise/config.java.toml b/.mise/config.java.toml index a5cbf11..d1d66d3 100644 --- a/.mise/config.java.toml +++ b/.mise/config.java.toml @@ -15,6 +15,12 @@ MAVEN_ARGS = "--no-transfer-progress" description = "Build the java-data1 workflow module" run = "mvn package" +# Namespaced aggregator picked up by the default config's globbed `check`. The +# compile triggers maven-compiler-plugin with -Xlint:all -Werror + Error Prone. +[tasks."check:java"] +description = "Run Java checks (javac -Xlint:all -Werror, Error Prone)" +run = "mvn -q compile" + [tasks."prefetch:java"] description = "Prefetch Java (Maven) dependencies" run = "mvn dependency:resolve --quiet" diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml new file mode 100644 index 0000000..bf30ccb --- /dev/null +++ b/.mise/config.linux.toml @@ -0,0 +1,65 @@ +# Linux-only mise config. Shared vars (rpath_flag, pylib_flag, conda_openssl, +# py313_unix, …) come from config.toml, which loads first; PYO3_PYTHON keeps its +# config.toml default (py313_unix). + +[tools] +# Ships the C `libclang.so` (+ clang resource headers) bindgen needs to build the +# deno web runner's libsqlite3-sys; a mise-only box usually only has libclang-cpp +# (the C++ API), not the C one. +"conda:clangxx" = "latest" + +[vars] +# `a_`/`b_`/`c_` tiers keep taplo's alphabetical reorder_keys from moving a var +# ahead of one it reads (`b_` reads `a_`, `c_` reads `b_`), which mise can't render. +# a_conda_arch: conda's target-triple dir (x86_64/aarch64 vs mise's x64/arm64). +a_conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" +a_conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" +b_clang_resource = "{{ vars.a_conda_clangxx }}/lib/clang/22" +b_clang_sysroot = "{{ vars.a_conda_clangxx }}/{{ vars.a_conda_arch }}-conda-linux-gnu/sysroot" +# bindgen loads libclang directly rather than driving the clang binary, so it +# can't auto-find its resource dir (builtins like stddef.h) or conda's bundled +# sysroot. Pass `-resource-dir` + `--sysroot` so header resolution is identical +# on every machine (a bare `-isystem` leaves builtin type macros unset and fails +# in CI). Using conda's sysroot rather than the host's /usr/include keeps bindgen +# self-contained. +c_bindgen_args = "-resource-dir {{ vars.b_clang_resource }} --sysroot={{ vars.b_clang_sysroot }}" +# clang-tidy gets the same resource-dir + conda sysroot as bindgen: the sysroot +# also stops clang scanning the host's multiple /usr/lib/gcc installs for +# libstdc++ (which otherwise warns, and -warnings-as-errors fails the run). +clang_resource_arg = "{{ vars.c_bindgen_args }}" + +[env] +# clang-sys finds conda's libclang.so via LIBCLANG_PATH; bindgen gets conda's +# resource headers + sysroot via BINDGEN_EXTRA_CLANG_ARGS (see c_bindgen_args). +BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.c_bindgen_args }}" +# rpath/pylib flags (shared, from config.toml): OPENSSL_DIR points openssl-sys at +# conda's OpenSSL, so anything linking it records conda's libssl soname; without +# an rpath the loader only finds it when conda's soname matches the system one +# (it broke when conda bumped to libssl.so.4). pylib_flag does the same for the +# CPython lib dir. No `-fuse-ld=lld` needed: the system linker accepts +# `-Wl,-rpath` directly. +CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +LIBCLANG_PATH = "{{ vars.a_conda_clangxx }}/lib" + +[tasks.preinstall] +# Preinstall: the shared cross-platform base (via _setup_all), then verify the +# system build prerequisites mise can't supply -- the C/C++ toolchain, gpg and +# archive tools the Dockerfile installs via apt. A workstation missing any is +# told to install them with its package manager (CI/Docker already have them). +depends = ["_setup_all"] +description = "Preinstall: shared base + verify system build prerequisites" +run = """ +pkgs="bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils" +missing="" +for cmd in gcc g++ make git curl gpg unzip bzip2 xz; do + command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd" +done +if [ -n "$missing" ]; then + echo "preinstall: missing required system tools:$missing" >&2 + echo "Install them with your package manager. On Debian/Ubuntu:" >&2 + echo " sudo apt-get install -y $pkgs" >&2 + exit 1 +fi +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml new file mode 100644 index 0000000..dd1b3b4 --- /dev/null +++ b/.mise/config.macos.toml @@ -0,0 +1,43 @@ +# macOS-only mise config. Shared vars (rpath_flag, pylib_flag, py313_unix, …) +# come from config.toml, which loads first; PYO3_PYTHON keeps its config.toml +# default (py313_unix). + +[tools] +# The LLD linker (ships ld64.lld) the RUSTFLAGS below use; Apple's /usr/bin/clang +# can't resolve `-fuse-ld=lld` without it. +"conda:lld" = "latest" + +[vars] +conda_lld = "{{ env.HOME }}/.local/share/mise/installs/conda-lld/latest" +# Point `-fuse-ld=` at the absolute ld64.lld so it resolves without a PATH lookup +# (the cargo subprocess mise spawns for a `cargo:` source build doesn't have +# sibling tools like conda:lld on PATH). +lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" +# Xcode's libclang (the lib dir beside `xcrun`'s clang); clang-sys doesn't always +# auto-find it on CI. The system libclang knows the SDK, so no bindgen args. +mac_libclang = "{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib" + +[env] +# lld_flag points Apple's clang at ld64.lld (see [vars]); the rpath/pylib flags +# (from config.toml) record conda's OpenSSL soname + the CPython lib dir so the +# runtime loader finds them. Applies to every rustc call, including the `cargo:` +# source-build subprocess. +CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" +LIBCLANG_PATH = "{{ vars.mac_libclang }}" + +[tasks.preinstall] +# Preinstall: verify Xcode's command-line tools (cc/clang etc., which mise can't +# supply), then the shared base + conda:lld (the darwin linker). A box missing +# the CLT is told to install them. +depends = ["_setup_all"] +description = "Preinstall: verify Xcode CLT, shared base, conda:lld (darwin linker)" +run = """ +xcode-select -p >/dev/null 2>&1 || { + echo "preinstall: Xcode command-line tools not found." >&2 + echo "Install them with: xcode-select --install" >&2 + exit 1 +} +mise install conda:lld +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.python.toml b/.mise/config.python.toml index e95af62..a52eda3 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -3,14 +3,11 @@ # the `pipx:*` entries below resolve through it. [tools] -"pipx:cmake" = "latest" "pipx:componentize-py" = "latest" "pipx:datamodel-code-generator" = { version = "latest", extras = "ruff" } "pipx:openapi-python-client" = "0.29.0" "pipx:pytest" = "latest" -python = "3.13" ruff = "latest" -uv = "0.11.8" # Use the GitHub release tarball, not `npm:pyodide`. The npm package is only # the runtime (pyodide.js, .wasm, stdlib, micropip) ~5 MB — it's designed for @@ -32,6 +29,10 @@ py_dirs = "services/ws-modules/ generated/python-ws/ generated/python-rest/" # that module's dir); used by both the bindings and componentize steps. wit_dir = "../../../generated/specs/wit" +[env] +# Keep ruff's cache under target/ (gitignored) instead of a repo-root .ruff_cache. +RUFF_CACHE_DIR = "{{ config_root }}/target/ruff-cache" + [tasks] ruff-check = "ruff check {{ vars.py_dirs }}" ruff-fmt = "ruff format {{ vars.py_dirs }}" @@ -67,7 +68,7 @@ UV_PYTHON = """\ {% else %}{{ exec(command='mise which python3.13') }}{% endif %}""" [tasks.build-et-ws-wheel] -depends = ["gen:python-ws"] +depends = ["build-et-cli", "gen:python-ws"] description = "Build the generated et-ws Python package as a wheel + ws-module pkg/" dir = "generated/python-ws" # Wheel goes straight into pkg/ alongside the committed et_ws.js shim so the @@ -81,7 +82,7 @@ cargo run -p et-cli -- module-package-json shell = "bash -euo pipefail -c" [tasks.build-et-rest-client-wheel] -depends = ["gen:python-rest"] +depends = ["build-et-cli", "gen:python-rest"] description = "Build the generated et-rest-client Python package as a wheel + ws-module pkg/" dir = "generated/python-rest" run = """ @@ -122,6 +123,7 @@ cargo run -p et-cli -- module-package-json shell = "bash -euo pipefail -c" [tasks.build-ws-wasi-graphics-info-module] +depends = ["build-et-cli"] description = "Build the WASI graphics-info Python module as a WASI Preview 2 component" dir = "services/ws-modules/wasi-graphics-info" # WIT lives once at generated/specs/wit/ — the runner's `runner` world is diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 044a9ec..b966aff 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -14,70 +14,73 @@ [tasks.build-ws-data1-module] description = "Build the data1 workflow WASM module" dir = "services/ws-modules/data1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-har1-module] +depends = ["build-et-cli"] description = "Build the har1 workflow WASM module" dir = "services/ws-modules/har1" # Chain with `&&` on one line rather than a multi-line `bash` block: wasm-pack # shells out to `cargo metadata`, and that subprocess can't find `cargo` under # Git Bash on Windows. The default shell (cmd there) resolves cargo like the # single-line wasm-pack modules do. -run = "wasm-pack build . --target web && cargo run -p et-cli -- module-package-json" +run = "wasm-pack build . --target web {{ vars.no_opt }} && cargo run -p et-cli -- module-package-json" [tasks.build-ws-face-detection-module] +depends = ["build-et-cli"] description = "Build the face detection workflow WASM module" dir = "services/ws-modules/face-detection" # See build-ws-har1-module: `&&` on one line under the default shell so # wasm-pack's `cargo metadata` subprocess can find cargo on Windows. -run = "wasm-pack build . --target web && cargo run -p et-cli -- module-package-json" +run = "wasm-pack build . --target web {{ vars.no_opt }} && cargo run -p et-cli -- module-package-json" [tasks.build-ws-comm1-module] description = "Build the comm1 workflow WASM module" dir = "services/ws-modules/comm1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-sensor1-module] description = "Build the sensor1 workflow WASM module" dir = "services/ws-modules/sensor1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-audio1-module] description = "Build the audio1 workflow WASM module" dir = "services/ws-modules/audio1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-video1-module] description = "Build the video1 workflow WASM module" dir = "services/ws-modules/video1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-bluetooth-module] description = "Build the bluetooth workflow WASM module" dir = "services/ws-modules/bluetooth" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-geolocation-module] description = "Build the geolocation workflow WASM module" dir = "services/ws-modules/geolocation" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-graphics-info-module] description = "Build the graphics info workflow WASM module" dir = "services/ws-modules/graphics-info" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-speech-recognition-module] description = "Build the speech recognition workflow WASM module" dir = "services/ws-modules/speech-recognition" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-nfc-module] description = "Build the nfc workflow WASM module" dir = "services/ws-modules/nfc" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-wasi-data1-module] +depends = ["build-et-cli"] description = "Build the Rust WASI data1 module as a WASI Preview 2 component" # The crate is a regular workspace member but its lib body is gated on # `#![cfg(target_os = "wasi")]`, so cargo check from the host target gets @@ -93,6 +96,7 @@ cargo run -p et-cli -- module-package-json --module-dir services/ws-modules/wasi shell = "bash -euo pipefail -c" [tasks.build-ws-wasi-comm1-module] +depends = ["build-et-cli"] description = "Build the Rust WASI comm1 module as a WASI Preview 2 component" run = """ cargo build --release -p et-ws-wasi-comm1 --target wasm32-wasip2 diff --git a/.mise/config.toml b/.mise/config.toml index 2801af7..458a098 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -11,34 +11,43 @@ # Run a check across everything: mise run check-all (or install-all) # Make a selection sticky: export MISE_ENV=dart +# Require release that introduced auto_env enabled in .miserc.toml +min_version = "2026.6.5" + [settings] -# Never auto-install tools before a `mise run`: installs are explicit -# (`mise install` / `mise run install-all`, and CI installs up front). This -# keeps cheap tasks like `print-all-langs`, and the pre-install `setup-aube` -# step, from eagerly pulling the whole toolchain — on CI and the CLI alike. +# exec_auto_install covers `mise exec`, +# task.run_auto_install covers `mise run`. +exec_auto_install = false task.run_auto_install = false [tools] action-validator = "latest" +"aqua:EmbarkStudios/cargo-deny" = "latest" +ast-grep = "latest" cargo-binstall = "latest" -"cargo:ast-grep" = "latest" -"cargo:cargo-deny" = "latest" +"aqua:rustwasm/wasm-pack" = "latest" "cargo:cargo-expand" = "latest" -"cargo:taplo-cli" = "latest" -"cargo:wasm-pack" = "latest" -"cargo:watchexec-cli" = "latest" +taplo = "latest" +watchexec = "latest" # vfox-chromedriver's PostInstall hook fails on Windows ("syntax of the # command is incorrect"). Chromedriver is only used by the # `test-ws-wasm-agent-chrome` / `ws-e2e-chrome` tasks, neither of which run # in the standard CI `test` flow, so skip it on Windows entirely. "chromedriver" = { version = "146", os = ["linux", "macos"] } cmake = "latest" -"conda:lld" = "latest" "conda:openssl" = "3" +conftest = "latest" dprint = "latest" editorconfig-checker = "latest" +# Retries a command with exponential backoff + jitter; wraps the model fetches +# below (see the `retry` var) so a transient HTTP 429 doesn't fail the build. +"github:dbohdan/recur" = "latest" "github:grok-rs/waitup" = "latest" -"github:wasm-bindgen/wasm-bindgen" = "0.2.114" +"github:owenlamont/ryl" = "latest" +"github:wasm-bindgen/wasm-bindgen" = "0.2.122" +hadolint = "latest" +ls-lint = "latest" +lychee = "latest" mprocs = "latest" node = "22" "npm:onnxruntime-web" = "latest" @@ -52,37 +61,57 @@ pipx = { version = "latest", os = ["linux", "macos"] } "pipx:semgrep" = "latest" # The default aqua backend has no darwin/amd64 prebuilt; fall back to # `npm:pnpm` on Intel Mac (node is already a mise tool above). +gh = "latest" "npm:pnpm" = { version = "latest", os = ["macos/x64"] } pnpm = { version = "latest", os = ["linux", "macos/arm64", "windows"] } protoc = "latest" +python = "3.13" rclone = "latest" rust = [ { version = "latest", components = "clippy,rust-analyzer", targets = "wasm32-unknown-unknown,wasm32-wasip2" }, { version = "nightly", components = "rust-src,rustfmt", targets = "wasm32-unknown-unknown,wasm32-wasip2" }, ] typos = "latest" +uv = "0.11.8" wasm-tools = "latest" yq = "latest" +zizmor = "latest" [vars] -# mise's conda tool install dirs, reused below by OPENSSL_DIR and the -# CARGO_TARGET_* RUSTFLAGS. -conda_lld = "{{ env.HOME }}/.local/share/mise/installs/conda-lld/latest" +# clang-tidy's resource-dir arg (points clang at its builtin headers, e.g. +# stddef.h). Empty default; config.linux.toml sets it from conda:clangxx. +clang_resource_arg = "" conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" -# Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env -# values below — kept as vars so those values stay on a single line. -lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" +# mise-managed CPython 3.13 (Linux/macOS); the PYO3_PYTHON default below. Windows +# overrides PYO3_PYTHON with its own py313_win in config.windows.toml. +py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" +# RUSTFLAGS fragments shared by the CARGO_TARGET_* env in config..toml rpath_flag = "-C link-arg=-Wl,-rpath,{{ vars.conda_openssl }}/lib" +# rpath to the mise CPython 3.13 lib dir so the et-ws-pyo3-runner binary finds +# libpython at runtime (libpython3.13.so on Linux, libpython3.13.dylib on macOS +# via its @rpath install name). pyo3 bakes an rpath to the resolved patch-version +# dir, which doesn't exist when mise installs under the "3.13" alias; point at +# the stable install path instead. +pylib_flag = "-C link-arg=-Wl,-rpath,{{ env.HOME }}/.local/share/mise/installs/python/3.13/lib" wasm_rustflags = "-C target-cpu=mvp -C target-feature=+mutable-globals,+sign-ext,+nontrapping-fptoint" +# Extra flag for the wasm-pack module builds; empty so wasm-opt runs. +# config.windows.toml overrides it to --no-opt where wasm-opt can't execute. +no_opt = "" # Source/dest paths for the fetch-*-rclone model downloads — kept as vars so # each `rclone copyto` command stays on a single line without continuations. -face1_src = ":http:amd/retinaface/resolve/main/weights/RetinaFace_int.onnx" face1_dst = "data/model-modules/model-face1/pkg/video_cv.onnx" +face1_src = ":http:amd/retinaface/resolve/main/weights/RetinaFace_int.onnx" +har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" har_repo = ":http:acd17sk/MET-Metabolic-Equivalent-of-Task-AI-Android-APP" har_src = "{{ vars.har_repo }}/raw/refs/heads/main/Models/onnx/hybrid_met.onnx" -har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" -mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" +mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" +# `retry` prefixes each fetch with recur for exponential-backoff-with-jitter +# retries (8 tries, 2s base doubling, capped 2m, 0-5s jitter); the *_http vars +# carry the per-host base URL + flags. Kept as vars so each task line fits. +gh_http = "--http-url https://github.com --progress --ignore-existing" +hf_http = "--http-url https://huggingface.co --progress --ignore-existing" +retry = "recur --attempts 8 --delay 2s --backoff 2s --max-delay 2m --jitter 0,5s --" [env] # Comma-separated list of every language env (one per .mise/config..toml). @@ -90,41 +119,20 @@ mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" # no MISE_ENV=all). Hardcoded rather than shell-discovered so it works on Windows # too — keep in sync when adding/removing a config..toml. ALL_LANGS = "dart,dotnet,java,python,rust,zig" +TAPLO_CONFIG = "{{ config_root }}/config/taplo.toml" +CLIPPY_CONF_DIR = "{{ config_root }}/config" # Use the conda:openssl install for Rust's OPENSSL_DIR so openssl-sys crate builds. OPENSSL_DIR = "{{ vars.conda_openssl }}" -# Use lld on macos as documented at https://github.com/cameron1024/dart-typegen -# Apple `/usr/bin/clang` rejects `-fuse-ld=lld` outright ("invalid linker -# name") unless it can resolve `ld64.lld` on `PATH`. These `RUSTFLAGS` apply -# to every `rustc` call — including the cargo subprocess mise spawns when a -# `cargo:` tool falls back to a source build (e.g. `cargo:taplo-cli`, whose -# QuickInstall prebuilt 404s on macOS), and mise's install-time `PATH` for -# that subprocess does NOT include sibling tools like `conda:lld`. Point -# `-fuse-ld=` at the absolute `ld64.lld` shipped by `conda:lld` so the flag -# resolves without any `PATH` lookup — same conda root the rpath link-arg -# already points into. -CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }}" -CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }}" -# Linux needs the same rpath the macOS flags above carry, for the same reason: -# OPENSSL_DIR points openssl-sys at conda's OpenSSL, so anything linking it -# (e.g. the ort-sys build script) records conda's `libssl` soname. Without an -# rpath the runtime loader only finds it when conda's soname happens to match -# the system one Ubuntu ships on the default path — which broke when conda -# bumped to `libssl.so.4` (no system equivalent). Point the loader at conda's -# own lib dir so the soname is irrelevant. No `-fuse-ld=lld`: the system -# linker on Linux accepts `-Wl,-rpath` directly, unlike Apple clang's lld dance. -CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }}" -CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }}" -# rclone retry tuning for the `fetch-*-rclone` model-download tasks. -# Hugging Face / GitHub raw responded HTTP 429 ("Too Many Requests") at -# least once with all three of rclone's default high-level retries -# failing inside the same second (no Retry-After honoured at the -# default `--retries-sleep 0s`). Force a 30 s pause between retries -# and bump the count. `RCLONE_*` env vars are rclone's own alternative -# to CLI flags — `rclone.conf` only configures named remotes, not -# global retry policy. + +# pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from +# PYO3_PYTHON, else the first python on PATH -- under mise that's the 32-bit +# Pyodide shim ("target architecture (64-bit) does not match ... (32-bit)"). Pin +# it to the mise-managed CPython 3.13. Always-loaded so `cargo check --workspace` +# builds the pyo3 runner without MISE_ENV=python. This is the Linux/macOS path; +# config.windows.toml overrides it with py313_win. +PYO3_PYTHON = "{{ vars.py313_unix }}" RCLONE_CONFIG = "{{ config_root }}/config/rclone.conf" -RCLONE_RETRIES = "10" -RCLONE_RETRIES_SLEEP = "30s" +RCLONE_RETRIES = "1" [tasks] ec = "ec" @@ -152,15 +160,6 @@ depends = ["fmt:*"] description = "Run all formatters: fmt:rust + any loaded guest fmt:" [tasks."check:rust"] -# The dep-audit tools (osv-scanner, cargo-deny, cargo-unmaintained) run -# in the standalone `.github/workflows/dependencies.yml` workflow so they -# only fire when Cargo.lock / Cargo.toml / deny.toml change, not on every -# `mise run check`. `osv-scanner` and `cargo-deny` are kept in `[tools]` -# for ad-hoc local invocation (e.g. `mise run osv-scanner`, -# `cargo deny check`); `cargo-unmaintained` is installed only by the -# workflow via `taiki-e/install-action` since it isn't useful enough -# locally to justify the cargo-build cost on every `mise install`. -# # Rust + universal repo-wide checks. Lives in the always-loaded default config, # so `check`'s `check:*` glob always matches at least this one (Rust is in the # default config this pass; the guest `check:` tasks come from their @@ -172,13 +171,20 @@ depends = [ "cargo-clippy", "cargo-doc-check", "cargo-fmt-check", + "docker-check", "dprint-check", "editorconfig-check", "gen-specs-check", + "hadolint-check", + "link-check", + "ls-lint-check", + "ryl-check", "semgrep-check", + "conftest-check", "taplo-check", "typos", "verification-check", + "zizmor-check", ] description = "Run the Rust + universal repo-wide checks" @@ -193,16 +199,32 @@ description = "Run all checks: check:rust + any loaded guest check:" [tasks.action-validator] description = "Validate GitHub Actions workflow YAML" -run = "action-validator .github/workflows/*.yml" +run = "action-validator .github/workflows/*.yaml" + +[tasks.zizmor-check] +description = "Audit GitHub Actions workflows for security issues (zizmor)" +# --offline so it needs no GH token and stays deterministic; config/zizmor.yaml +# turns off the hash-pin policy (we pin actions by tag, not commit). +run = "zizmor --offline --no-progress -c config/zizmor.yaml .github/workflows" + +[tasks.ls-lint-check] +description = "Lint file and directory naming conventions (ls-lint)" +run = "ls-lint --config config/ls-lint.yaml" [tasks.ast-grep-check] +# --no-ignore hidden so the gha-* YAML rules reach .github/workflows (ast-grep +# skips dot-dirs by default); gitignored paths like target/ stay skipped. description = "Run ast-grep structural-search rules" -run = "ast-grep scan --error -c config/ast-grep/sgconfig.yml" +run = "ast-grep scan --error --no-ignore hidden -c config/ast-grep/sgconfig.yaml" [tasks.semgrep-check] description = "Run Semgrep generic-mode rules (e.g. Cargo.toml style)" run = "semgrep scan --config config/semgrep --error --metrics=off ." +[tasks.hadolint-check] +description = "Lint the Dockerfiles with hadolint" +run = "git ls-files '*Dockerfile' '*Dockerfile.*' | xargs hadolint --config config/hadolint.yaml" + [tasks.cargo-check] run = "cargo check --workspace" @@ -218,7 +240,7 @@ run = "cargo clippy --keep-going --workspace --tests" [tasks.cargo-doc-check] description = "Build rustdoc with -D warnings to fail on any doc issues" env = { RUSTDOCFLAGS = "-D warnings" } -run = "cargo doc --workspace --no-deps --document-private-items" +run = "cargo doc --keep-going --workspace --no-deps --document-private-items" [tasks.cargo-clippy-fix] run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace --tests" @@ -226,24 +248,54 @@ run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace [tasks.taplo-fmt] run = "taplo format" +[tasks.conftest-check] +depends = ["conftest-check-*"] +description = "Run all conftest policy checks (one per parsed file format)" + +[tasks.conftest-check-toml] +description = "Run conftest OPA/Rego policies over the TOML config + lock files" +# --combine feeds every listed file to one policy evaluation as an array of +# {path, contents}, so a policy can both check each file and compare across files +# (the mise wasm-bindgen pin vs Cargo.lock). --parser toml: Cargo.lock has no +# .toml suffix. The TOML rule namespaces are listed explicitly, so the per-file +# `gha` (YAML) namespace is never evaluated against this combined TOML input. +run = """ +ns="--namespace cross --namespace cargo --namespace mise --namespace pyproject" +git ls-files '*Cargo.toml' '.mise/config*.toml' '*pyproject.toml' Cargo.lock | + xargs conftest test --combine --parser toml $ns -p config/conftest/policy +""" +shell = "bash -euo pipefail -c" + +[tasks.conftest-check-yaml] +description = "Run conftest OPA/Rego policies over the GitHub Actions workflow YAML" +run = "conftest test --parser yaml --namespace gha -p config/conftest/policy .github/workflows" + [tasks.taplo-check] -# `taplo lint` reads .taplo.toml's `[[rule]] schema` entries for editor / +# `taplo lint` reads config/taplo.toml's `[[rule]] schema` entries for editor / # LSP-style validation, but silently ignores their nested constraints in # CLI mode (verified: injecting `path = "../foo"` into a member Cargo.toml # doesn't fire `no-path-deps` via the rule alone). We work around it by # applying each schema explicitly with `taplo lint --schema file://...`. -# When adding a new schema under `config/taplo/`, append a `taplo lint` -# line here too. See `config/taplo/RULE-IDEAS.md` for the investigation. +# conftest-check-toml (config/conftest/policy) intentionally duplicates these +# lints; running both is fine. run = """ -# Baseline TOML validation across every .toml file the auto-config picks up. taplo lint +# The --schema lints below pass file://$PWD URLs, which aren't well-formed on +# Windows (drive-letter paths); run them on Linux/macOS only. conftest-check-toml +# applies the same rules on every OS. +[ "${OS:-}" = "Windows_NT" ] && exit 0 + # All member Cargo.tomls — exclude workspace root and any build output. -# (Avoid `mapfile`: it's bash 4+ only and macOS ships bash 3.2.) -members=() -while IFS= read -r line; do - members+=("$line") -done < <(find . -name Cargo.toml -not -path './target/*' -not -path './Cargo.toml') +# `-prune` target/ and .git so find never descends into them: `-not -path` +# alone still walks the whole tree, and target/debug/deps/ churns constantly +# under a concurrent `cargo-check`, racing find into "No such file" errors. +# POSIX sh (no bash arrays / process substitution -- the tasks run on busybox +# ash on Windows): collect the paths into a temp file once and feed each schema +# run via xargs. +members="$(mktemp)" +trap 'rm -f "$members"' EXIT +find . -path ./target -prune -o -path ./.git -prune -o -name Cargo.toml ! -path ./Cargo.toml -print >"$members" # Banned deps only need to be checked at the workspace root: member # crates are required to reference deps via `workspace = true` (the @@ -252,10 +304,11 @@ done < <(find . -name Cargo.toml -not -path './target/*' -not -path './Cargo.tom taplo lint --schema "file://$PWD/config/taplo/no-banned-deps.schema.json" Cargo.toml # Schemas that apply to every member crate. -taplo lint --schema "file://$PWD/config/taplo/no-path-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/no-wildcard-or-git-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/require-workspace-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.json" "${members[@]}" +xargs taplo lint --schema "file://$PWD/config/taplo/no-path-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/no-wildcard-or-git-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/require-workspace-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/no-lib-name.schema.json" <"$members" # `require-lints-section` exempts generated/rust-rest/Cargo.toml because # progenitor's emitted `src/lib.rs` contains patterns our workspace lint @@ -264,17 +317,35 @@ taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.j # on the generated source. The exemption is about the lint table, not # about the deps — its `[dependencies]` still inherits from # `[workspace.dependencies]` like every other crate. -lint_inheritable=() -for f in "${members[@]}"; do - [[ "$f" == "./generated/rust-rest/Cargo.toml" ]] && continue - lint_inheritable+=("$f") -done -taplo lint --schema "file://$PWD/config/taplo/require-lints-section.schema.json" "${lint_inheritable[@]}" +grep -vxF './generated/rust-rest/Cargo.toml' "$members" | + xargs taplo lint --schema "file://$PWD/config/taplo/require-lints-section.schema.json" + +# mise task `run` must be a string, not an array: taplo's reorder_arrays would +# re-sort an array and scramble an ordered command sequence (use a multiline +# string, one command per line). Applies to every .mise/config*.toml. +taplo lint --schema "file://$PWD/config/taplo/mise-run-not-array.schema.json" .mise/config*.toml + +# ubi: is deprecated (use http:); cargo: builds from source (slow), allowed only +# for cargo-expand + dart-typegen. Two separate rules, two reasons. +taplo lint --schema "file://$PWD/config/taplo/mise-no-ubi-backend.schema.json" .mise/config*.toml +taplo lint --schema "file://$PWD/config/taplo/mise-cargo-backend-allowlist.schema.json" .mise/config*.toml """ shell = "bash -euo pipefail -c" [tasks.typos] -run = "typos" +run = "typos --config config/typos.toml" + +# Part of the `check` aggregate. lychee makes live network requests, so +# config/lychee.toml is tuned (retries, 429-tolerance, disk cache) to keep it +# from being flaky. Checks every URL in the repo's Markdown + Rust sources; +# lychee extracts URLs from .rs comments and string literals too. +[tasks.link-check] +description = "Check that URLs in .md and .rs files are reachable (network)" +run = "lychee --config config/lychee.toml '**/*.md' '**/*.rs'" + +[tasks.ryl-check] +description = "Lint YAML with ryl (a yamllint-compatible Rust linter)" +run = "ryl -c config/ryl.yaml .github config" [tasks.setup-aube] # `aube` is an OPTIONAL faster npm backend, kept out of the mandatory `[tools]` @@ -282,14 +353,13 @@ run = "typos" # back to the classical `lib/node_modules` layout without it (see # `find_npm_modules_path_in`). # -# CI (check.yml / test.yml) runs this task in its own allowed-to-fail step, +# CI (check.yaml / test.yaml) runs this task in its own allowed-to-fail step, # placed BEFORE the main `mise install` so the `npm:onnxruntime-web` install # can pick up aube as its backend. # # Platform split: aqua ships aube for linux/macos-arm64/windows; macos/x64 has # no aqua asset, so use the cargo backend there. The branch is rendered by -# mise's tera templating, so it works regardless of the task shell (cmd on -# Windows). +# mise's tera templating to pick the right backend per platform. description = "Install the optional aube npm backend (platform-specific, best-effort)" run = """ {% if os() == "macos" and arch() == "x64" %} @@ -298,10 +368,88 @@ mise install cargo:aube@latest mise install aube@latest {% endif %} """ +shell = "bash -euo pipefail -c" + +[tasks._setup_all] +description = "Private shared cross-platform preinstall" +hide = true +run = """ +mise settings experimental=true +mise settings set cargo.binstall true +mise install cargo-binstall +mise install node +mise install conda:openssl +""" +shell = "bash -euo pipefail -c" [tasks.osv-scanner] -depends = ["cargo-check"] -run = "osv-scanner --lockfile Cargo.lock" +# Scans the committed Cargo.lock only -- no build dependency, so the same task +# serves both ad-hoc local use and the dependencies workflow. +run = "osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml" + +[tasks."gen:osv-scanner"] +description = "Regenerate config/osv-scanner.toml from config/deny.toml's [advisories].ignore list" +# osv-scanner and cargo-deny must ignore the same advisory IDs; config/deny.toml +# is the source of truth (it carries the per-ID rationale). grep/sort/sed only, so +# the `dependencies` workflow can run it (via mise) next to the audit binaries. +run = """ +{ + echo '# AUTO-GENERATED from config/deny.toml by `mise run gen:osv-scanner`.' + echo 'IgnoredVulns = [' + grep -oE '"RUSTSEC-[0-9]{4}-[0-9]{4}"' config/deny.toml | sort -u | sed 's/.*/ { id = & },/' + echo ']' +} > config/osv-scanner.toml +""" +shell = "bash -euo pipefail -c" + +[tasks.cargo-deny-check] +description = "Audit dependencies with cargo-deny (advisories, bans, licenses, sources)" +run = "cargo deny check --config config/deny.toml" + +[tasks.cargo-unmaintained-check] +description = "Flag dependencies whose upstream repo is archived or unmaintained" +# cargo-unmaintained is deliberately NOT a mise [tool]: it has no prebuilt and a +# from-source `cargo install` is too heavy to pay on every `mise install`. The +# `dependencies` workflow installs it via taiki-e/install-action; this task just +# runs the binary so the audit command lives beside the others. A GITHUB_TOKEN in +# the env lets it check each upstream repo's archival status. +run = "cargo unmaintained" + +[tasks."gen:dockerignore"] +description = "Regenerate .dockerignore from .gitignore + Docker-only excludes" +# Docker reads only the root .dockerignore (never .gitignore or nested ignore +# files), and a bare pattern there is root-anchored -- whereas a slash-less +# .gitignore pattern matches at any depth. So mirror .gitignore, add the +# Docker-only excludes (`.git/`, plus the build recipe + docs so editing them +# doesn't bust the `COPY . .` cache), and `**/`-prefix each non-anchored pattern. +run = ''' +{ + echo "# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit." + echo "# Docker reads only this file; patterns are **/-prefixed to match at any depth" + echo "# like .gitignore. Edit .gitignore and regenerate." + echo + { + cat .gitignore + printf '%s\n' '.git/' 'Dockerfile*' '/README.md' '.dockerignore' + } | awk ' + /^[[:space:]]*#/ { print; next } + /^[[:space:]]*$/ { print; next } + { + neg = ""; line = $0 + if (substr(line, 1, 1) == "!") { neg = "!"; line = substr(line, 2) } + if (substr(line, 1, 1) == "/") { print neg substr(line, 2); next } + core = line; sub(/\/$/, "", core) + if (index(core, "/") > 0) print neg line; else print neg "**/" line + } + ' +} >.dockerignore +''' +shell = "bash -euo pipefail -c" + +[tasks.docker-check] +depends = ["gen:dockerignore"] +description = "Fail if .dockerignore is stale vs .gitignore (run `mise run gen:dockerignore`)" +run = "git diff --exit-code -- .dockerignore" [tasks.prefetch-ci] # `build-ws-wasm-agent` produces `services/ws-wasm-agent/pkg/`, which the @@ -397,7 +545,7 @@ description = "Fail if any checked-in artifact under generated/ is stale" # indentation differently across nightlies; pipx-driven python codegen # emits subtly different output). Linux/macOS runs are canonical. run = """ -[[ "${OS:-}" == "Windows_NT" ]] && { echo "gen-specs-check: skipped on Windows (non-deterministic)"; exit 0; } +[ "${OS:-}" = "Windows_NT" ] && { echo "gen-specs-check: skipped on Windows (non-deterministic)"; exit 0; } git diff --exit-code -- generated """ shell = "bash -euo pipefail -c" @@ -432,7 +580,9 @@ shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent] description = "Build the WebSocket WASM client" dir = "services/ws-wasm-agent" -run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" +# {{ vars.no_opt }} is --no-opt on Windows (wasm-opt can't run on Nano), empty +# elsewhere -- the served, size-optimised artifact is built on Linux/macOS. +run = "wasm-pack build . --target web {{ vars.no_opt }} -- -Z build-std=std,panic_abort" [tasks.build-ws-wasm-agent.env] # Scope these flags to the wasm target rather than setting plain `RUSTFLAGS`: @@ -444,14 +594,17 @@ run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS = "{{ vars.wasm_rustflags }}" RUSTUP_TOOLCHAIN = "nightly" +[tasks.build-et-cli] +# Pre-build et-cli once. The parallel build-ws-* module tasks each call +# `cargo run -p et-cli -- module-package-json`; without this they race rebuilding +# et-cli, and one execs the binary mid-replace (ENOENT -- surfaced on macOS). +description = "Build et-cli (prereq for the module-package-json steps)" +run = "cargo build -p et-cli" + [tasks.build-modules] # Native `depends` glob over `build-ws-*`: that matches `build-ws-wasm-agent` # (always loaded here, no `-module` suffix — so the glob never matches zero) -# plus every loaded `build-ws-*-module`. The module builds now live in MISE_ENV -# configs: the Rust ones in config.rust.toml, the guest ones (dart-comm1, -# dotnet-data1, java-data1, zig-data1, pydata1, pyface1, wasi-graphics-info) in -# theirs — each joins when its env is loaded. CI sets MISE_ENV to every language -# (or use `build-modules-all`). A nested `mise run` here broke tools on Windows. +# plus every loaded `build-ws-*-module`. depends = ["build-ws-*"] description = "Build all loaded WebAssembly modules" @@ -484,15 +637,11 @@ depends = ["openobserve", "ws-server"] run = "open http://localhost:5080/" [tasks.fetch-face1-rclone] -run = """ -rclone copyto {{ vars.face1_src }} {{ vars.face1_dst }} --http-url https://huggingface.co --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.face1_src }} {{ vars.face1_dst }} {{ vars.hf_http }}" shell = "bash -euo pipefail -c" [tasks.fetch-har-motion1-rclone] -run = """ -rclone copyto {{ vars.har_src }} {{ vars.har_dst }} --http-url https://github.com --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.har_src }} {{ vars.har_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" [tasks.fetch-mnist-rclone] @@ -500,9 +649,7 @@ shell = "bash -euo pipefail -c" # wasi-nn end-to-end. Sourced from the canonical ONNX Model Zoo; size 26143 # bytes. Lands under the module's gitignored pkg/ so et-modules-service can # serve it statically alongside the .wasm. -run = """ -rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} --http-url https://github.com --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" [tasks.download-models] diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml new file mode 100644 index 0000000..5dd411e --- /dev/null +++ b/.mise/config.windows.toml @@ -0,0 +1,98 @@ +# Windows-only mise config. Everything here is Windows-scoped, so it needs no +# per-entry `os = ["windows"]` guards or `{% if os() == 'windows' %}` +# conditionals; values here override the cross-platform defaults in config.toml. +# Shared vars (rpath_flag, py313_*, …) come from config.toml, which loads first. + +[tools] +# msys2 mirror packages: git (cargo git deps), gpg (so mise verifies tool +# downloads), make (some -sys build scripts shell out to it). The MSVC CRT + +# Windows SDK can't come from mise, so the build uses the LLVM mingw toolchain +# (the gnullvm target below); llvm-mingw bundles clang + lld + the mingw-w64 +# runtime/headers/libs + the libclang.dll bindgen loads. Pinned so its install +# dir is deterministic for the PATH entry in Dockerfile.nanoserver -- bump both. +"conda:m2-git" = "latest" +"conda:m2-gnupg" = "latest" +"conda:m2-make" = "latest" +"github:mstorsjo/llvm-mingw" = { version = "20260602", matching = "ucrt-x86_64" } + +# busybox-w32 `sh` (POSIX ash) -- the shell the bash-tasks run on. A native Win32 +# single exe with no cygwin/msys runtime, so it loads on a bare Nano Server, +# unlike conda's msys2 bash whose msys-2.0.dll imports KERNEL32!IdnToAscii / +# IdnToUnicode that Nano's stripped kernel32 forwarder doesn't re-export +# (-> 0xC0000139; unfixable -- the system file + registry are ACL-locked). +# Renamed `ash.exe` so mise -- whose POSIX-shell list is bash/sh/zsh/fish/ksh/ +# dash, NOT ash -- doesn't rewrite the task PATH into msys form, which native +# busybox-w32 can't resolve. Checksum-pinned so a silent upstream rebuild fails +# loudly; `.exe` because mise writes the bin name verbatim and Windows launches +# by extension. +[tools."http:busybox"] +bin = "ash.exe" +checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" +url = "https://frippery.org/files/busybox/busybox64u.exe" +version = "1.37.0" + +[vars] +# busybox `ash` install path (for MISE_BASH_PATH), keyed by the pinned +# http:busybox version above -- keep in sync. +winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\ash.exe' +# mise-managed CPython 3.13; the mise data dir is LOCALAPPDATA\mise. +py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" +# clang-sys finds libclang on PATH (Docker's gnu build has llvm-mingw there); a +# workstation with system LLVM also works. +win_libclang = "C:\\Program Files\\LLVM\\bin" +# wasm-pack's wasm-opt (binaryen) can't execute on Nano Server (os error 3), so +# the module + wasm-agent builds pass --no-opt; they reference {{ vars.no_opt }}. +no_opt = "--no-opt" + +[env] +# Run `shell = "bash …"` tasks on busybox ash rather than the Windows WSL +# launcher (mise can't auto-find an http-backend shell). busybox ash accepts the +# `-euo pipefail -c` the shell string passes; the task bodies are POSIX. +MISE_BASH_PATH = "{{ vars.winsh }}" +# Build for the LLVM mingw target. gnullvm, not gnu: llvm-mingw ships +# compiler-rt + libunwind, not the GCC runtime (libgcc/libgcc_eh) the gnu +# target's link line demands. The linker is clang and the raw-dylib import tool +# is llvm-dlltool, both from llvm-mingw on PATH. These can live in [env] here +# (unlike config.toml) because this file only loads on Windows, so there's no +# empty off-Windows value to break Linux cargo. +CARGO_BUILD_TARGET = "x86_64-pc-windows-gnullvm" +CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_LINKER = "clang" +CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTFLAGS = "-Cdlltool=llvm-dlltool" +LIBCLANG_PATH = "{{ vars.win_libclang }}" +PYO3_PYTHON = "{{ vars.py313_win }}" + +[tasks.preinstall] +# Preinstall, shared by Dockerfile.nanoserver and a real workstation. The busybox +# `sh` (http:busybox) is installed by the cmd bootstrap that runs before any bash +# task (Dockerfile.nanoserver / the README), so this task's `bash -euo pipefail` +# shell (= busybox ash via MISE_BASH_PATH) works. Installs +# the build utils (git/gpg/make), llvm-mingw and rust, pip-installs pipx (the +# backend for the pipx:* tools, since aqua has no Windows pipx build), then flips +# rustup's default host to gnullvm so both host and target link via llvm-mingw +# rather than the absent MSVC link.exe (no Visual Studio Build Tools needed). The +# prereqs mise CAN'T supply (a recent mise + the VC++ runtime) are in the README. +depends = ["_setup_all"] +description = "Preinstall: toolchain + pipx + flip rustup to the gnullvm host" +run = """ +mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust + +# pipx backs the pipx:* tools but isn't a mise tool on Windows. Install it with +# whatever python is on PATH: the runner's system python on CI/workstations (its +# Scripts dir, where pipx lands, is already on PATH); Dockerfile.nanoserver puts +# mise's python on PATH before this so the command resolves on Nano too. +python -m pip install pipx + +# rustup installed components/targets against the pre-flip default host, so after +# flipping to gnullvm reinstall the EXACT stable + nightly versions mise pinned as +# gnullvm toolchains, each carrying the components + wasm targets config.toml's +# `rust` tool lists. Pre-adding the wasm target means wasm-pack finds it present +# and never downloads rust-std mid-build. mise sets RUSTUP_TOOLCHAIN to the bare +# stable version, which now resolves to -gnullvm. +ver="$(mise exec -- rustc --version | awk '{print $2}')" +mise exec -- rustup set default-host x86_64-pc-windows-gnullvm +wasm="-t wasm32-unknown-unknown -t wasm32-wasip2" +mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnullvm" -c clippy -c rust-analyzer $wasm +mise exec -- rustup toolchain install nightly-x86_64-pc-windows-gnullvm -c rust-src -c rustfmt $wasm +mise exec -- rustup default "${ver}-x86_64-pc-windows-gnullvm" +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.zig.toml b/.mise/config.zig.toml index a2bc4d1..0404a45 100644 --- a/.mise/config.zig.toml +++ b/.mise/config.zig.toml @@ -7,6 +7,14 @@ # REST-client generation step in et-int-gen detects the missing binary at # runtime and is a no-op when it's not on PATH. "github:christianhelle/openapi2zig" = { version = "latest", os = ["linux/x64", "macos", "windows"] } +# C tooling for the zig-data1 module's hand-written C (services/ws-modules/ +# zig-data1/src/*.c). Two conda packages because mise hands the `clang-format` +# name to conda:clang-format, so conda:clang-tools omits it and only supplies +# clang-tidy. Both are conda-forge's native LLVM binaries. cpplint (Google C++ +# style) has no prebuilt binary -- pipx is its only distribution. +"conda:clang-format" = "latest" +"conda:clang-tools" = "latest" +"pipx:cpplint" = "latest" zig = "latest" [tasks.zig-check] @@ -15,14 +23,41 @@ run = "zig fmt --check services/ws-modules/ generated/zig-rest/" [tasks.zig-fmt] run = "zig fmt services/ws-modules/ generated/zig-rest/" +[tasks.clang-format] +description = "Format C sources (clang-format)" +run = "git ls-files '*.c' '*.h' | xargs clang-format -i --style=file:config/clang-format.yaml" + +[tasks.clang-format-check] +description = "Check C source formatting (clang-format)" +run = "git ls-files '*.c' '*.h' | xargs clang-format --dry-run --Werror --style=file:config/clang-format.yaml" + +[tasks.clang-tidy-check] +description = "Lint C sources (clang-tidy)" +# Flags after `--` are the compile args; clang_resource_arg points clang at its +# builtin headers (set per host -- see config.linux.toml). {{ }} renders to empty +# where unset, leaving a bare `--`. +run = """ +git ls-files '*.c' '*.h' | + xargs -I{} clang-tidy --config-file=config/clang-tidy.yaml {} -- {{ vars.clang_resource_arg }} +""" +shell = "bash -euo pipefail -c" + +[tasks.cpplint-check] +description = "Lint C sources for Google C++ style (cpplint)" +run = """ +git ls-files '*.c' '*.h' | + xargs cpplint --quiet --linelength=120 --filter=-legal/copyright,-build/include_subdir +""" +shell = "bash -euo pipefail -c" + # Namespaced aggregators picked up by the default config's globbed `check`/`fmt`. [tasks."check:zig"] -depends = ["zig-check"] -description = "Run Zig checks (format-check)" +depends = ["zig-check", "clang-format-check", "clang-tidy-check", "cpplint-check"] +description = "Run Zig + C checks (zig fmt-check, clang-format, clang-tidy, cpplint)" [tasks."fmt:zig"] -depends = ["zig-fmt"] -description = "Format Zig sources" +depends = ["zig-fmt", "clang-format"] +description = "Format Zig + C sources" [tasks.build-ws-zig-data1-module] description = "Build the zig-data1 workflow WASM module" diff --git a/.miserc.toml b/.miserc.toml new file mode 100644 index 0000000..36be63a --- /dev/null +++ b/.miserc.toml @@ -0,0 +1,8 @@ +# Early-init mise settings (read before the regular config, so they can affect +# which config files load). auto_env turns on platform environments: mise then +# loads .mise/config..toml automatically for the current OS -- so +# config.linux.toml on Linux, config.macos.toml on macOS, and config.windows.toml +# on Windows, with no manual MISE_ENV. Setting it here also silences mise's +# 2026.12 "auto_env is becoming default" rollout warning. +# See https://mise.jdx.dev/configuration/environments.html +auto_env = true diff --git a/CLAUDE.md b/CLAUDE.md index 9b90d79..c7a793d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,25 @@ on a follow-up fix-up pass. The most common offenders are: long `reason = "…"` strings on lint attributes, JSON `description` fields, markdown table rows, and CI-task `description` fields. +## Document each thing exactly once + +Document each thing **once**, in the single place it is most relevant — +the code, config entry, or task it describes — and **nowhere else**. Do +not restate the same explanation in a second comment, in the README, in +this file, or in a commit message that competes with it. + +Do not even add a pointer to where something is documented ("see X", +"as described in Y", "(documented in Z)"). Such cross-references are +themselves duplication: they go stale, and they multiply the places that +must change when the thing changes. Trust the developer to find the +relevant documentation themselves — they know how to read the code, +`grep`, and follow the obvious file. + +When you find yourself about to explain something that is already +explained elsewhere, stop: either the existing spot is the right home (so +say nothing here), or this is the better home (so move it here and remove +the original). One canonical location, never two. + ## Prerequisites Install [`mise`](https://mise.jdx.dev/) with shell integration, then configure: @@ -75,6 +94,40 @@ use the `*-all` variants (`check-all`, `test-all`, `build-modules-all`, (e.g., `mise run build-ws-face-detection-module` for the Rust modules, or `MISE_ENV=zig mise run build-ws-zig-data1-module`). +## Formatters & checks by file type + +The `mise run ` formatters and checks per file type (`fmt` / `check` run +them all; guest rows need their `MISE_ENV` loaded). + +| File type | Formatter task(s) | +| --------- | ------------------------------- | +| `*.rs` | `cargo-fmt`, `cargo-clippy-fix` | +| `*.toml` | `taplo-fmt` | +| `*.py` | `ruff-fmt` | +| `*.dart` | `fmt:dart` | +| `*.zig` | `fmt:zig` | +| `*.c` | `clang-format` | +| `*.cs` | `fmt:dotnet` | + +| File type | Check task(s) | +| --------- | ---------------------------------------------------------------------------------------- | +| `*.rs` | `cargo-check`, `cargo-clippy`, `cargo-fmt-check`, `cargo-doc-check`, `ast-grep-check` | +| `*.toml` | `taplo-check`, `conftest-check-toml`, `semgrep-check` | +| `*.yaml` | `ast-grep-check`, `conftest-check-yaml`, `ryl-check`, `action-validator`, `zizmor-check` | +| `*.json` | `semgrep-check` | +| `*.py` | `check:python` | +| `*.dart` | `check:dart` | +| `*.zig` | `check:zig` | +| `*.c` | `clang-format-check`, `clang-tidy-check`, `cpplint-check` | +| `*.cs` | `check:dotnet` | +| `*.java` | `check:java` | + +`dprint-fmt` / `dprint-check` cover `*.md`, `*.yaml`, `*.json`/`*.jsonc`, +`*.ts`/`*.js`, `*.css`, `*.html`, `*.java`, and `Dockerfile*`; `hadolint-check` +also lints Dockerfiles, and `link-check` scans `*.md` + `*.rs`. Every file is +covered by `editorconfig-check` and `typos`, file and directory names by +`ls-lint-check`, and `*.yml` is rejected by `semgrep-check` (use `*.yaml`). + ## Architecture This is a WebSocket-based edge computing framework. @@ -236,12 +289,99 @@ If a function is private but needs testing, add a `[lib]` target to the crate an Every file under `tests/` must start with `#![cfg(test)]` (placed after the file's `//!` doc comment, if any). +## Tools must work on every OS + +Every tool in the `.mise/config*.toml` `[tools]` tables must install and run on +every supported OS (Linux, macOS, Windows). Do **not** `os`-scope a tool, or +otherwise skip it on a platform, without explicit operator permission — prefer a +prebuilt-binary backend (aqua/github/http) over a `cargo:` source build, which is +usually what forces a platform exclusion. The one place tool skips need no +permission is the Dockerfiles (`MISE_DISABLE_TOOLS`), where trimming an image to +just what its build needs is expected. + +## Linting + +Lint checks must be expressed through one of the repo's linters — **never** as a +bespoke shell script, whether a standalone file or a mise task `run`. The +available linters: + +- **ast-grep** (`config/ast-grep/rules/`) — structural rules for code **and + YAML** (e.g. GitHub Actions workflows). +- **semgrep** (`config/semgrep/`) — incl. `languages: [generic]`, which works on + TOML/text (e.g. `mise-config.yaml` lints `.mise/config*.toml`). +- **taplo** JSON-schemas (`config/taplo/`) — TOML structure, applied via + `taplo lint --schema` in `taplo-check`. +- **conftest** (`config/conftest/policy/`) — Rego policies over the combined + TOML/YAML config set, for cross-file checks the schema linters can't express. +- plus hadolint, ls-lint (file/dir naming), zizmor (Actions security), ryl + (YAML), lychee (links), clang-format / clang-tidy / cpplint (C, in the zig + config), editorconfig-checker, typos, and action-validator for their domains. + +ast-grep has no TOML grammar, so it **cannot** lint TOML — use a taplo schema or +a semgrep `generic` rule there. If none of the above can express a check, +propose adding a new mise-installable linter rather than scripting it by hand. + +## No `scripts/` directory + +Do not create a `scripts/` directory or drop loose shell/Python scripts in the +repo. Every script belongs in one of two places: + +- **Short and simple** → an inline `mise` task (`run = """ … """` in + `.mise/config.toml` or a `.mise/config..toml`). It stays discoverable + via `mise tasks` and runs as `mise run `. +- **More involved** → its own tool directory under `utilities/` with its own + `README.md` documenting what it does and how to run it. + ## Rust Workspace Single Cargo workspace (`Cargo.toml`). Shared dependency versions are declared in `[workspace.dependencies]`. Add new deps there, not in individual crate `[dependencies]`. +## Clippy lints + +**Never weaken or disable a lint to make code pass — not the workspace lint +config (`[workspace.lints.*]`, `.clippy.toml` thresholds, the ast-grep / taplo +rules) — without explicit operator permission.** Setting a denied lint to +`allow` or raising a threshold is a project-policy change, not a fix. If a lint +is in the way, fix the code (or justify it with a scoped +`#[expect(..., reason = "…")]`); if you believe the lint itself is wrong, stop +and ask. + +Clearing an `#[expect(...)]` that clippy reports as **unfulfilled** is normally +fine and expected — that's the intended cleanup. **The one exception is +`clippy::cognitive_complexity`** (and other macro-expansion-sensitive lints): +do **not** auto-remove those `#[expect]`s, because of the gotcha below. + +**Feature-unification gotcha.** `clippy::cognitive_complexity` can fire in the +full-workspace build but not in an isolated `cargo clippy -p `, because +`-p` enables fewer features (e.g. the `tracing` macros expand further when the +whole workspace's tracing/otel features are on). So its `#[expect]` can look +_unfulfilled_ in an isolated run yet be _required_ by CI. **Validate with the +full `mise run check`, not an isolated `-p` clippy, before touching one of these +`#[expect]`s.** (The clean fix — `[resolver] feature-unification = "workspace"` +so every build sees the same features — is still nightly-only via +`-Z feature-unification`.) + +The workspace denies a broad set of clippy lints (see `[workspace.lints.clippy]` +in `Cargo.toml`), including restriction lints. One you'll hit often: +**`clippy::single_call_fn`** fires on a private function called from exactly one +site. Do **not** inline the function just to silence it — a function that is a +distinct, named step (kept separate for readability, or that will gain more +callers) is legitimate. Keep it and annotate with `#[expect(...)]` and a real +justification: + +```rust +#[expect(clippy::single_call_fn, reason = "distinct step of X; kept separate for readability and future reuse")] +fn helper(...) { ... } +``` + +Use `#[expect(...)]` rather than `#[allow(...)]` (the workspace denies +`unfulfilled_lint_expectations`, so an `expect` that stops applying fails the +build instead of silently lingering). The same applies to other restriction +lints whose pattern is intentional in a given spot — prefer a justified +`#[expect(..., reason = "…")]` over contorting the code to dodge the lint. + ## Naming conventions - **`.map_err` wrappers must be named `map_*`.** Extension methods that diff --git a/Cargo.lock b/Cargo.lock index e5f6a73..8e1a00e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3998,8 +3998,8 @@ version = "0.1.0" dependencies = [ "asyncapi-rust", "base64 0.22.1", + "et-path", "fs-err", - "lets_find_up", "log", "rstest", "schemars 1.2.1", @@ -4160,6 +4160,7 @@ version = "0.1.0" dependencies = [ "clap", "edge-toolkit", + "et-path", "fs-err", "serde", "serde_json", @@ -4244,6 +4245,13 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "et-path" +version = "0.1.0" +dependencies = [ + "tempfile", +] + [[package]] name = "et-rest-client" version = "0.1.0" @@ -4611,6 +4619,7 @@ dependencies = [ name = "et-ws-wasi-comm1" version = "0.1.0" dependencies = [ + "et-path", "serde_json", "wit-bindgen 0.57.1", ] @@ -4619,6 +4628,7 @@ dependencies = [ name = "et-ws-wasi-data1" version = "0.1.0" dependencies = [ + "et-path", "serde_json", "wit-bindgen 0.57.1", ] @@ -4635,10 +4645,10 @@ dependencies = [ "et-ws-runner-common", "et-ws-test-server", "futures-util", + "int-otlp-mock", "opentelemetry 0.31.0", "opentelemetry-http 0.31.0", "ort", - "otlp-mock", "pollster", "reqwest 0.13.4", "rstest", @@ -6249,6 +6259,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "int-otlp-mock" +version = "0.1.0" +dependencies = [ + "actix-rt", + "actix-web", + "serde_json", +] + [[package]] name = "inventory" version = "0.3.24" @@ -6636,12 +6655,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "lets_find_up" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8052b3d5cfa8bae8af3b44aae11a43e9fa48ce0ae477c4a39733a8deff34059" - [[package]] name = "libc" version = "0.2.186" @@ -7944,15 +7957,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "otlp-mock" -version = "0.1.0" -dependencies = [ - "actix-rt", - "actix-web", - "serde_json", -] - [[package]] name = "outref" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index c36e4bd..9fe0539 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "libs/edge-toolkit", "libs/et-otlp", "libs/otlp-mock", + "libs/path", "libs/web", "libs/ws-runner-common", "services/ws-modules/audio1", @@ -65,6 +66,7 @@ deno_runtime = { version = "0.257.0", features = ["transpile", "hmr"] } edge-toolkit = { path = "libs/edge-toolkit", version = "0.1.0" } et-modules-service = { path = "services/modules", version = "0.1.0" } et-otlp = { path = "libs/et-otlp", version = "0.1.0" } +et-path = { path = "libs/path", version = "0.1.0" } et-rest-client = { path = "generated/rust-rest", version = "0.1.0", default-features = false } et-storage-service = { path = "services/storage", version = "0.1.0" } et-web = { path = "libs/web", version = "0.1.0" } @@ -81,7 +83,6 @@ hostname = "0.4" humantime-serde = "1" js-sys = "0.3" kdl = { version = "6", features = ["v1"] } -lets_find_up = "0.0.4" local-ip-address = "0.6" log = "0.4" onnx-extractor = "0.3" @@ -98,7 +99,7 @@ opentelemetry-otlp = { version = "0.31", default-features = false, features = [ ] } opentelemetry_sdk = "0.31" ort = { version = "=2.0.0-rc.10", default-features = false, features = ["copy-dylibs", "download-binaries"] } -otlp-mock = { path = "libs/otlp-mock", version = "0.1.0" } +int-otlp-mock = { path = "libs/otlp-mock", version = "0.1.0" } pollster = "0.4" pretty_yaml = "0.6" prettyplease = "0.2" @@ -189,7 +190,7 @@ unsafe_op_in_unsafe_fn = "deny" # sibling test crates that don't import them. Tracked at rust-lang/rust#95513; re-enable when # rustc gains a workspace-aware view. # unused_crate_dependencies = "deny" -unused_results = "warn" +unused_results = "deny" [workspace.lints.rustdoc] private_intra_doc_links = { level = "deny", priority = 8 } @@ -342,7 +343,7 @@ ignore = [ # via wgpu. Compile-time only. "paste", # Archived upstream at GnomedDev/proc-macro-error-2 (the maintained - # fork of the original proc-macro-error). Both now unmaintained. + # fork of the original proc-macro-error). Both unmaintained. # Pulled by getset via neli via local-ip-address. Compile-time only. "proc-macro-error-attr2", "proc-macro-error2", diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c45132a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,158 @@ +# Build, test, and serve edge-toolkit from a clean, minimal Ubuntu, split into +# stages so each can be cached and targeted independently: +# +# build-minimal mise + the always-loaded toolchain (.mise/config.toml only) +# build + the guest-language toolchains (.mise/config..toml) +# prefetch download all dependencies + ONNX models +# precompile build the WASM/JS modules (drops target/ to stay slim) +# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) +# server release build of et-ws-server, served by default (final stage) +# +# Each stage `FROM`s the previous, so installed tools, downloaded deps, and the +# built module pkg/ carry forward. Stop early with `--target`. +# +# The build stages run the mise setup verbatim (install mise -> configure -> +# install conda:openssl -> install), split so build-minimal (the always-loaded +# tools) caches separately from the guest languages (build). The OpenObserve +# (`o2`/`open-o2`) and `ws-server` runtime services are intentionally skipped -- +# they aren't build/test steps. +# +# It also catches setup drift: a missing or wrong step fails the build. Anything +# language-specific must come from mise (the "installed into the local +# workspace" promise); the only apt packages below are universal build prereqs a +# normal dev machine has, so a failure that needs another system lib is itself a +# finding worth documenting. +# +# A plain build produces the SERVER image (final stage): a release et-ws-server, +# served automatically. A GitHub token avoids mise's anonymous GitHub rate limit +# during install-all: +# DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . +# docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 +# (drop --secret to build tokenless; install-all may then hit rate limits) +# +# To run the verification suite, target the non-final `test` stage and pass the +# host GPU (`docker build` can't attach one). The stage bundles mesa-vulkan- +# drivers, so the wgpu test gets a real Intel/AMD GPU via the DRI node (or a +# software fallback if none is passed): +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# NVIDIA via `--gpus all` is wired but UNVERIFIED (its in-container Vulkan ICD +# doesn't initialize yet) -- prefer a DRI device. + +# --- build-minimal: mise + the always-loaded toolchain (config.toml only). --- +# Copies just .mise/config.toml + installs the default tools, so this layer is +# reused until the always-loaded toolset changes -- not when a guest config does. +FROM ubuntu:24.04 AS build-minimal + +# Universal prereqs a typical dev box already has; everything else is mise's job. +# gcc, g++, libc6-dev and make are the C/C++ toolchain rustc links through (`cc`) +# and that C/C++ `-sys` crates build with (make for build scripts that shell out +# to it) -- leaner than build-essential, which also pulls dpkg-dev + perl. +# curl + ca-certificates fetch the mise installer and tool downloads; git is for +# cargo + repo operations; gnupg (gpg + gpg-agent + dirmngr) lets mise verify +# downloads (bare `gpg` lacks the agent/dirmngr it needs); xz-utils, unzip and +# bzip2 unpack mise's tool archives (e.g. the pyodide .tar.bz2). libicu74 is .NET +# runtime ICU for the dotnet-data1 module -- without it the dotnet CLI +# FailFast-aborts at startup ("Couldn't find a valid ICU package installed on the +# system"; minimal Ubuntu ships no ICU). The "74" tracks the Ubuntu base +# (74 = 24.04) -- bump it alongside the FROM line; .NET needs libicu on minimal +# systems (else set System.Globalization.Invariant=true). +# Vulkan for the wgpu test (libvulkan1 + mesa-vulkan-drivers) is installed in the +# test stage, not here. +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# Install mise and put it + its shims on PATH; in a non-interactive build that's +# the equivalent of the shell integration -- every `mise` / `mise run` below +# then resolves the workspace tools. +RUN curl -fsSL https://mise.run | sh +ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" +# cargo-expand is a dev-only macro-debugging tool with no role in image builds, +# so skip it here (this ENV is inherited by every stage below). +ENV MISE_DISABLE_TOOLS=cargo:cargo-expand + +WORKDIR /workspace +# The default tools need the always-loaded config plus config.linux.toml (where +# preinstall + the Linux env live) and .miserc.toml (auto_env, which auto-loads +# config.linux.toml); the other guest-language configs come in the build stage +# below. These are repo configs, so they're copied + trusted before mise runs. +COPY .miserc.toml .miserc.toml +COPY .mise/config.toml .mise/config.toml +COPY .mise/config.linux.toml .mise/config.linux.toml + +RUN mise trust + +# Preinstall via the shared preinstall task (the same a Linux workstation runs): +# its setup-all base enables experimental + cargo.binstall and installs +# cargo-binstall, node and conda:openssl; then `mise install` adds the rest of +# the always-loaded tools. A GitHub token (if provided) lifts the anonymous rate +# limit for the release fetches. MISE_JOBS=1 serializes the install: the `rust` +# tool is a two-version list (latest + nightly) that otherwise runs two +# rustup-inits at once, racing on the shared rustup binary's self-update (exit 1). +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + sh -c 'mise run preinstall && MISE_JOBS=1 mise install' + +# --- build: add the guest-language toolchains (config..toml). --- +# install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are +# already installed by build-minimal, so this adds dart/dotnet/java/zig/etc. +FROM build-minimal AS build +COPY .mise/ .mise/ +RUN mise trust +ENV MISE_ENV="dart,dotnet,java,python,rust,zig" +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + mise install-all + +# --- prefetch: download all dependencies + ONNX models. --- +# The full source is needed from here on (module builds, cargo fetch, pnpm). +FROM build AS prefetch +COPY . . +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + mise run prefetch + +# --- precompile: build the WASM/JS modules (needed by test and server). --- +# `&& rm -rf target` in the SAME layer: build-modules leaves multi-GB cargo +# intermediates in target/, but the module outputs live in each module's pkg/. +# Dropping target/ here keeps it out of this layer and the stages built on it; +# test and server recompile only what they need. +FROM prefetch AS precompile +RUN mise run build-modules && rm -rf target/ + +# --- test: the full suite (Rust + web runner + every guest language). --- +# Compiled AND run at `docker run` time (precompile keeps no target/), so the +# multi-GB debug test binaries never bake into a layer -- they live in the +# ephemeral container and vanish when it exits. The wgpu compute test needs +# Vulkan: libvulkan1 (the loader) + mesa-vulkan-drivers give a real Intel/AMD GPU +# when the host DRI node is passed with `--device /dev/dri`, else a CPU (lavapipe) +# fallback so the suite still runs. (NVIDIA's `--gpus` Vulkan path doesn't +# initialize in a container.) Both live here, not the build stage, to keep that +# layer cached. +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# NVIDIA via `--gpus all` is wired (NVIDIA_DRIVER_CAPABILITIES=all below, needs +# the NVIDIA Container Toolkit) but UNVERIFIED -- its in-container Vulkan ICD +# doesn't initialize yet, so prefer a DRI device for now. +FROM precompile AS test +ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=all +RUN apt-get update \ + && apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers \ + && rm -rf /var/lib/apt/lists/* +CMD ["mise", "run", "test"] + +# --- server: release build of et-ws-server, the default image (final stage). --- +# A plain `docker build` produces this. The release binary is copied out and +# target/ dropped in the SAME layer so the build intermediates don't bloat the +# image; the binary finds its libs via baked rpaths and serves each module from +# its pkg/ (none of which live in target/). mise stays on PATH and MISE_ENV is +# set, so the server's `mise where` module-path lookups resolve. +# docker run --rm -p 8080:8080 edge-toolkit # then open http://localhost:8080 +FROM precompile AS server +RUN mise exec -- cargo build --release -p et-ws-server \ + && cp target/release/et-ws-server /usr/local/bin/et-ws-server \ + && rm -rf target/ +EXPOSE 8080 8443 +CMD ["et-ws-server"] diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver new file mode 100644 index 0000000..b4ec877 --- /dev/null +++ b/Dockerfile.nanoserver @@ -0,0 +1,262 @@ +# escape=` +# Parser directive (must be line 1): sets the line-continuation / escape +# character to a backtick instead of the default `\`, so backslashes in Windows +# paths (C:\..., trailing `C:\`) stay literal and lines continue with a backtick. + +# hadolint runs ShellCheck (POSIX sh) over every RUN, but these are Windows cmd, +# so it false-errors on cmd `if` (SC1072) and on `\x` in paths like C:\gh_token +# (SC1001). hadolint/hadolint#645. Suppress just those two for this file (a file- +# scoped pragma, so the Linux Dockerfiles keep full ShellCheck coverage). +# hadolint global ignore=SC1072,SC1001 + +# Staged to mirror the Linux Dockerfile so the layers cache + target the same +# way: vcruntime -> build-minimal -> build -> prefetch -> precompile -> test / +# server. Each stage `FROM`s the previous, so installed tools, fetched deps and +# built module pkg/ carry forward; `--target ` stops early. The +# Windows-only bootstrap (VC++ runtime, mise.exe, HOME/TEMP, the gnu-host rust +# flip) lives in build-minimal. Where Linux stops at a clean build/test/serve, +# Windows is still proving the toolchain installs at all -- the stages past +# `build` are the CI frontier (see "Known unknowns" at the foot). +# +# Base: Nano Server, the smallest Windows base (~120 MB vs Server Core's ~1.25 +# GB). It has no MSI/installer stack, no PowerShell, and runs unprivileged -- so +# the Visual Studio Build Tools installer can't run here at all. That's the +# point: nothing is installed the Windows way. mise (a single .exe the workflow +# stages in the build context) is copied in, and `mise install` pulls the rest. + +# --- vcruntime: donor stage for the VC++ runtime DLLs. --- +# Nano Server omits the VC++ runtime that msvc-built executables (mise.exe, and +# the rust/cargo mise installs) need just to *start* -- without it mise.exe dies +# with 0xC0000135 (DLL not found). No base Windows image ships it (not even +# Server Core), and mise can't install it (mise needs it to run), so we install +# the official VC++ redistributable on a Server Core donor -- which has the +# installer stack Nano Server lacks -- and copy the runtime DLLs forward. This +# plus mise are the only non-mise bootstrap bits; the rest is mise. +FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS vcruntime +SHELL ["cmd", "/S", "/C"] +# `|| ver>nul` swallows the redistributable's 3010 ("reboot required") exit code. +RUN curl -fsSL -o vc_redist.exe https://aka.ms/vs/17/release/vc_redist.x64.exe && ` + (vc_redist.exe /install /quiet /norestart || ver>nul) && ` + del vc_redist.exe + +# --- build-minimal: mise + the always-loaded toolchain (config.toml only). --- +# Mirrors the Linux build-minimal: copies just .mise/config.toml + installs the +# default tools, so this layer is reused until the always-loaded toolset changes, +# not when a guest config or the source does. All the Windows-only bootstrap +# (VC++ DLLs, mise.exe, HOME, TEMP) and the gnu-host rust flip live here too, +# since every later stage builds on it. +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 AS build-minimal +COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ +COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ +COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ + +# cmd is the only shell Nano Server has; ContainerAdministrator so we can write +# under C:\ (the backtick line continuations below rely on the escape directive +# documented at the top of this file). +USER ContainerAdministrator +SHELL ["cmd", "/S", "/C"] + +# Diagnostic (informational -- never fails the build): does this base image ship +# a bash? The `bash -euo pipefail` mise tasks need one, and Nano Server doesn't +# bundle it. Record which it is in the build log so the "is bash present" unknown +# is answered up front. +RUN where bash >nul 2>&1 && (echo [bash-check] pre-installed: & where bash) || ` + echo [bash-check] bash is not pre-installed on this base image + +# Set HOME to the admin profile so the container has a sane home (Nano Server +# leaves it unset). This also renders the config's `{{ env.HOME }}` paths; note +# mise still resolves its *global config* to C:\.config\mise on Windows +# regardless of HOME (trusted explicitly below). +ENV HOME=C:\Users\ContainerAdministrator + +# Sanity-check that HOME is the running user's real profile: it must exist and +# end with the current %USERNAME%. USERPROFILE is echoed for diagnostics. +RUN echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%] USERNAME=[%USERNAME%] & ` + if not exist "%HOME%\" exit /b 1 & ` + echo %HOME%| findstr /i /e /c:"%USERNAME%" >nul & ` + if errorlevel 1 exit /b 1 + +# Nano Server's default temp dir can be missing/unwritable, which fails rustup's +# "persist temporary file" step (Access denied) installing toolchains. Point +# TEMP/TMP at a writable dir on C: (same volume as mise's installs). +RUN if not exist C:\Temp mkdir C:\Temp +ENV TEMP=C:\Temp ` + TMP=C:\Temp + +# wasm-pack's binary cache (binary-install's Cache::new) resolves its dir via +# dirs_next, which on Windows calls the SHGetKnownFolderPath shell API that Nano +# lacks (the same gap that stops pipx) -- so it fails with "couldn't find your +# home directory, is $HOME not set?" (build-ws-wasm-agent's `-Z build-std` hits +# it first). WASM_PACK_CACHE makes wasm-pack use Cache::at() instead, which +# stores the path verbatim and never calls dirs (the dir is created on first +# use). It's a transient build cache, so it lives under the temp dir above. +ENV WASM_PACK_CACHE=C:\Temp\wasm-pack + +# Serialize mise's installs (MISE_JOBS=1). On Nano Server the parallel cargo: +# source builds -- no gnu binstall prebuilts exist, so they fall back to `cargo +# install` -- race on rustup's shared .rustup\downloads dir: one build's +# component .partial vanishes before another rustup can rename it ("cannot find +# the file", os error 2). Windows CI doesn't hit this (msvc binstall prebuilts +# mean no source builds); the gnu Docker path does. One job at a time avoids it. +ENV MISE_JOBS=1 + +# Full Rust backtraces so any panic surfaces in the CI log. +ENV RUST_BACKTRACE=full + +# Verbose mise is disabled -- it floods the log with tool/task resolution and the +# exact bash it launches. Re-enable it (noisy but invaluable) when diagnosing a +# Windows build failure on the bare-Nano-Server path: +# ENV MISE_VERBOSE=1 ` +# MISE_LOG_LEVEL=debug + +# mise.zip is staged in the build context by the windows job in +# .github/workflows/docker.yaml (which pins the version) and copied in here. We +# don't curl a versioned URL inside the build: the classic Windows builder +# doesn't substitute build-args into RUN, and mise's `mise-latest-windows-x64.zip` +# prebuilt is stale (2026.3.0, which predates the nested `task.run_auto_install` +# in .mise/config.toml and errors on it). Extract with Nano Server's tar.exe (zip +# is mise\bin\mise.exe) and put that on PATH so the steps below can call `mise`. +COPY mise.zip C:\mise.zip +RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip +# Prepend mise's bin, and re-list System32 + Windows explicitly: the base image +# exposes its search path as `Path`, which Docker's case-sensitive `${PATH}` +# doesn't match, so it expands empty -- dropping System32 (findstr, net, sc, ...) +# off PATH for every step below. Also put llvm-mingw's bin on PATH: the cargo: +# tool builds (windows-sys etc.) invoke `x86_64-w64-mingw32-dlltool` + the gnu +# linker by name, and mise's cargo backend doesn't activate llvm-mingw onto the +# build subprocess's PATH. The dir is the pinned-version install (config.toml); +# it doesn't exist until preinstall installs llvm-mingw, which is fine -- the +# cargo: builds that need it run afterwards. +ENV LLVMBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-mstorsjo-llvm-mingw\20260602\bin +ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${LLVMBIN};${PATH}" + +WORKDIR C:\workspace +# config.toml (always-loaded) + config.windows.toml (the Windows tools/env/task) +# + .miserc.toml (auto_env=true, which makes mise load config.windows.toml on +# Windows). The guest-language configs come in the build stage below. +COPY .mise/config.toml .mise/config.toml +COPY .mise/config.windows.toml .mise/config.windows.toml +COPY .miserc.toml .miserc.toml + +# gh_token (a GitHub token) is OPTIONAL. With it, mise's GitHub fetches use the +# authenticated rate limit; without it they fall back to the lower anonymous one, +# which can rate-limit `install`. The classic Windows builder has no +# BuildKit secrets, so the token is read per-RUN from a file in the build context +# (it can't be a build-arg substituted into RUN). The glob makes it optional: +# paired with an always-present file (.miserc.toml) so COPY still succeeds when +# gh_token is absent; both land in C:\token (a dir off the WORKDIR ancestor chain, +# so the paired file is inert). WARNING: when a token IS supplied it bakes into +# this image layer -- do NOT publish an image built with one (see the README). +COPY .miserc.toml gh_token* C:/token/ + +# Docker-only mise settings, written inline in cmd (no bash needed): `mise +# settings` writes the global config (C:\.config\mise on Windows, not +# auto-trusted -- hence the explicit trust); experimental enables the conda/cargo +# backends; cargo.binstall lets cargo: tools use prebuilt binaries where they +# exist; aqua signature checks (attestations/cosign/slsa) are off (their +# sigstore/TUF step can't find a cache dir here); pipx.uvx=false makes the pipx: +# backend use pip not uv (uv's PE-resource trampoline isn't implemented on Nano). +RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN=nul + +# Pin CARGO_HOME/RUSTUP_HOME to the dirs mise's rust tool uses (C:\.cargo / +# C:\.rustup -- see its exec_env). preinstall sets the gnullvm default +# toolchain in C:\.rustup, but when mise's cargo backend builds a `cargo:` tool it +# spawns `cargo install` WITHOUT rust's exec_env, so rustup would otherwise look +# in the empty %USERPROFILE%\.rustup and fail with "no default is configured". +# Setting these globally makes every cargo/rustup invocation find the default. +ENV CARGO_HOME=C:\.cargo ` + RUSTUP_HOME=C:\.rustup + +# preinstall pip-installs pipx (the pipx:* backend). Nano has no system python, +# so install mise's python and put it on PATH first; pipx then lands in that +# python's Scripts dir -- never invoked here (this image disables pipx:semgrep and +# builds no python guest, and pipx can't run on Nano anyway), so that's fine. +RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN=.toml). --- +# MISE_ENV lists only the guests that install on Nano Server: dotnet is excluded +# (its installer is a PowerShell script, and Nano has no PowerShell) and python is +# excluded (its tools install via pipx, which can't run on Nano). dart/java/rust/ +# zig install cleanly. Use `mise install` rather than install-all (which forces +# $ALL_LANGS) so this restricted MISE_ENV is honored; it persists to later stages. +FROM build-minimal AS build +COPY .mise/ .mise/ +RUN mise trust +ENV MISE_ENV=dart,java,rust,zig +RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN=\release\; copy it onto PATH (C:\mise\bin) and drop target/ in +# the same layer so the build intermediates don't bloat the image. The server +# serves each module from its pkg/ (none of which live in target/). +FROM precompile AS server +RUN mise exec -- cargo build --release -p et-ws-server && ` + copy target\x86_64-pc-windows-gnullvm\release\et-ws-server.exe C:\mise\bin\ && ` + if exist target rmdir /s /q target +EXPOSE 8080 8443 +CMD ["et-ws-server"] diff --git a/README.md b/README.md index ef443c3..3cb0df5 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ data stay on the device or your own network, never sent to an external cloud ser ## mise -Please install [`mise`](https://mise.jdx.dev/), including the shell integration. +Please install [`mise`](https://mise.jdx.dev/) (2026.6.5 or later), +including the shell integration. It is needed for all use of this repository. The `mise` configuration lives under [`.mise/`](.mise/): the always-loaded @@ -28,58 +29,132 @@ all of them at once. The following works for Linux, macOS and Windows, and all tools "installed" are only installed into the local workspace, so no need for admin/root privileges. -Configure it with: +Then run the preinstall task for your platform — it configures mise and installs +the shared basics (cargo-binstall, node, the openssl dev files) plus whatever +that platform needs. Do any manual prerequisites in your platform's section +below first. + +### GitHub rate limits + +`mise install` downloads many tools from GitHub releases, which are subject to +[GitHub's REST API rate limits](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api). +Unauthenticated requests share a low per-IP limit, so installs can fail with +`GitHub rate limit exceeded`. Authenticate with the +[`gh` CLI](https://cli.github.com/) and let `mise` reuse its token — this raises +the limit and needs no token scopes: ```bash -mise settings experimental=true -mise settings set cargo.binstall true +mise install gh ``` -Pre-install `cargo-install`, which can be done using: - ```bash -mise use -g cargo-binstall +gh auth login # any method; no scopes needed +mise settings set github.credential_command "gh auth token" +mise token github # verify: should resolve a token ``` -### Windows only +`gh` often stores the token in the OS keyring rather than in +`~/.config/gh/hosts.yml`, so mise's default `hosts.yml` lookup finds nothing; +`credential_command` asks `gh` for the token on demand and works either way. The +setting is written to your global `~/.config/mise/config.toml`, so it stays +per-machine and out of the repo. -On Windows only, `pipx` also needs to be pre-installed. -See the Windows section of [pipx instructions](https://pipx.pypa.io/stable/how-to/install-pipx/). +### Microsoft VC++ runtime -### MacOS only +mise.exe links the Microsoft VC++ runtime (`vcruntime140.dll`), so it must be +present or mise won't start. It's preinstalled on Windows 10/11 and Server, so +you already have it — only Nano Server omits it, and there the Docker build +installs the [VC++ Redistributable](https://aka.ms/vs/17/release/vc_redist.x64.exe). -On MacOS, the Xcode Command Line Tools (`clang`, `git`, `make`, etc.) must be -installed first: +### Windows shell -```bash -xcode-select --install -``` - -We also need to install a better linker into the workspace. +On Windows, install the shell: ```bash -mise install conda:lld +mise install http:busybox ``` ### All OS -Before installing dependencies, please the install openssl development files -separately: +To install dependencies: ```bash -mise install conda:openssl +mise run preinstall +mise install-all ``` -Then install the remaining dependencies: +The `preinstall` task will advise if there are any required dependencies are +are missing, such as Xcode Command Line Tools on MacOS. + +### Install failures + +`mise install` runs tool installs in parallel. If they fail intermittently — a +download race, or a `cargo:` source build colliding with another — serialize +them with `MISE_JOBS=1`: ```bash -mise install-all +MISE_JOBS=1 mise install-all ``` +This is the same workaround both Docker builds bake in, so reach for it first if +a local install or build misbehaves. + ## Contributing Use `mise run fmt-all` and `mise run check-all` to run formatters and checkers. +## Building and running with Docker + +[`Dockerfile`](Dockerfile) reproduces the mise setup above on a clean, minimal +Ubuntu, in stages (`build` → `prefetch` → `precompile` → `test`/`server`). A +plain build produces the **server** image (the final stage): a release build of +`et-ws-server`, served automatically. `mise install-all` fetches many tools from +GitHub releases, so build with a GitHub token to avoid the anonymous rate limit +(see [GitHub rate limits](#github-rate-limits)), passed as a BuildKit secret so +it never lands in an image layer: + +```bash +GITHUB_TOKEN="$(gh auth token)" DOCKER_BUILDKIT=1 \ + docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . +docker run --rm -p 8080:8080 edge-toolkit +``` + +Then open (add `-p 8443:8443` for TLS). The server needs +no GPU. OpenObserve/`o2` is optional — OTLP export is off when no collector is +configured. (Drop `--secret` to build without a token; `install-all` may then hit +rate limits.) + +The full test suite is a **separate, non-final stage**, so build it explicitly +with `--target test`. The WebGPU compute test needs a GPU, and `docker build` +can't attach one (no `--gpus` for build), so it runs at `docker run` time. The +`test` stage bundles `mesa-vulkan-drivers`, so passing the host DRI node gives +wgpu a real Intel/AMD GPU (and a software fallback if you pass nothing): + +```bash +docker build --target test -t edge-toolkit-test . +docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD GPU +``` + +NVIDIA via `--gpus all` (with the NVIDIA Container Toolkit) is wired but +**unverified** — its in-container Vulkan ICD doesn't initialize yet, so prefer a +DRI device. The image skips the `o2`/`ws-server` README steps (runtime services). + +### CI + +The [`docker-linux`](.github/workflows/docker-linux.yaml) and +[`docker-windows`](.github/workflows/docker-windows.yaml) workflows rebuild these +images when their respective `Dockerfile` is modified. + +[`Dockerfile.nanoserver`](Dockerfile.nanoserver) starts from **Nano Server** +(the smallest Windows base, ~120 MB) — which has no installer stack, or shell. + +A `gh_token` file (a GitHub token) in the build context is **optional** for a +manual `Dockerfile.nanoserver` build — without it, mise uses GitHub's anonymous +rate limit and may be throttled. The classic Windows builder has no BuildKit +secrets, so it **bakes that file into an image layer**: never +publish an image built with a `gh_token`. (The Linux build passes the token as a +BuildKit secret instead, so it never lands in a layer.) + ## Run ws agent in browser ### Build modules and run the WS server diff --git a/config/ast-grep/rules/gha-default-shell-bash.yaml b/config/ast-grep/rules/gha-default-shell-bash.yaml new file mode 100644 index 0000000..10933b7 --- /dev/null +++ b/config/ast-grep/rules/gha-default-shell-bash.yaml @@ -0,0 +1,28 @@ +id: gha-default-shell-bash +language: yaml +severity: error +message: | + Every GitHub Actions workflow must declare the default run shell as bash, + exactly: + defaults: + run: + shell: bash + so steps run in bash on every runner (Windows included) without per-step + `shell:`. (Pairs with gha-no-step-shell, which forbids per-step `shell:` -- so + this top-level default is the only place the shell is set.) +files: + - .github/workflows/*.yaml +# Fire on any workflow whose document does NOT contain the exact +# defaults -> run -> shell: bash nesting. The `has`/`pattern` matches that +# specific three-level mapping (a stray `shell: bash` on a step does NOT +# satisfy it -- it isn't under defaults.run), and `not` turns "present" into +# "required". +rule: + kind: stream + not: + has: + stopBy: end + pattern: | + defaults: + run: + shell: bash diff --git a/config/ast-grep/rules/gha-no-step-shell.yaml b/config/ast-grep/rules/gha-no-step-shell.yaml new file mode 100644 index 0000000..8bf8a2d --- /dev/null +++ b/config/ast-grep/rules/gha-no-step-shell.yaml @@ -0,0 +1,15 @@ +id: gha-no-step-shell +language: yaml +severity: error +message: | + Don't set `shell:` on a workflow step. Every workflow defaults to bash + (defaults.run.shell) and steps must use it -- write the step in bash (e.g. + `sc`/`net`/`printf` instead of PowerShell cmdlets, even on Windows runners, + which have Git Bash) rather than overriding the shell per step. +files: + - .github/workflows/*.yaml +rule: + pattern: "shell: $VALUE" + inside: + kind: block_sequence + stopBy: end diff --git a/config/ast-grep/rules/no-allow-attributes.yml b/config/ast-grep/rules/no-allow-attributes.yaml similarity index 100% rename from config/ast-grep/rules/no-allow-attributes.yml rename to config/ast-grep/rules/no-allow-attributes.yaml diff --git a/config/ast-grep/rules/no-cargo-manifest-dir.yaml b/config/ast-grep/rules/no-cargo-manifest-dir.yaml new file mode 100644 index 0000000..b177792 --- /dev/null +++ b/config/ast-grep/rules/no-cargo-manifest-dir.yaml @@ -0,0 +1,16 @@ +id: no-cargo-manifest-dir +language: Rust +severity: error +message: | + `CARGO_MANIFEST_DIR` is forbidden outside the project-root helper. It points + at the crate directory, not the repository root, so reaching repo files + through it needs brittle `..` hops. Locate the repository root with + `et_path::find_project_root_from_manifest()` in a `build.rs` (compile time) or + `edge_toolkit::config::get_project_root()` (runtime) instead. +rule: + any: + - pattern: env!("CARGO_MANIFEST_DIR") + - pattern: std::env::var("CARGO_MANIFEST_DIR") + - pattern: env::var("CARGO_MANIFEST_DIR") +ignores: + - libs/path/src/lib.rs diff --git a/config/ast-grep/rules/no-consecutive-expect.yml b/config/ast-grep/rules/no-consecutive-expect.yaml similarity index 100% rename from config/ast-grep/rules/no-consecutive-expect.yml rename to config/ast-grep/rules/no-consecutive-expect.yaml diff --git a/config/ast-grep/rules/no-current-dir.yaml b/config/ast-grep/rules/no-current-dir.yaml new file mode 100644 index 0000000..79acf78 --- /dev/null +++ b/config/ast-grep/rules/no-current-dir.yaml @@ -0,0 +1,16 @@ +id: no-current-dir +language: Rust +severity: error +message: | + `std::env::current_dir()` is forbidden outside the project-root helper. The + current directory is not a reliable anchor for repository files -- it depends + on where the binary happens to be invoked. Use + `edge_toolkit::config::get_project_root()` at runtime, or + `et_path::find_project_root()` from a build script at compile time; both + locate the repository root via the `.dprint.jsonc` marker. +rule: + any: + - pattern: std::env::current_dir() + - pattern: env::current_dir() +ignores: + - libs/edge-toolkit/src/config.rs diff --git a/config/ast-grep/rules/no-doctest.yml b/config/ast-grep/rules/no-doctest.yaml similarity index 100% rename from config/ast-grep/rules/no-doctest.yml rename to config/ast-grep/rules/no-doctest.yaml diff --git a/config/ast-grep/rules/no-extern-crate-self.yml b/config/ast-grep/rules/no-extern-crate-self.yaml similarity index 100% rename from config/ast-grep/rules/no-extern-crate-self.yml rename to config/ast-grep/rules/no-extern-crate-self.yaml diff --git a/config/ast-grep/rules/no-inline-mod.yml b/config/ast-grep/rules/no-inline-mod.yaml similarity index 100% rename from config/ast-grep/rules/no-inline-mod.yml rename to config/ast-grep/rules/no-inline-mod.yaml diff --git a/config/ast-grep/rules/no-map-err.yml b/config/ast-grep/rules/no-map-err.yaml similarity index 100% rename from config/ast-grep/rules/no-map-err.yml rename to config/ast-grep/rules/no-map-err.yaml diff --git a/config/ast-grep/rules/no-mixed-doc-line-comments.yml b/config/ast-grep/rules/no-mixed-doc-line-comments.yaml similarity index 100% rename from config/ast-grep/rules/no-mixed-doc-line-comments.yml rename to config/ast-grep/rules/no-mixed-doc-line-comments.yaml diff --git a/config/ast-grep/rules/no-non-ascii.yml b/config/ast-grep/rules/no-non-ascii.yaml similarity index 100% rename from config/ast-grep/rules/no-non-ascii.yml rename to config/ast-grep/rules/no-non-ascii.yaml diff --git a/config/ast-grep/rules/no-relative-path-literal.yaml b/config/ast-grep/rules/no-relative-path-literal.yaml new file mode 100644 index 0000000..9605465 --- /dev/null +++ b/config/ast-grep/rules/no-relative-path-literal.yaml @@ -0,0 +1,24 @@ +id: no-relative-path-literal +language: Rust +severity: error +message: | + Relative path literals ("." / ".." / "./..." / "../...") are forbidden: they + hardcode a location relative to the process's current directory or this + crate's directory, which is brittle. Anchor paths to the repository root + instead -- `edge_toolkit::config::get_project_root()` (runtime) or + `et_path::find_project_root_from_manifest()` (build scripts) -- and join the + remainder onto it. (Char literals like `'.'` for splitting are unaffected.) +rule: + kind: string_content + regex: '^\.\.?(/.*)?$' +ignores: + # wasmtime's bindgen! has no macro-string support, so its wit `path:` must be + # a literal resolved against this crate's dir (the one unavoidable repo `..`). + - services/ws-wasi-runner/src/bindings.rs + # relative_path_from() deliberately emits "." / ".." for generated mise.toml + # and compose.yaml output. + - libs/path/src/lib.rs + # --module-dir defaults to the current directory. + - utilities/cli/src/cli.rs + # Asserts the relative paths that the generators above produce. + - utilities/cli/tests/scenario_generation.rs diff --git a/config/ast-grep/rules/no-result-alias.yml b/config/ast-grep/rules/no-result-alias.yaml similarity index 100% rename from config/ast-grep/rules/no-result-alias.yml rename to config/ast-grep/rules/no-result-alias.yaml diff --git a/config/ast-grep/rules/no-result-string-err.yml b/config/ast-grep/rules/no-result-string-err.yaml similarity index 100% rename from config/ast-grep/rules/no-result-string-err.yml rename to config/ast-grep/rules/no-result-string-err.yaml diff --git a/config/ast-grep/rules/no-shadow-result.yml b/config/ast-grep/rules/no-shadow-result.yaml similarity index 100% rename from config/ast-grep/rules/no-shadow-result.yml rename to config/ast-grep/rules/no-shadow-result.yaml diff --git a/config/ast-grep/rules/no-std-env-var.yml b/config/ast-grep/rules/no-std-env-var.yaml similarity index 79% rename from config/ast-grep/rules/no-std-env-var.yml rename to config/ast-grep/rules/no-std-env-var.yaml index a4e4cd7..b586dc7 100644 --- a/config/ast-grep/rules/no-std-env-var.yml +++ b/config/ast-grep/rules/no-std-env-var.yaml @@ -17,4 +17,6 @@ constraints: # Allow a string literal ("RUST_LOG") or a const identifier (RUST_LOG). regex: '^"?RUST_' ignores: - - generated/** + # The project-root helper reads CARGO_MANIFEST_DIR (a cargo build variable, + # not app config) to locate the repo root from a build script. + - libs/path/src/lib.rs diff --git a/config/ast-grep/rules/prefer-self-mod-use.yml b/config/ast-grep/rules/prefer-self-mod-use.yaml similarity index 100% rename from config/ast-grep/rules/prefer-self-mod-use.yml rename to config/ast-grep/rules/prefer-self-mod-use.yaml diff --git a/config/ast-grep/rules/use-mod-order.yml b/config/ast-grep/rules/use-mod-order.yaml similarity index 100% rename from config/ast-grep/rules/use-mod-order.yml rename to config/ast-grep/rules/use-mod-order.yaml diff --git a/config/ast-grep/sgconfig.yml b/config/ast-grep/sgconfig.yaml similarity index 100% rename from config/ast-grep/sgconfig.yml rename to config/ast-grep/sgconfig.yaml diff --git a/config/clang-format.yaml b/config/clang-format.yaml new file mode 100644 index 0000000..e09a977 --- /dev/null +++ b/config/clang-format.yaml @@ -0,0 +1,6 @@ +# clang-format style for C sources, applied via `mise run clang-format` (and +# clang-format-check). Passed to clang-format as --style=file:. LLVM base +# with the repo's 120-column limit and 4-space indent (matches the C sources). +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 diff --git a/config/clang-tidy.yaml b/config/clang-tidy.yaml new file mode 100644 index 0000000..aaf5045 --- /dev/null +++ b/config/clang-tidy.yaml @@ -0,0 +1,5 @@ +# clang-tidy checks for C sources, applied via `mise run clang-tidy-check` +# (passed as --config-file=). Bug-finding analysis only; opinionated +# readability/style checks are left off to avoid churn on the small C surface. +Checks: "clang-analyzer-*,bugprone-*,performance-*,portability-*" +WarningsAsErrors: "*" diff --git a/.clippy.toml b/config/clippy.toml similarity index 100% rename from .clippy.toml rename to config/clippy.toml diff --git a/config/conftest/policy/cargo.rego b/config/conftest/policy/cargo.rego new file mode 100644 index 0000000..f56ac86 --- /dev/null +++ b/config/conftest/policy/cargo.rego @@ -0,0 +1,202 @@ +# Cargo.toml policy, evaluated over conftest's `--combine` input (an array of +# {path, contents}). Run with `--namespace cargo` (or `--all-namespaces`). Paths +# come from `git ls-files`, so the workspace root is exactly "Cargo.toml" and any +# other match is a member crate. +package cargo + +is_member(file) if { + endswith(file.path, "Cargo.toml") + file.path != "Cargo.toml" +} + +# Every dependency spec across the dep tables, as [path, name, spec] triples. +dep contains [file.path, name, spec] if { + some file in input + endswith(file.path, "Cargo.toml") + some table in {"dependencies", "dev-dependencies", "build-dependencies"} + some name, spec in file.contents[table] +} + +dep contains [file.path, name, spec] if { + some file in input + endswith(file.path, "Cargo.toml") + some name, spec in file.contents.workspace.dependencies +} + +dep contains [file.path, name, spec] if { + some file in input + endswith(file.path, "Cargo.toml") + some _, tgt in file.contents.target + some table in {"dependencies", "dev-dependencies", "build-dependencies"} + some name, spec in tgt[table] +} + +# Banned crates -> rejection reason. Members must use workspace = true, so the +# root's [workspace.dependencies] is the only place a ban can bite. +banned := { + "anyhow": "define a thiserror enum instead", + "ring": "use aws-lc-rs (transitive via rcgen only; gated in config/deny.toml)", + "ureq": "use reqwest::blocking or reqwest -- one HTTPS stack only", +} + +deny contains msg if { + some [path, name, _] in dep + path == "Cargo.toml" + reason := banned[name] + msg := sprintf("%s: banned dependency %q -- %s", [path, name, reason]) +} + +# Member crates: no path deps, no wildcard versions, no inline git deps. Pins live +# in the root [workspace.dependencies]; members reference them via workspace = true. +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + is_object(spec) + spec.path + msg := sprintf("%s: dependency %q uses a path dep; use workspace = true instead", [path, name]) +} + +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + wildcard(spec) + msg := sprintf("%s: dependency %q uses a wildcard version; pin via [workspace.dependencies]", [path, name]) +} + +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + is_object(spec) + spec.git + msg := sprintf("%s: dependency %q is an inline git dep; pin via [workspace.dependencies]", [path, name]) +} + +wildcard(spec) if { + is_string(spec) + contains(spec, "*") +} + +wildcard(spec) if { + is_object(spec) + contains(spec.version, "*") +} + +# Member [dependencies]/[dev-dependencies] must inherit via workspace = true so +# pins stay in [workspace.dependencies]. (build-dependencies not covered yet.) +deny contains msg if { + some file in input + is_member(file) + some table in {"dependencies", "dev-dependencies"} + some name, spec in file.contents[table] + not spec.workspace == true + msg := sprintf("%s: dependency %q must reference [workspace.dependencies] via workspace = true", [file.path, name]) +} + +# A crate with a [lib] must disable the doctest harness (runnable examples belong +# in tests/ files) and must not rename the lib (keep it the package name). +deny contains msg if { + some file in input + is_member(file) + file.contents.lib + not file.contents.lib.doctest == false + msg := sprintf("%s: [lib] must set doctest = false", [file.path]) +} + +deny contains msg if { + some file in input + is_member(file) + file.contents.lib.name + msg := sprintf("%s: [lib] must not set name (keep it the package name)", [file.path]) +} + +# Every member must inherit the workspace lint tables. generated/rust-rest is +# exempt: progenitor's emitted source trips lints the workspace table denies. +deny contains msg if { + some file in input + is_member(file) + file.path != "generated/rust-rest/Cargo.toml" + not file.contents.lints.workspace == true + msg := sprintf("%s: add [lints] workspace = true", [file.path]) +} + +# Every crate must be a registered workspace member: its directory must appear in +# the root manifest's explicit [workspace].members list (no orphan crates). +workspace_member contains m if { + some file in input + file.path == "Cargo.toml" + some m in file.contents.workspace.members +} + +deny contains msg if { + some file in input + is_member(file) + dir := trim_suffix(file.path, "/Cargo.toml") + not workspace_member[dir] + msg := sprintf("%s: crate is not registered in the root [workspace].members", [file.path]) +} + +# Shared [package] metadata must be inherited from [workspace.package] via +# `.workspace = true`, so the values stay defined in exactly one place. +inherited_package_field := {"edition", "license", "repository"} + +deny contains msg if { + some file in input + is_member(file) + some field in inherited_package_field + not file.contents.package[field].workspace == true + msg := sprintf("%s: [package] %s must inherit via %s.workspace = true", [file.path, field, field]) +} + +# Crate names are namespaced: "edge-toolkit" or "et-" for normal crates, "int-" +# for internal (publish = false) ones. +allowed_crate_name(name, _) if startswith(name, "edge-toolkit") + +allowed_crate_name(name, _) if startswith(name, "et-") + +allowed_crate_name(name, pkg) if { + startswith(name, "int-") + pkg.publish == false +} + +deny contains msg if { + some file in input + is_member(file) + name := file.contents.package.name + not allowed_crate_name(name, file.contents.package) + msg := sprintf("%s: crate name %q must start with edge-toolkit/et- (int- if publish=false)", [file.path, name]) +} + +# An empty `features = []` on a dependency is pointless noise -- drop it. +deny contains msg if { + some [path, name, spec] in dep + is_object(spec) + spec.features == [] + msg := sprintf("%s: dependency %q has an empty features = []; remove it", [path, name]) +} + +# A feature must not share its name with a dependency: it shadows the implicit +# feature an optional dep creates and is confusing. generated/rust-rest is exempt +# -- its generator emits a `tracing` feature beside a (non-optional) `tracing` dep. +is_dep_name(file, name) if { + some table in {"dependencies", "dev-dependencies", "build-dependencies"} + file.contents[table][name] +} + +deny contains msg if { + some file in input + is_member(file) + file.path != "generated/rust-rest/Cargo.toml" + some feat, _ in file.contents.features + is_dep_name(file, feat) + msg := sprintf("%s: feature %q shares its name with a dependency; rename it", [file.path, feat]) +} + +# Dependency overrides ([patch]/[replace]) belong in the root manifest, where +# they apply workspace-wide and stay in one place; a member can't override deps. +deny contains msg if { + some file in input + is_member(file) + some table in {"patch", "replace"} + file.contents[table] + msg := sprintf("%s: [%s] belongs in the root manifest, not a member crate", [file.path, table]) +} diff --git a/config/conftest/policy/gha.rego b/config/conftest/policy/gha.rego new file mode 100644 index 0000000..df06cb4 --- /dev/null +++ b/config/conftest/policy/gha.rego @@ -0,0 +1,21 @@ +# GitHub Actions workflow policy, evaluated per file: conftest reads each .yaml +# independently (no --combine, since there are no cross-file YAML rules). +# Replicates the gha-* ast-grep rules; running both is fine. Selected with +# `--namespace gha`, so it only runs against workflow YAML, never the TOML inputs. +package gha + +# Every workflow must set the default run shell to bash, so steps run in bash on +# every runner (Windows included) without a per-step `shell:`. +deny contains msg if { + not input.defaults.run.shell == "bash" + msg := "workflow must set defaults.run.shell: bash" +} + +# Steps must not override the shell -- rely on the workflow default (write the +# step in bash rather than switching to PowerShell on Windows runners). +deny contains msg if { + some name, job in input.jobs + some step in job.steps + step.shell + msg := sprintf("job %q sets shell: on a step; use the workflow default (bash)", [name]) +} diff --git a/config/conftest/policy/mise.rego b/config/conftest/policy/mise.rego new file mode 100644 index 0000000..0cb09c3 --- /dev/null +++ b/config/conftest/policy/mise.rego @@ -0,0 +1,79 @@ +# .mise/config*.toml policy, evaluated over conftest's `--combine` input (an array +# of {path, contents}). Run with `--namespace mise` (or `--all-namespaces`). +package mise + +is_mise(file) if startswith(file.path, ".mise/config") + +# A task `run` must be a string, not an array: taplo's reorder_arrays would +# re-sort the commands of an array form and scramble the sequence. +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_array(task.run) + msg := sprintf("%s: task %q run must be a string, not an array", [file.path, name]) +} + +# A multiline `run` must use `shell = "bash -euo pipefail -c"` so a failing +# command fails the task instead of being masked. +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_string(task.run) + contains(task.run, "\n") + not task.shell == "bash -euo pipefail -c" + msg := sprintf("%s: task %q has a multiline run; set shell = \"bash -euo pipefail -c\"", [file.path, name]) +} + +# Task descriptions must be single-line (keep them under the 120-char limit). +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_string(task.description) + contains(task.description, "\n") + msg := sprintf("%s: task %q description must be a single line", [file.path, name]) +} + +# `cargo:` tools build from source; prefer a prebuilt backend. Allowlist the two +# that have no prebuilt binary. +allowed_cargo_tool := {"cargo:cargo-expand", "cargo:dart-typegen"} + +deny contains msg if { + some file in input + is_mise(file) + some name, _ in file.contents.tools + startswith(name, "cargo:") + not allowed_cargo_tool[name] + msg := sprintf("%s: tool %q builds from source; use a prebuilt backend", [file.path, name]) +} + +# `ubi:` is deprecated; the `http:` backend replaces it. +deny contains msg if { + some file in input + is_mise(file) + some name, _ in file.contents.tools + startswith(name, "ubi:") + msg := sprintf("%s: tool %q uses the deprecated ubi backend; use http: instead", [file.path, name]) +} + +# Tools should work on every OS (CLAUDE.md "Tools must work on every OS"). Any +# os-scoped [tools] entry must be a genuinely platform-specific one in this list. +allowed_os_scoped_tool := { + "chromedriver", + "pipx", + "npm:pnpm", + "pnpm", + "github:christianhelle/openapi2zig", +} + +deny contains msg if { + some file in input + is_mise(file) + some name, spec in file.contents.tools + is_object(spec) + spec.os + not allowed_os_scoped_tool[name] + msg := sprintf("%s: tool %q is os-scoped; tools must work on every OS (or allowlist it)", [file.path, name]) +} diff --git a/config/conftest/policy/pyproject.rego b/config/conftest/policy/pyproject.rego new file mode 100644 index 0000000..17f6f58 --- /dev/null +++ b/config/conftest/policy/pyproject.rego @@ -0,0 +1,31 @@ +# pyproject.toml policy, evaluated over conftest's --combine input ({path, +# contents}); selected with --namespace pyproject. +package pyproject + +# A `path = ".."` uv source points at the parent directory, which isn't a package +# -- almost always a mistake. Sibling packages are referenced by their specific +# relative path (e.g. ../../../generated/python-rest), not the bare parent. +deny contains msg if { + some file in input + endswith(file.path, "pyproject.toml") + some name, src in file.contents.tool.uv.sources + src.path == ".." + msg := sprintf("%s: [tool.uv.sources] %q uses path = \"..\"; point at the package path", [file.path, name]) +} + +# uv_build must be pinned to exactly the mise uv version, so `uv build` uses the +# matching backend (no out-of-range warning). Bump both together when upgrading uv. +uv_version := v if { + some file in input + endswith(file.path, ".mise/config.toml") + v := file.contents.tools.uv +} + +deny contains msg if { + some file in input + endswith(file.path, "pyproject.toml") + some req in file.contents["build-system"].requires + startswith(req, "uv_build") + req != sprintf("uv_build==%s", [uv_version]) + msg := sprintf("%s: pin uv_build==%s to match mise's uv (found %q)", [file.path, uv_version, req]) +} diff --git a/config/conftest/policy/wasm-bindgen-sync.rego b/config/conftest/policy/wasm-bindgen-sync.rego new file mode 100644 index 0000000..9962ae8 --- /dev/null +++ b/config/conftest/policy/wasm-bindgen-sync.rego @@ -0,0 +1,29 @@ +# Cross-file invariants, evaluated over conftest's `--combine` input (an array of +# {path, contents}). Run with `--namespace cross`. +package cross + +# The mise `github:wasm-bindgen` pin must equal the wasm-bindgen package version +# in Cargo.lock. wasm-pack requires the wasm-bindgen CLI to match the crate +# version exactly; when they match it uses the on-PATH (mise) binary, otherwise it +# downloads its own. Keeping them equal avoids that download. +mise_pin := pin if { + some file in input + endswith(file.path, ".mise/config.toml") + pin := file.contents.tools["github:wasm-bindgen/wasm-bindgen"] +} + +lock_version := ver if { + some file in input + endswith(file.path, "Cargo.lock") + some pkg in file.contents.package + pkg.name == "wasm-bindgen" + ver := pkg.version +} + +deny contains msg if { + mise_pin != lock_version + msg := sprintf( + "wasm-bindgen: mise pin %q != Cargo.lock %q; bump the pin in .mise/config.toml", + [mise_pin, lock_version], + ) +} diff --git a/deny.toml b/config/deny.toml similarity index 96% rename from deny.toml rename to config/deny.toml index 3f3628c..1f31006 100644 --- a/deny.toml +++ b/config/deny.toml @@ -1,6 +1,5 @@ -# `cargo deny check` config. Run locally via `cargo deny check`, in CI via -# `.github/workflows/cargo-deny.yml`. Every option below is intentional; -# cargo-deny's `init` template (with all defaults explicit) is not used. +# cargo-deny config, applied via `mise run cargo-deny-check` -- locally and in +# .github/workflows/dependencies.yaml. [advisories] version = 2 diff --git a/config/dprint.jsonc b/config/dprint.jsonc new file mode 100644 index 0000000..046ec61 --- /dev/null +++ b/config/dprint.jsonc @@ -0,0 +1,41 @@ +{ + "dockerfile": { + "associations": ["**/Dockerfile", "**/*.dockerfile", "**/Dockerfile.nanoserver"], + }, + "java": { + }, + "json": { + }, + // Match the repo-wide 120 line-length set in .editorconfig and ruff.toml, + // otherwise dprint's bundled ruff would reformat Python files to its + // default and fight with `mise run ruff-fmt`. + "ruff": { + "lineLength": 120, + }, + "malva": { + }, + "markdown": { + }, + "markup": { + }, + "typescript": { + }, + "yaml": { + }, + "excludes": [ + "**/node_modules", + "**/*-lock.json", + "data/", + ], + "plugins": [ + "https://github.com/speakeasy-api/dprint-plugin-java/releases/latest/download/dprint_plugin_java.wasm", + "https://plugins.dprint.dev/g-plane/malva-v0.15.2.wasm", + "https://plugins.dprint.dev/g-plane/markup_fmt-v0.27.0.wasm", + "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", + "https://plugins.dprint.dev/dockerfile-0.4.0.wasm", + "https://plugins.dprint.dev/json-0.21.3.wasm", + "https://plugins.dprint.dev/markdown-0.21.1.wasm", + "https://plugins.dprint.dev/ruff-0.7.10.wasm", + "https://plugins.dprint.dev/typescript-0.95.15.wasm", + ], +} diff --git a/config/hadolint.yaml b/config/hadolint.yaml new file mode 100644 index 0000000..6c53554 --- /dev/null +++ b/config/hadolint.yaml @@ -0,0 +1,16 @@ +# hadolint config. Each entry below is a best-practice rule we consciously skip, +# with the reason; drop one once it no longer applies. +ignored: + # Pinning apt package versions is brittle: they float with the Ubuntu base + # image + security updates, so an exact pin breaks on every base bump. We + # install distro defaults deliberately -- the base FROM line is the version pin. + - DL3008 + # Same for npm: `npm install -g pnpm` tracks latest; we don't pin it here. + - DL3016 + # `curl ... | sh` (the mise installer): if curl fails, sh gets empty input and + # the next `mise` command then fails anyway, so the pipe failure is caught + # downstream -- no need to set SHELL -o pipefail just for that one line. + - DL4006 + # Consecutive RUNs are kept separate on purpose -- for layer caching and so a + # change to one step doesn't bust the others. + - DL3059 diff --git a/config/ls-lint.yaml b/config/ls-lint.yaml new file mode 100644 index 0000000..bf4187e --- /dev/null +++ b/config/ls-lint.yaml @@ -0,0 +1,43 @@ +# ls-lint file/directory naming linter (https://ls-lint.org), run via +# `mise run ls-lint-check`. ls-lint walks the working tree (not git), so the +# build/vendor/codegen trees are ignored below and only hand-written source is +# linted. Each rule lists the casings actually used in the repo today. +ls: + # Directories are kebab-case (most) or snake_case (a few generated-style + # names); the regex alternative permits leading-dot tool dirs (.github, .mise). + .dir: kebab-case | snake_case | regex:\.[a-z]+ + + .rs: snake_case | kebab-case + .py: snake_case | regex:__[a-z]+__ + .js: snake_case + .zig: snake_case + .dart: snake_case + .wit: snake_case | kebab-case + .rego: snake_case | kebab-case + .yaml: snake_case | kebab-case + .cs: PascalCase + .java: PascalCase + +ignore: + - .git + - target + - node_modules + - generated + # Bundled data/model assets -- names are not hand-chosen. + - data + # Build/vendor output that ls-lint would otherwise walk (it can't read + # .gitignore). The `**/` form matches each at any depth. + - "**/.venv" + - "**/__pycache__" + - "**/.pytest_cache" + - "**/.ruff_cache" + - "**/.zig-cache" + - "**/zig-out" + - "**/.dart_tool" + - "**/obj" + - "**/bin/Debug" + - "**/bin/Release" + # Built module artifacts (wheels, .wasm, generated JS) live under each pkg/. + - "**/pkg" + # Zig codegen templates dir whose name carries a literal `.in` suffix. + - "**/zig.in" diff --git a/config/lychee.toml b/config/lychee.toml new file mode 100644 index 0000000..e3bd895 --- /dev/null +++ b/config/lychee.toml @@ -0,0 +1,31 @@ +# Config for the `link-check` task, part of the `check` aggregate. lychee makes +# live network requests, so this is tuned to tolerate transient failures +# (retries, 429-tolerance, disk cache) and to skip reserved hosts + non-prose +# trees. + +# Retry transient failures; cap each request so a hung host can't stall the run. +max_retries = 3 +retry_wait_time = 2 +timeout = 20 + +# Cache results on disk (.lycheecache) so reruns skip recently-checked URLs. +cache = true +max_cache_age = "2d" + +# Treat GitHub's anonymous throttle (429) as reachable -- the link is valid, +# we're just rate-limited. +accept = ["200..=299", "429"] + +# Skip loopback (localhost, 127.0.0.1) and private/reserved IPs (e.g. the +# 10.0.0.1:9000 test fixture) -- not real external links. +exclude_loopback = true +exclude_private = true + +exclude_path = ["data", "generated", "target"] + +# URL patterns (regex) to skip: doc/test placeholder hosts (e.g. http://host:8080/) +# and `{...}` format-string templates (.../{repo}/{git_ref}/...), not real links. +exclude = [ + '^https?://host[:/]', + '\{|%7B', +] diff --git a/osv-scanner.toml b/config/osv-scanner.toml similarity index 54% rename from osv-scanner.toml rename to config/osv-scanner.toml index 4095200..25a1e54 100644 --- a/osv-scanner.toml +++ b/config/osv-scanner.toml @@ -1,8 +1,4 @@ -# Ignore list mirrored from `deny.toml`'s `[advisories].ignore`. See -# `deny.toml` for the per-ID rationale (which crate pulls the -# vulnerable version, why the bug isn't reachable in this runner, -# and what upstream change drops the ignore). Keep both files in -# sync -- when an ID is removed from `deny.toml`, remove it here too. +# AUTO-GENERATED from config/deny.toml by `mise run gen:osv-scanner`. IgnoredVulns = [ { id = "RUSTSEC-2023-0071" }, { id = "RUSTSEC-2024-0436" }, diff --git a/config/ryl.yaml b/config/ryl.yaml new file mode 100644 index 0000000..90048fa --- /dev/null +++ b/config/ryl.yaml @@ -0,0 +1,14 @@ +# Starts from yamllint's standard ruleset, then aligns the cosmetic rules with the +# repo's other tools so they never disagree. +extends: default + +rules: + # dprint (the YAML formatter) writes one space before an inline `#`; yamllint + # defaults to two. Defer to dprint so the two can't fight. + comments: + min-spaces-from-content: 1 + # Match the repo's 120-char editorconfig limit (yamllint defaults to 80). + line-length: + max: 120 + # The repo's workflows and configs don't use a `---` document-start marker. + document-start: disable diff --git a/config/semgrep/cargo-toml.yml b/config/semgrep/cargo-toml.yaml similarity index 100% rename from config/semgrep/cargo-toml.yml rename to config/semgrep/cargo-toml.yaml diff --git a/config/semgrep/mise-config.yaml b/config/semgrep/mise-config.yaml new file mode 100644 index 0000000..e8c84fe --- /dev/null +++ b/config/semgrep/mise-config.yaml @@ -0,0 +1,26 @@ +rules: + - id: single-line-description-in-mise-config + languages: [generic] + paths: + include: + - "/.mise/config*.toml" + pattern-regex: 'description = """' + message: >- + mise task `description` fields must be single-line `"..."` strings, not + multi-line `"""..."""`. Keep the description short enough to fit on one + line (under the 120-char limit). + severity: ERROR + + - id: multiline-task-run-needs-bash-shell + languages: [generic] + paths: + include: + - "/.mise/config*.toml" + pattern-regex: 'run = ("""|'''''')\n(?:(?!\1\n).*\n)*?\1\n(?!shell = "bash -euo pipefail -c")' + message: >- + A mise task with a multiline `run` must be immediately followed by + `shell = "bash -euo pipefail -c"` (on the line right after the closing + `"""`/`'''`), so a failing command fails the task instead of being masked. + Put `shell` directly after `run`. bash is installed before any task on + Windows (see preinstall), so even the setup tasks can use it. + severity: ERROR diff --git a/config/semgrep/mise-config.yml b/config/semgrep/mise-config.yml deleted file mode 100644 index 3bfd21f..0000000 --- a/config/semgrep/mise-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -rules: - - id: single-line-description-in-mise-config - languages: [generic] - paths: - include: - - "/.mise/config*.toml" - pattern-regex: 'description = """' - message: >- - mise task `description` fields must be single-line `"..."` strings, not - multi-line `"""..."""`. Keep the description short enough to fit on one - line (under the 120-char limit). - severity: ERROR diff --git a/config/semgrep/no-trailing-backslash.yml b/config/semgrep/no-trailing-backslash.yaml similarity index 95% rename from config/semgrep/no-trailing-backslash.yml rename to config/semgrep/no-trailing-backslash.yaml index d30a60c..b035e9d 100644 --- a/config/semgrep/no-trailing-backslash.yml +++ b/config/semgrep/no-trailing-backslash.yaml @@ -9,7 +9,7 @@ rules: - "/.mise/config.python.toml" - "/README.md" - "/utilities/cli/README.md" - - "/services/ws-server/Dockerfile" + - "**/Dockerfile*" # Generated deployment artifacts (regen-verification output). - "/verification/**/mise.toml" - "/verification/**/compose.yaml" diff --git a/config/semgrep/prefer-yaml-extension.yaml b/config/semgrep/prefer-yaml-extension.yaml new file mode 100644 index 0000000..752d6af --- /dev/null +++ b/config/semgrep/prefer-yaml-extension.yaml @@ -0,0 +1,13 @@ +rules: + - id: prefer-yaml-extension + languages: [generic] + paths: + include: + - "*.yml" + exclude: + # External upstream clones + generated trees are not our prose. + - "/data" + - "/generated" + pattern-regex: \A + message: "Use the .yaml extension, not .yml -- rename the file." + severity: ERROR diff --git a/config/semgrep/prefer-yaml-toml.yaml b/config/semgrep/prefer-yaml-toml.yaml new file mode 100644 index 0000000..a1b8407 --- /dev/null +++ b/config/semgrep/prefer-yaml-toml.yaml @@ -0,0 +1,21 @@ +rules: + - id: prefer-yaml-toml-over-json + languages: [generic] + paths: + include: + - "*.json" + - "*.jsonc" + - "*.jsonl" + exclude: + # Formats whose tooling mandates JSON/JSONC -- everything else should be + # YAML or TOML. Add a file here only if its tool can't read YAML/TOML. + - "*.schema.json" # JSON Schema documents + - "package.json" # npm manifests + - "/.vscode/*.json" # editor config + - "dprint.jsonc" # dprint config + - ".dprint.jsonc" + pattern-regex: \A + message: >- + Prefer YAML or TOML over JSON/JSONC/JSONL for config. If a tool requires + this format, add the file to this rule's paths.exclude allowlist. + severity: ERROR diff --git a/.taplo.toml b/config/taplo.toml similarity index 82% rename from .taplo.toml rename to config/taplo.toml index 4247c12..4ba02e3 100644 --- a/.taplo.toml +++ b/config/taplo.toml @@ -1,8 +1,12 @@ -# Build output is off-limits: it's gitignored and (per CLAUDE.md) holds agent -# scratch files under target/scratch/. taplo's baseline `taplo lint` / `format` -# don't honour .gitignore, and they auto-associate any stray `Cargo.toml` with -# the SchemaStore Cargo schema — so a scratch fixture there breaks the check. -exclude = ["target/**"] +# Build output and external checkouts are off-limits. Both are gitignored, but +# taplo's baseline `taplo lint` / `format` don't honour .gitignore: `target/` +# holds scratch + build artifacts (and auto-associates any stray `Cargo.toml` +# with the SchemaStore Cargo schema), and `data/` holds external upstream repo +# clones whose TOML files we must never reformat. target/ is also CLAUDE.md's +# scratch area, so scratch TOML is skipped here too -- to lint or test a scratch +# TOML, pipe it through `taplo format -` / `taplo lint -` (stdin has no path to +# exclude). +exclude = ["target/**", "data/**"] [formatting] column_width = 120 @@ -49,11 +53,7 @@ schema = { path = "config/taplo/require-workspace-deps.schema.json" } # Banned dep names across every dep table — `[dependencies]`, # `[dev-dependencies]`, `[build-dependencies]`, `[workspace.dependencies]`, -# and the `[target.*]` variants. Currently `anyhow` (use a `thiserror` -# enum) and `ureq` (use `reqwest::blocking` -- keeps us on one HTTPS -# stack). Replaces the previous per-crate `no-anyhow` ast-grep rule -# which caught uses but not declarations. Add new bans by extending the -# `depTable.properties` list in the schema. +# and the `[target.*]` variants. [[rule]] include = ["**/Cargo.toml"] schema = { path = "config/taplo/no-banned-deps.schema.json" } diff --git a/config/taplo/mise-cargo-backend-allowlist.schema.json b/config/taplo/mise-cargo-backend-allowlist.schema.json new file mode 100644 index 0000000..18340f8 --- /dev/null +++ b/config/taplo/mise-cargo-backend-allowlist.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise [tools] -- cargo backend allowlist", + "description": "`cargo:` builds from source (slow installs); allow only the two with no prebuilt binary.", + "type": "object", + "properties": { + "tools": { + "type": "object", + "propertyNames": { + "anyOf": [ + { "not": { "pattern": "^cargo:" } }, + { "enum": ["cargo:cargo-expand", "cargo:dart-typegen"] } + ] + } + } + } +} diff --git a/config/taplo/mise-no-ubi-backend.schema.json b/config/taplo/mise-no-ubi-backend.schema.json new file mode 100644 index 0000000..6b0fe8f --- /dev/null +++ b/config/taplo/mise-no-ubi-backend.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise [tools] -- no ubi backend", + "description": "The `ubi:` backend is deprecated; use `http:` instead.", + "type": "object", + "properties": { + "tools": { + "type": "object", + "propertyNames": { "not": { "pattern": "^ubi:" } } + } + } +} diff --git a/config/taplo/mise-run-not-array.schema.json b/config/taplo/mise-run-not-array.schema.json new file mode 100644 index 0000000..2bf550d --- /dev/null +++ b/config/taplo/mise-run-not-array.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise tasks — `run` must be a string, not an array", + "description": "`run` must be a string, not an array: taplo reorder_arrays would re-sort the commands.", + "type": "object", + "properties": { + "tasks": { + "type": "object", + "additionalProperties": { + "properties": { + "run": { + "type": "string", + "description": "use a multiline string (one command per line); an array gets re-sorted by taplo" + } + } + } + } + } +} diff --git a/config/taplo/no-banned-deps.schema.json b/config/taplo/no-banned-deps.schema.json index a74bfd9..8d06166 100644 --- a/config/taplo/no-banned-deps.schema.json +++ b/config/taplo/no-banned-deps.schema.json @@ -10,7 +10,7 @@ "const": "forbidden; define a thiserror enum instead" }, "ring": { - "const": "forbidden; use aws-lc-rs. Transitive via rcgen only (gated in deny.toml's bans.deny wrappers)." + "const": "forbidden; use aws-lc-rs. Transitive via rcgen only (gated in config/deny.toml's bans.deny)." }, "ureq": { "const": "forbidden; use reqwest::blocking (sync) or reqwest (async). One HTTPS stack only." diff --git a/config/taplo/no-lib-name.schema.json b/config/taplo/no-lib-name.schema.json new file mode 100644 index 0000000..65eebbc --- /dev/null +++ b/config/taplo/no-lib-name.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Workspace member Cargo.toml — [lib] must not override the crate name", + "description": "The library name must default from the package name; an explicit `[lib] name` rename is banned.", + "type": "object", + "properties": { + "lib": { + "type": "object", + "properties": { + "name": false + } + } + } +} diff --git a/config/typos.toml b/config/typos.toml new file mode 100644 index 0000000..c436852 --- /dev/null +++ b/config/typos.toml @@ -0,0 +1,4 @@ +# data/ holds vendored demo + model assets (not our own prose), so its spelling +# is out of scope for the typos check. +[files] +extend-exclude = ["data/"] diff --git a/config/zizmor.yaml b/config/zizmor.yaml new file mode 100644 index 0000000..8d9fb3d --- /dev/null +++ b/config/zizmor.yaml @@ -0,0 +1,16 @@ +# zizmor GitHub Actions audit config, applied via `mise run zizmor-check`. +# +# We pin actions by tag (`@v4`), not by commit hash: tags are readable and +# Dependabot keeps them current. So the `unpinned-uses` audit's default +# hash-pin policy is turned off here -- `any` accepts a tag/branch ref. +rules: + unpinned-uses: + config: + policies: + "*": any + obfuscation: + # test.yaml's matrix `include` is built with `fromJSON(... format(...))` + # precisely because the OS list depends on `github.event_name` / the + # workflow_dispatch input -- it cannot "be reduced to a constant". + ignore: + - test.yaml diff --git a/generated/python-rest/pyproject.toml b/generated/python-rest/pyproject.toml index 9699f50..07d7199 100644 --- a/generated/python-rest/pyproject.toml +++ b/generated/python-rest/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "et_rest_client" diff --git a/generated/python-ws/pyproject.toml b/generated/python-ws/pyproject.toml index fe8b7d9..1837cb2 100644 --- a/generated/python-ws/pyproject.toml +++ b/generated/python-ws/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "et_ws" diff --git a/libs/edge-toolkit/Cargo.toml b/libs/edge-toolkit/Cargo.toml index ac952ce..4bc2cc0 100644 --- a/libs/edge-toolkit/Cargo.toml +++ b/libs/edge-toolkit/Cargo.toml @@ -12,8 +12,8 @@ doctest = false [dependencies] asyncapi-rust = { workspace = true, optional = true } base64.workspace = true +et-path.workspace = true fs-err.workspace = true -lets_find_up.workspace = true log.workspace = true schemars = { workspace = true, optional = true } secrecy.workspace = true diff --git a/libs/edge-toolkit/src/config.rs b/libs/edge-toolkit/src/config.rs index 295b88c..abcafd4 100644 --- a/libs/edge-toolkit/src/config.rs +++ b/libs/edge-toolkit/src/config.rs @@ -12,20 +12,12 @@ use crate::ports::Services; pub const LOCALHOST: &str = "127.0.0.1"; /// Helper to find repository root. +/// +/// This is the one sanctioned `current_dir()`. #[expect(clippy::missing_panics_doc, clippy::unwrap_used)] #[must_use] pub fn get_project_root() -> PathBuf { - match lets_find_up::find_up(".taplo.toml") { - Ok(Some(mut path)) => { - assert!(path.pop(), "Failed to drop the filename"); - path - } - Ok(None) => std::env::current_dir().unwrap(), - Err(err) => { - log::error!("{err}"); - std::env::current_dir().unwrap() - } - } + et_path::find_project_root(&std::env::current_dir().unwrap()) } /// Returns the default module search paths for ws-server. diff --git a/libs/otlp-mock/Cargo.toml b/libs/otlp-mock/Cargo.toml index 3a617f8..8007e21 100644 --- a/libs/otlp-mock/Cargo.toml +++ b/libs/otlp-mock/Cargo.toml @@ -1,5 +1,6 @@ [package] -name = "otlp-mock" +name = "int-otlp-mock" +publish = false description = "In-process mock OTLP/HTTP-JSON collector for trace-propagation tests" version = "0.1.0" edition.workspace = true diff --git a/libs/path/Cargo.toml b/libs/path/Cargo.toml new file mode 100644 index 0000000..e44633e --- /dev/null +++ b/libs/path/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "et-path" +description = "Path utilities" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/libs/path/src/lib.rs b/libs/path/src/lib.rs new file mode 100644 index 0000000..2ea31e0 --- /dev/null +++ b/libs/path/src/lib.rs @@ -0,0 +1,104 @@ +//! Path utilities shared across the workspace. +//! +//! Root-finding has two entry points: [`find_project_root`] (from an explicit +//! start, used by the runtime `edge_toolkit::config::get_project_root`) and +//! [`find_project_root_from_manifest`] (from `CARGO_MANIFEST_DIR`, used by build +//! scripts). The path builders [`absolute_from`] and [`relative_path_from`] +//! render POSIX-style paths for `mise.toml` / `docker-compose.yaml` generation. + +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; + +/// Marker file that identifies the repository root. `.dprint.jsonc` is present +/// at the root (and nowhere above it), so it is a reliable anchor. +const ROOT_MARKER: &str = ".dprint.jsonc"; + +/// Walk up from `start` to the first ancestor that contains `.dprint.jsonc`. +/// +/// Falls back to `start` itself when no ancestor has the marker, mirroring the +/// runtime helper's "use what we have" behaviour. +#[must_use] +pub fn find_project_root(start: &Path) -> PathBuf { + start + .ancestors() + .find(|dir| dir.join(ROOT_MARKER).is_file()) + .map_or_else(|| start.to_path_buf(), Path::to_path_buf) +} + +/// Locate the repository root from a build script. +/// +/// Reads `CARGO_MANIFEST_DIR` (which cargo sets for build scripts), so this is +/// the single sanctioned use of that variable (see the `no-cargo-manifest-dir` +/// ast-grep rule). Outside a build script the variable is unset and this +/// returns a meaningless path; use [`find_project_root`] with an explicit +/// start, or `edge_toolkit::config::get_project_root`, instead. +#[must_use] +pub fn find_project_root_from_manifest() -> PathBuf { + let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_default(); + find_project_root(Path::new(&manifest)) +} + +/// Resolve `path` against `base` when relative, then lexically normalize it. +#[must_use] +pub fn absolute_from(base: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + normalize_path(path) + } else { + normalize_path(&base.join(path)) + } +} + +/// Build a relative path from `from_dir` to `target`, always joined with `/`. +/// +/// The result is rendered as a POSIX string regardless of host OS, because +/// every caller writes it into generated `mise.toml` / `docker-compose.yaml` +/// output -- both of which expect forward-slash separators even on Windows. +#[must_use] +pub fn relative_path_from(from_dir: &Path, target: &Path) -> String { + let from_components = normal_components(&normalize_path(from_dir)); + let target_components = normal_components(&normalize_path(target)); + let common_len = from_components + .iter() + .zip(target_components.iter()) + .take_while(|(from, target)| from == target) + .count(); + + let mut parts: Vec = Vec::new(); + for _ in common_len..from_components.len() { + parts.push("..".to_string()); + } + for component in target_components.iter().skip(common_len) { + parts.push(component.to_string_lossy().into_owned()); + } + + if parts.is_empty() { + ".".to_string() + } else { + parts.join("/") + } +} + +fn normal_components(path: &Path) -> Vec { + path.components() + .filter_map(|component| match component { + Component::Normal(value) => Some(value.to_os_string()), + Component::Prefix(_) | Component::RootDir | Component::CurDir | Component::ParentDir => None, + }) + .collect() +} + +fn normalize_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(Path::new("/")), + Component::CurDir => {} + Component::ParentDir => { + let _popped = normalized.pop(); + } + Component::Normal(value) => normalized.push(value), + } + } + normalized +} diff --git a/libs/path/tests/find.rs b/libs/path/tests/find.rs new file mode 100644 index 0000000..02d3232 --- /dev/null +++ b/libs/path/tests/find.rs @@ -0,0 +1,32 @@ +#![cfg(test)] +#![expect( + clippy::unwrap_used, + reason = "test code: failed tempdir/fs setup should fail the test" +)] + +use std::fs; + +use et_path::find_project_root; +use tempfile::tempdir; + +#[test] +fn finds_marker_in_an_ancestor() { + let root = tempdir().unwrap(); + fs::write(root.path().join(".dprint.jsonc"), "{}").unwrap(); + let nested = root.path().join("a/b/c"); + fs::create_dir_all(&nested).unwrap(); + + // Canonicalize both sides so the macOS /var -> /private/var symlink on the + // tempdir doesn't fail an otherwise-correct match. + let found = fs::canonicalize(find_project_root(&nested)).unwrap(); + assert_eq!(found, fs::canonicalize(root.path()).unwrap()); +} + +#[test] +fn falls_back_to_start_when_marker_is_absent() { + let dir = tempdir().unwrap(); + let nested = dir.path().join("x"); + fs::create_dir_all(&nested).unwrap(); + + assert_eq!(find_project_root(&nested), nested); +} diff --git a/libs/ws-runner-common/Cargo.toml b/libs/ws-runner-common/Cargo.toml index c2b80e5..adb2e0b 100644 --- a/libs/ws-runner-common/Cargo.toml +++ b/libs/ws-runner-common/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [lib] doctest = false -name = "et_ws_runner_common" path = "src/lib.rs" [dependencies] diff --git a/pom.xml b/pom.xml index e7e46f3..85c603c 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,41 @@ ${project.basedir}/services/ws-modules/java-data1/src/main/java + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + true + + -Xlint:all + -Werror + -XDcompilePolicy=simple + --should-stop=ifError=FLOW + -Xplugin:ErrorProne + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + + + + com.google.errorprone + error_prone_core + 2.49.0 + + + + org.teavm teavm-maven-plugin diff --git a/ruff.toml b/ruff.toml index 8311165..32defa1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,11 @@ +# Kept at the repo root (unlike config/deny.toml, config/osv-scanner.toml and +# config/taplo.toml) on purpose: editors/IDEs (the VS Code Ruff extension, +# PyCharm) and pre-commit auto-discover ruff config only as a root ruff.toml / +# .ruff.toml / pyproject.toml. ruff has no config-path env var, so moving this +# under config/ would silently drop every editor back to ruff's defaults +# (line-length 88, no isort). The mise tasks could pass --config, but the live +# editor experience can't, so it stays here. +# # Match the repo-wide 120 line-length set in .editorconfig. Without this, ruff # would use its default of 88, so files that ruff considered "already # formatted" could still trip the editorconfig-check. diff --git a/services/ws-modules/pydata1/pyproject.toml b/services/ws-modules/pydata1/pyproject.toml index 90063bd..b1ee14b 100644 --- a/services/ws-modules/pydata1/pyproject.toml +++ b/services/ws-modules/pydata1/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "pydata1" diff --git a/services/ws-modules/pyface1/pyproject.toml b/services/ws-modules/pyface1/pyproject.toml index 91a47e0..e858cda 100644 --- a/services/ws-modules/pyface1/pyproject.toml +++ b/services/ws-modules/pyface1/pyproject.toml @@ -11,7 +11,7 @@ dev = ["pytest"] [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "pyface1" diff --git a/services/ws-modules/wasi-comm1/Cargo.toml b/services/ws-modules/wasi-comm1/Cargo.toml index 8616d30..e195743 100644 --- a/services/ws-modules/wasi-comm1/Cargo.toml +++ b/services/ws-modules/wasi-comm1/Cargo.toml @@ -17,5 +17,9 @@ test = false serde_json.workspace = true wit-bindgen.workspace = true +# Build script (runs on the host) locates the repo root to emit ET_WIT_DIR. +[build-dependencies] +et-path.workspace = true + [lints] workspace = true diff --git a/services/ws-modules/wasi-comm1/build.rs b/services/ws-modules/wasi-comm1/build.rs new file mode 100644 index 0000000..17f2602 --- /dev/null +++ b/services/ws-modules/wasi-comm1/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_WIT_DIR` (absolute path to the shared WIT directory) so the +//! `wit_bindgen::generate!` invocation in `src/lib.rs` locates it via `env!`, +//! instead of a `..`-relative path that hardcodes this crate's depth below the +//! repository root. + +fn main() { + let wit_dir = et_path::find_project_root_from_manifest().join("generated/specs/wit"); + println!("cargo:rustc-env=ET_WIT_DIR={}", wit_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/services/ws-modules/wasi-comm1/src/lib.rs b/services/ws-modules/wasi-comm1/src/lib.rs index 112ad89..cf7337a 100644 --- a/services/ws-modules/wasi-comm1/src/lib.rs +++ b/services/ws-modules/wasi-comm1/src/lib.rs @@ -28,7 +28,8 @@ #![expect(unsafe_code)] wit_bindgen::generate!({ - path: "../../../generated/specs/wit", + // ET_WIT_DIR is the absolute path to generated/specs/wit, emitted by build.rs. + path: env!("ET_WIT_DIR"), world: "module", generate_all, }); diff --git a/services/ws-modules/wasi-data1/Cargo.toml b/services/ws-modules/wasi-data1/Cargo.toml index 1659c18..4a26e04 100644 --- a/services/ws-modules/wasi-data1/Cargo.toml +++ b/services/ws-modules/wasi-data1/Cargo.toml @@ -19,5 +19,9 @@ test = false serde_json.workspace = true wit-bindgen.workspace = true +# Build script (runs on the host) locates the repo root to emit ET_WIT_DIR. +[build-dependencies] +et-path.workspace = true + [lints] workspace = true diff --git a/services/ws-modules/wasi-data1/build.rs b/services/ws-modules/wasi-data1/build.rs new file mode 100644 index 0000000..17f2602 --- /dev/null +++ b/services/ws-modules/wasi-data1/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_WIT_DIR` (absolute path to the shared WIT directory) so the +//! `wit_bindgen::generate!` invocation in `src/lib.rs` locates it via `env!`, +//! instead of a `..`-relative path that hardcodes this crate's depth below the +//! repository root. + +fn main() { + let wit_dir = et_path::find_project_root_from_manifest().join("generated/specs/wit"); + println!("cargo:rustc-env=ET_WIT_DIR={}", wit_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/services/ws-modules/wasi-data1/src/lib.rs b/services/ws-modules/wasi-data1/src/lib.rs index 8e22e0b..5daf6d8 100644 --- a/services/ws-modules/wasi-data1/src/lib.rs +++ b/services/ws-modules/wasi-data1/src/lib.rs @@ -28,7 +28,8 @@ #![expect(unsafe_code)] wit_bindgen::generate!({ - path: "../../../generated/specs/wit", + // ET_WIT_DIR is the absolute path to generated/specs/wit, emitted by build.rs. + path: env!("ET_WIT_DIR"), world: "module", generate_all, }); diff --git a/services/ws-modules/wasi-graphics-info/pyproject.toml b/services/ws-modules/wasi-graphics-info/pyproject.toml index 18e7fe6..49bbfa2 100644 --- a/services/ws-modules/wasi-graphics-info/pyproject.toml +++ b/services/ws-modules/wasi-graphics-info/pyproject.toml @@ -11,7 +11,7 @@ repository = "https://github.com/edge-toolkit/core" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "wasi_graphics_info" diff --git a/services/ws-modules/zig-data1/src/util.c b/services/ws-modules/zig-data1/src/util.c index 57cfea0..5a70034 100644 --- a/services/ws-modules/zig-data1/src/util.c +++ b/services/ws-modules/zig-data1/src/util.c @@ -4,6 +4,7 @@ // Returns the sum of all bytes in buf, mod 256. uint8_t byte_sum(const uint8_t *buf, size_t len) { uint8_t acc = 0; - for (size_t i = 0; i < len; i++) acc += buf[i]; + for (size_t i = 0; i < len; i++) + acc += buf[i]; return acc; } diff --git a/services/ws-wasi-runner/Cargo.toml b/services/ws-wasi-runner/Cargo.toml index 73185cf..a793ae0 100644 --- a/services/ws-wasi-runner/Cargo.toml +++ b/services/ws-wasi-runner/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [lib] doctest = false -name = "et_ws_wasi_runner" path = "src/lib.rs" [[bin]] @@ -65,7 +64,7 @@ wgpu.workspace = true [dev-dependencies] et-ws-test-server.workspace = true -otlp-mock.workspace = true +int-otlp-mock.workspace = true rstest.workspace = true [lints] diff --git a/services/ws-wasi-runner/src/bindings.rs b/services/ws-wasi-runner/src/bindings.rs index 7a08e5d..f6cd0a0 100644 --- a/services/ws-wasi-runner/src/bindings.rs +++ b/services/ws-wasi-runner/src/bindings.rs @@ -23,6 +23,11 @@ //! bindgen-generated marker structs would be opaque and the `resource_table` //! couldn't carry real wgpu objects. wasmtime::component::bindgen!({ + // Sanctioned `..` exception: unlike `wit_bindgen::generate!` (the guest + // modules), wasmtime's `bindgen!` has no macro-string support, so `path:` + // can't be an `env!(...)` fed by build.rs -- it must be a literal resolved + // against this crate's dir. Kept relative-to-manifest as the one place a + // repo-relative `..` is unavoidable. path: "../../generated/specs/wit", world: "runner", imports: { default: async }, diff --git a/services/ws-wasi-runner/tests/otel_propagation.rs b/services/ws-wasi-runner/tests/otel_propagation.rs index 04fdd53..e07754b 100644 --- a/services/ws-wasi-runner/tests/otel_propagation.rs +++ b/services/ws-wasi-runner/tests/otel_propagation.rs @@ -41,7 +41,7 @@ use edge_toolkit::config::{OtlpConfig, OtlpProtocol}; #[cfg_attr(windows, ignore = "pkg/package.json 404 on Windows -- see comment above")] fn trace_ids_propagate_between_runner_and_server() { // 1. Start the mock collector. Both processes will export to it. - let mock = otlp_mock::start(); + let mock = int_otlp_mock::start(); // 2. Init OTLP in the test process *before* spawning the test server, // so the global tracing subscriber + propagator are in place when diff --git a/services/ws-web-runner/Cargo.toml b/services/ws-web-runner/Cargo.toml index 722b019..fd807f0 100644 --- a/services/ws-web-runner/Cargo.toml +++ b/services/ws-web-runner/Cargo.toml @@ -9,7 +9,6 @@ rust-version.workspace = true [lib] doctest = false -name = "et_ws_web_runner" path = "src/lib.rs" [[bin]] diff --git a/utilities/cli/Cargo.toml b/utilities/cli/Cargo.toml index a11e7d5..3fabf1f 100644 --- a/utilities/cli/Cargo.toml +++ b/utilities/cli/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] clap.workspace = true edge-toolkit.workspace = true +et-path.workspace = true fs-err.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/utilities/cli/src/cli.rs b/utilities/cli/src/cli.rs new file mode 100644 index 0000000..69fe6a5 --- /dev/null +++ b/utilities/cli/src/cli.rs @@ -0,0 +1,33 @@ +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; +use et_cli::OutputType; + +#[derive(Parser)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Generate deployment config from a cluster input YAML. + GenerateDeployment { + #[arg(long)] + input_file: PathBuf, + #[arg(long)] + output_dir: PathBuf, + #[arg(long, value_enum, default_value_t)] + output_type: OutputType, + }, + /// Regenerate verification outputs using verification input/output naming conventions. + RegenVerification { + #[arg(long, default_value = "verification")] + verification_root: PathBuf, + }, + /// Generate pkg/package.json from module metadata. + ModulePackageJson { + #[arg(long, default_value = ".")] + module_dir: PathBuf, + }, +} diff --git a/utilities/cli/src/deployment_types/docker_compose.rs b/utilities/cli/src/deployment_types/docker_compose.rs index dcb8865..ccd4664 100644 --- a/utilities/cli/src/deployment_types/docker_compose.rs +++ b/utilities/cli/src/deployment_types/docker_compose.rs @@ -1,16 +1,15 @@ use std::path::Path; use edge_toolkit::input::ClusterInput; +use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use crate::error::CliError; -use crate::{ - OutputType, absolute_from, cluster_module_names, module_registry, relative_path_from, resolve_module_paths, -}; +use crate::{OutputType, cluster_module_names, module_registry, resolve_module_paths}; pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let output_path = output_dir.join(OutputType::DockerCompose.output_file_name()); - let workspace_root = std::env::current_dir()?; + let workspace_root = edge_toolkit::config::get_project_root(); let output_abs = absolute_from(&workspace_root, output_dir); let workspace_rel = relative_path_from(&output_abs, &workspace_root); let openobserve_env_file_rel = relative_path_from(&output_abs, &workspace_root.join("config/o2.env")); diff --git a/utilities/cli/src/deployment_types/mise.rs b/utilities/cli/src/deployment_types/mise.rs index ad942fc..8c44e4d 100644 --- a/utilities/cli/src/deployment_types/mise.rs +++ b/utilities/cli/src/deployment_types/mise.rs @@ -1,15 +1,16 @@ use std::path::Path; use edge_toolkit::input::ClusterInput; +use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use toml::{Table, Value}; use crate::error::CliError; -use crate::{absolute_from, cluster_module_names, module_registry, relative_path_from, resolve_module_paths}; +use crate::{cluster_module_names, module_registry, resolve_module_paths}; pub fn generate_mise_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let output_path = output_dir.join("mise.toml"); - let workspace_root = std::env::current_dir()?; + let workspace_root = edge_toolkit::config::get_project_root(); let output_abs = absolute_from(&workspace_root, output_dir); let ws_server_dir = workspace_root.join("services/ws-server"); let workspace_rel = relative_path_from(&output_abs, &workspace_root); diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index 979150d..683a3dc 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -75,7 +75,7 @@ pub enum CliError { #[error("Verification input file {0:?} has no file stem")] MissingFileStem(PathBuf), - #[error("Verification root {0:?} does not contain any scenario files under */input/*.yaml or */input/*.yml")] + #[error("Verification root {0:?} does not contain any scenario files under */input/*.yaml")] NoScenarios(PathBuf), #[error("Unsupported deployment_type {0:?}. Supported values are currently: mise, docker-compose")] diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 60ee1a6..68b3220 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -4,11 +4,11 @@ )] use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::ffi::OsString; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use clap::ValueEnum; use edge_toolkit::input::ClusterInput; +use et_path::relative_path_from; use fs_err as fs; use serde::Deserialize; @@ -270,7 +270,7 @@ fn discover_verification_scenarios(verification_root: &Path) -> Result PathBuf { - if path.is_absolute() { - normalize_path(path) - } else { - normalize_path(&base.join(path)) - } -} - -/// Build a relative path from `from_dir` to `target`, always joined with `/`. -/// -/// The result is rendered as a POSIX string regardless of host OS, because -/// every caller writes it into generated `mise.toml` / `docker-compose.yaml` -/// output -- both of which expect forward-slash separators even on Windows. -#[must_use] -pub fn relative_path_from(from_dir: &Path, target: &Path) -> String { - let from_components = normal_components(&normalize_path(from_dir)); - let target_components = normal_components(&normalize_path(target)); - let common_len = from_components - .iter() - .zip(target_components.iter()) - .take_while(|(from, target)| from == target) - .count(); - - let mut parts: Vec = Vec::new(); - for _ in common_len..from_components.len() { - parts.push("..".to_string()); - } - for component in target_components.iter().skip(common_len) { - parts.push(component.to_string_lossy().into_owned()); - } - - if parts.is_empty() { - ".".to_string() - } else { - parts.join("/") - } -} - -fn normal_components(path: &Path) -> Vec { - path.components() - .filter_map(|component| match component { - Component::Normal(value) => Some(value.to_os_string()), - Component::Prefix(_) | Component::RootDir | Component::CurDir | Component::ParentDir => None, - }) - .collect() -} - -fn normalize_path(path: &Path) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(Path::new("/")), - Component::CurDir => {} - Component::ParentDir => { - let _popped = normalized.pop(); - } - Component::Normal(value) => normalized.push(value), - } - } - normalized -} - #[must_use] pub fn cluster_module_names(cluster: &ClusterInput) -> Vec { cluster diff --git a/utilities/cli/src/main.rs b/utilities/cli/src/main.rs index 726e7c5..a54194a 100644 --- a/utilities/cli/src/main.rs +++ b/utilities/cli/src/main.rs @@ -1,38 +1,11 @@ #![expect(clippy::print_stdout, reason = "CLI tool: println! is the intended UX")] -use std::path::PathBuf; +use clap::Parser as _; +use et_cli::{CliError, generate_deployment, generate_module_package_json, regenerate_verification}; -use clap::{Parser, Subcommand}; -use et_cli::{CliError, OutputType, generate_deployment, generate_module_package_json, regenerate_verification}; +mod cli; -#[derive(Parser)] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Generate deployment config from a cluster input YAML. - GenerateDeployment { - #[arg(long)] - input_file: PathBuf, - #[arg(long)] - output_dir: PathBuf, - #[arg(long, value_enum, default_value_t)] - output_type: OutputType, - }, - /// Regenerate verification outputs using verification input/output naming conventions. - RegenVerification { - #[arg(long, default_value = "verification")] - verification_root: PathBuf, - }, - /// Generate pkg/package.json from module metadata. - ModulePackageJson { - #[arg(long, default_value = ".")] - module_dir: PathBuf, - }, -} +use crate::cli::{Cli, Commands}; fn main() -> Result<(), CliError> { let cli = Cli::parse();