From 5bd404ee750154cdc332689be9867039ffdb26b4 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 12 Jun 2026 17:17:37 +0800 Subject: [PATCH 01/60] Dockers --- .dockerignore | 42 ++++++++-- .github/workflows/docker.yml | 78 ++++++++++++++++++ .gitignore | 12 +++ .mise/config.toml | 116 +++++++++++++++++++++++++-- Dockerfile | 148 +++++++++++++++++++++++++++++++++++ Dockerfile.windows | 53 +++++++++++++ README.md | 58 ++++++++++++++ 7 files changed, 496 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 Dockerfile create mode 100644 Dockerfile.windows diff --git a/.dockerignore b/.dockerignore index 008e65c..5d973ab 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,38 @@ -target/ -.git/ -.ruff_cache/ -.vscode/ -services/ws-server/storage/ +# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit. +# Docker reads only this file; patterns are **/-prefixed to match at any depth +# like .gitignore. Edit .gitignore and regenerate. + +**/.claude/ +**/*.wasm +**/*.onnx +**/target/ **/.DS_Store +services/ws-wasm-agent/pkg/ +services/ws-server/static/models/ +**/.zig-cache/ +**/zig-out/ +**/*.o +**/*.pem +**/mprocs.log +**/__pycache__/ +**/.pytest_cache/ +**/.python-version +**/uv.lock +**/node_modules/ +**/pnpm-lock.yaml +**/.venv/ +# .NET build output. `obj/` is safe globally (nothing tracked is named obj/), +# but `bin/` is scoped to the module so it never matches a Rust crate's +# `src/bin/` (e.g. utilities/int-gen/src/bin/). +**/obj/ +services/ws-modules/dotnet-data1/bin/ +# Editor dir (but keep the shared recommended-extensions list), ruff cache, and +# the ws-server's runtime file storage. +.vscode/* +!.vscode/extensions.json +**/.ruff_cache/ +services/ws-server/storage/ +**/.git/ +**/Dockerfile* +README.md +**/.dockerignore diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..80364eb --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,78 @@ +--- +name: docker + +# Build the verification images. Only fires when a Dockerfile (or the build +# context's .dockerignore / this workflow) changes, or on manual dispatch -- +# these builds are heavy (every toolchain + a release compile), so they don't +# run on every push. +"on": + pull_request: + paths: + - Dockerfile + - Dockerfile.windows + - .dockerignore + - .github/workflows/docker.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + # Build the Linux image (default `server` stage) and confirm the server runs. + linux: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v4 + + # The image is large (every language toolchain + prefetched models + a + # release build); the runner's root disk (~14 GB free) can't hold it. Move + # Docker's storage to the ~70 GB /mnt scratch disk before building. + - name: Move Docker storage to /mnt + run: | + sudo systemctl stop docker docker.socket + sudo mkdir -p /mnt/docker + echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json + sudo systemctl start docker + docker info --format 'Docker Root Dir: {{.DockerRootDir}}' + + # GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub limit during + # install-all; passed as a BuildKit secret so it never lands in a layer. + - name: Build the edge-toolkit image (server stage) + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + DOCKER_BUILDKIT=1 docker build \ + --secret id=gh_token,env=GITHUB_TOKEN \ + -t edge-toolkit . + + - name: Smoke-test the ws-server + run: | + docker run -d --rm -p 8080:8080 --name et edge-toolkit + ok= + for _ in $(seq 1 30); do + if curl -fsS http://localhost:8080/health; then ok=1; break; fi + sleep 2 + done + docker logs et || true + docker stop et + test -n "$ok" || { echo "ws-server did not become healthy"; exit 1; } + + # Build the Windows setup sketch. Windows containers can only be built on a + # Windows Docker host, so CI is the only place this is exercised (we can't + # build it on the Linux dev/CI boxes). Expect to iterate here -- see + # Dockerfile.windows and the README for the known unknowns. + windows: + runs-on: windows-2022 + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build the Windows setup image (sketch) + run: docker build -f Dockerfile.windows -t edge-toolkit-windows . diff --git a/.gitignore b/.gitignore index 067250e..cf8fc36 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,15 @@ __pycache__/ uv.lock node_modules/ pnpm-lock.yaml +.venv/ +# .NET build output. `obj/` is safe globally (nothing tracked is named obj/), +# but `bin/` is scoped to the module so it never matches a Rust crate's +# `src/bin/` (e.g. utilities/int-gen/src/bin/). +obj/ +services/ws-modules/dotnet-data1/bin/ +# Editor dir (but keep the shared recommended-extensions list), ruff cache, and +# the ws-server's runtime file storage. +.vscode/* +!.vscode/extensions.json +.ruff_cache/ +services/ws-server/storage/ diff --git a/.mise/config.toml b/.mise/config.toml index 2801af7..c2c7b23 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -35,6 +35,10 @@ cargo-binstall = "latest" cmake = "latest" "conda:lld" = "latest" "conda:openssl" = "3" +# Ships the C `libclang.so` (+ clang resource headers) that bindgen needs to +# build the deno web runner's libsqlite3-sys. Linux-only: macOS/Windows use +# their system/Xcode libclang. Consumed by LIBCLANG_PATH / BINDGEN_EXTRA_CLANG_ARGS. +"conda:clangxx" = { version = "latest", os = ["linux"] } dprint = "latest" editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" @@ -55,12 +59,14 @@ pipx = { version = "latest", os = ["linux", "macos"] } "npm:pnpm" = { version = "latest", os = ["macos/x64"] } pnpm = { version = "latest", os = ["linux", "macos/arm64", "windows"] } protoc = "latest" +python = "3.13" rclone = "latest" rust = [ { version = "latest", components = "clippy,rust-analyzer", targets = "wasm32-unknown-unknown,wasm32-wasip2" }, { version = "nightly", components = "rust-src,rustfmt", targets = "wasm32-unknown-unknown,wasm32-wasip2" }, ] typos = "latest" +uv = "0.11.8" wasm-tools = "latest" yq = "latest" @@ -69,10 +75,34 @@ yq = "latest" # CARGO_TARGET_* RUSTFLAGS. conda_lld = "{{ env.HOME }}/.local/share/mise/installs/conda-lld/latest" conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" +conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" +clang_resource = "{{ vars.conda_clangxx }}/lib/clang/22" +# conda's target-triple dir uses x86_64/aarch64; mise's arch() is x64/arm64. +conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" +clang_sysroot = "{{ vars.conda_clangxx }}/{{ vars.conda_arch }}-conda-linux-gnu/sysroot" +# bindgen loads libclang directly rather than driving the clang binary, so it +# can't auto-find its resource dir (builtins like stddef.h/stdint.h) or conda's +# bundled sysroot. Pass `-resource-dir` (not just `-isystem .../include`, which +# leaves the builtin type macros unset and fails in CI) + `--sysroot` so header +# resolution is identical on every machine. +bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_sysroot }}" +# mise-managed CPython 3.13 interpreter, composed into PYO3_PYTHON below. Split +# per-OS because the mise data dir differs (LOCALAPPDATA\mise on Windows, +# HOME-based elsewhere). `get_env(..., default='')` guards the Windows-only +# LOCALAPPDATA so the var still renders on Linux/macOS -- a bare `env.LOCALAPPDATA` +# is "not found in context" off-Windows and hard-fails the whole config render. +py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" +py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" # Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env # values below — kept as vars so those values stay on a single line. lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" rpath_flag = "-C link-arg=-Wl,-rpath,{{ vars.conda_openssl }}/lib" +# rpath to the mise CPython 3.13 lib dir so the et-ws-pyo3-runner binary finds +# libpython at runtime (libpython3.13.so on Linux, libpython3.13.dylib on macOS +# via its @rpath install name). pyo3 bakes an rpath to the resolved patch-version +# dir, which doesn't exist when mise installs under the "3.13" alias; point at +# the stable install path instead. +pylib_flag = "-C link-arg=-Wl,-rpath,{{ env.HOME }}/.local/share/mise/installs/python/3.13/lib" wasm_rustflags = "-C target-cpu=mvp -C target-feature=+mutable-globals,+sign-ext,+nontrapping-fptoint" # Source/dest paths for the fetch-*-rclone model downloads — kept as vars so # each `rclone copyto` command stays on a single line without continuations. @@ -102,8 +132,8 @@ OPENSSL_DIR = "{{ vars.conda_openssl }}" # `-fuse-ld=` at the absolute `ld64.lld` shipped by `conda:lld` so the flag # resolves without any `PATH` lookup — same conda root the rpath link-arg # already points into. -CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }}" -CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }}" +CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" # Linux needs the same rpath the macOS flags above carry, for the same reason: # OPENSSL_DIR points openssl-sys at conda's OpenSSL, so anything linking it # (e.g. the ort-sys build script) records conda's `libssl` soname. Without an @@ -112,8 +142,27 @@ CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_ # bumped to `libssl.so.4` (no system equivalent). Point the loader at conda's # own lib dir so the soname is irrelevant. No `-fuse-ld=lld`: the system # linker on Linux accepts `-Wl,-rpath` directly, unlike Apple clang's lld dance. -CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }}" -CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }}" +CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +# bindgen (pulled in transitively by the deno web runner) needs the C +# `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ +# API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys +# at it, and feed bindgen conda's own resource headers + bundled sysroot (see +# `bindgen_args`). Using conda's sysroot rather than the host's `/usr/include` +# makes bindgen self-contained, so CI resolves headers identically to a local box +# -- no environment-specific divergence. Same conda-managed, no-admin approach as +# OPENSSL_DIR; Linux-only (macOS/Windows use system/Xcode libclang). mise can't +# omit an env key, so off-Linux these render empty -> clang-sys ignores them. +LIBCLANG_PATH = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% endif %}" +BINDGEN_EXTRA_CLANG_ARGS = "{% if os() == 'linux' %}{{ vars.bindgen_args }}{% endif %}" +# pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from +# PYO3_PYTHON, else the first python on PATH -- under mise that's the 32-bit +# Pyodide shim, so the build dies with "target architecture (64-bit) does not +# match your python interpreter (32-bit)". Pin it to the mise-managed CPython +# 3.13 (the `python` tool above). Always-loaded so `cargo check --workspace` +# builds the pyo3 runner without `MISE_ENV=python`. Direct path, not `exec()`: +# an exec runs on every command and a failure hard-fails the config render. +PYO3_PYTHON = "{% if os() == 'windows' %}{{ vars.py313_win }}{% else %}{{ vars.py313_unix }}{% endif %}" # rclone retry tuning for the `fetch-*-rclone` model-download tasks. # Hugging Face / GitHub raw responded HTTP 429 ("Too Many Requests") at # least once with all three of rclone's default high-level retries @@ -172,6 +221,7 @@ depends = [ "cargo-clippy", "cargo-doc-check", "cargo-fmt-check", + "docker-check", "dprint-check", "editorconfig-check", "gen-specs-check", @@ -218,7 +268,7 @@ run = "cargo clippy --keep-going --workspace --tests" [tasks.cargo-doc-check] description = "Build rustdoc with -D warnings to fail on any doc issues" env = { RUSTDOCFLAGS = "-D warnings" } -run = "cargo doc --workspace --no-deps --document-private-items" +run = "cargo doc --keep-going --workspace --no-deps --document-private-items" [tasks.cargo-clippy-fix] run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace --tests" @@ -239,11 +289,14 @@ run = """ taplo lint # All member Cargo.tomls — exclude workspace root and any build output. +# `-prune` target/ and .git so find never descends into them: `-not -path` +# alone still walks the whole tree, and target/debug/deps/ churns constantly +# under a concurrent `cargo-check`, racing find into "No such file" errors. # (Avoid `mapfile`: it's bash 4+ only and macOS ships bash 3.2.) members=() while IFS= read -r line; do members+=("$line") -done < <(find . -name Cargo.toml -not -path './target/*' -not -path './Cargo.toml') +done < <(find . -path ./target -prune -o -path ./.git -prune -o -name Cargo.toml ! -path ./Cargo.toml -print) # Banned deps only need to be checked at the workspace root: member # crates are required to reference deps via `workspace = true` (the @@ -303,6 +356,57 @@ mise install aube@latest depends = ["cargo-check"] run = "osv-scanner --lockfile Cargo.lock" +[tasks."gen:osv-scanner"] +description = "Regenerate osv-scanner.toml from deny.toml's [advisories].ignore list" +# osv-scanner and cargo-deny must ignore the same advisory IDs; deny.toml is the +# source of truth (it carries the per-ID rationale). grep/sort/sed only, so the +# `dependencies` workflow can run it (via mise) next to the audit binaries. +run = """ +{ + echo '# AUTO-GENERATED from deny.toml by `mise run gen:osv-scanner`.' + echo 'IgnoredVulns = [' + grep -oE '"RUSTSEC-[0-9]{4}-[0-9]{4}"' deny.toml | sort -u | sed 's/.*/ { id = & },/' + echo ']' +} >osv-scanner.toml +""" +shell = "bash -euo pipefail -c" + +[tasks."gen:dockerignore"] +description = "Regenerate .dockerignore from .gitignore + Docker-only excludes" +# Docker reads only the root .dockerignore (never .gitignore or nested ignore +# files), and a bare pattern there is root-anchored -- whereas a slash-less +# .gitignore pattern matches at any depth. So mirror .gitignore, add the +# Docker-only excludes (`.git/`, plus the build recipe + docs so editing them +# doesn't bust the `COPY . .` cache), and `**/`-prefix each non-anchored pattern. +run = ''' +{ + echo "# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit." + echo "# Docker reads only this file; patterns are **/-prefixed to match at any depth" + echo "# like .gitignore. Edit .gitignore and regenerate." + echo + { + cat .gitignore + printf '%s\n' '.git/' 'Dockerfile*' '/README.md' '.dockerignore' + } | awk ' + /^[[:space:]]*#/ { print; next } + /^[[:space:]]*$/ { print; next } + { + neg = ""; line = $0 + if (substr(line, 1, 1) == "!") { neg = "!"; line = substr(line, 2) } + if (substr(line, 1, 1) == "/") { print neg substr(line, 2); next } + core = line; sub(/\/$/, "", core) + if (index(core, "/") > 0) print neg line; else print neg "**/" line + } + ' +} >.dockerignore +''' +shell = "bash -euo pipefail -c" + +[tasks.docker-check] +depends = ["gen:dockerignore"] +description = "Fail if .dockerignore is stale vs .gitignore (run `mise run gen:dockerignore`)" +run = "git diff --exit-code -- .dockerignore" + [tasks.prefetch-ci] # `build-ws-wasm-agent` produces `services/ws-wasm-agent/pkg/`, which the # static workspace references via a `link:` path. Run it before `pnpm install` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4b788df --- /dev/null +++ b/Dockerfile @@ -0,0 +1,148 @@ +# Verify the README's mise setup end-to-end from a clean, minimal Ubuntu, split +# into stages so each can be cached and targeted independently: +# +# build install mise + every toolchain (README setup only; no build/test) +# prefetch download all dependencies + ONNX models +# precompile build the WASM/JS modules (drops target/ to stay slim) +# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) +# server release build of et-ws-server, served by default (final stage) +# +# Each stage `FROM`s the previous, so installed tools, downloaded deps, and the +# built module pkg/ carry forward. Stop early with `--target`. +# +# It follows README.md's "mise" setup verbatim (install mise -> configure -> +# install conda:openssl -> install-all). The OpenObserve (`o2`/`open-o2`) and +# `ws-server` steps from the README's "Run ws agent" section are intentionally +# skipped -- they are runtime services, not build/test verification. +# +# The goal is to catch README drift: if a documented step is missing or wrong, +# it fails. Anything language-specific must come from mise (per the README's +# "installed into the local workspace" promise); the only apt packages below are +# universal build prereqs a normal dev machine has, so a failure that needs +# another system lib is itself a finding to fold back into the README. +# +# A plain build produces the SERVER image (final stage): a release et-ws-server, +# served automatically. A GitHub token avoids mise's 60-req/hr anonymous limit +# (README "GitHub rate limits"): +# DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . +# docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 +# (drop --secret to build tokenless; install-all may then hit rate limits) +# +# To run the verification suite, target the non-final `test` stage and pass the +# host GPU (`docker build` can't attach one). The stage bundles mesa-vulkan- +# drivers, so the wgpu test gets a real Intel/AMD GPU via the DRI node (or a +# software fallback if none is passed): +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# NVIDIA via `--gpus all` is wired but UNVERIFIED (its in-container Vulkan ICD +# doesn't initialize yet) -- prefer a DRI device. + +# --- build: install mise + every language toolchain (README "mise" setup). --- +# No prefetch/build/test, so this layer is reused until the setup itself changes. +FROM ubuntu:24.04 AS build + +# Universal prereqs a typical dev box already has; everything else is mise's job. +# build-essential : cc/ld to link Rust binaries + build C deps +# curl + ca-certs : the mise installer and tool downloads +# git : cargo + repo operations +# xz-utils/unzip/bzip2 : mise unpacking tool archives (e.g. the pyodide .tar.bz2) +# libicu74 : .NET runtime ICU, for the dotnet-data1 module's build/test. +# Without it the dotnet CLI FailFast-aborts at startup with +# "Couldn't find a valid ICU package installed on the system" +# (minimal Ubuntu ships no ICU). The "74" tracks the Ubuntu +# base (74 = 24.04) -- bump it alongside the FROM line. README +# gap: the README's setup should note that .NET needs libicu on +# minimal systems (or set System.Globalization.Invariant=true). +# libvulkan1 : Vulkan loader for the wgpu compute test (wasi-graphics-info). +# Just the loader -- no software driver (lavapipe): the GPU's +# real ICD is injected at `docker run` time by the NVIDIA +# Container Toolkit (`--gpus all`), so the test runs on actual +# hardware. (AMD/Intel: add mesa-vulkan-drivers + --device /dev/dri.) +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl git xz-utils unzip bzip2 libicu74 \ + libvulkan1 \ + && rm -rf /var/lib/apt/lists/* + +# README: "Please install mise, including the shell integration." In a +# non-interactive build, putting mise + its shims on PATH is the equivalent -- +# every `mise` / `mise run` below then resolves the workspace tools. +RUN curl -fsSL https://mise.run | sh +ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" + +# README "Configure it" + "Pre-install cargo-install". A GitHub token (if +# provided) lifts the anonymous rate limit for the cargo-binstall release fetch. +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + sh -c 'mise settings experimental=true \ + && mise settings set cargo.binstall true \ + && mise use -g cargo-binstall' + +WORKDIR /workspace +COPY . . + +# A fresh checkout's config is untrusted and mise would prompt interactively +# (which a build can't answer). The README omits this because a dev's first +# `mise` command prompts once; trust it up front instead. +RUN mise trust + +# README "All OS": openssl dev files, then every language toolchain. `mise +# install node` first per the README's note that mise may need node to install +# other tools. `install-all` == `MISE_ENV="$ALL_LANGS" mise install`. +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + sh -c 'mise install node \ + && mise install conda:openssl \ + && mise install-all' + +# Mirrors what the *-all tasks set internally, so the bare task names in the +# stages below act on all languages, not just Rust. Inherited by every stage. +ENV MISE_ENV="dart,dotnet,java,python,rust,zig" + +# --- prefetch: download all dependencies + ONNX models. --- +FROM build AS prefetch +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + mise run prefetch + +# --- precompile: build the WASM/JS modules (needed by test and server). --- +# `&& rm -rf target` in the SAME layer: build-modules leaves multi-GB cargo +# intermediates in target/, but the module outputs live in each module's pkg/. +# Dropping target/ here keeps it out of this layer and the stages built on it; +# test and server recompile only what they need. +FROM prefetch AS precompile +RUN mise run build-modules && rm -rf target + +# --- test: the full suite (Rust + web runner + every guest language). --- +# Compiled AND run at `docker run` time (precompile keeps no target/), so the +# multi-GB debug test binaries never bake into a layer -- they live in the +# ephemeral container and vanish when it exits. The wgpu compute test needs a +# Vulkan device; mesa-vulkan-drivers provides one -- a real Intel/AMD GPU when the +# host DRI node is passed with `--device /dev/dri`, else a CPU (lavapipe) fallback +# so the suite still runs. (NVIDIA's `--gpus` Vulkan path doesn't initialize in a +# container.) Installed here, not the build stage, to keep that layer cached. +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# NVIDIA via `--gpus all` is wired (NVIDIA_DRIVER_CAPABILITIES=all below, needs +# the NVIDIA Container Toolkit) but UNVERIFIED -- its in-container Vulkan ICD +# doesn't initialize yet, so prefer a DRI device for now. +FROM precompile AS test +ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=all +RUN apt-get update \ + && apt-get install -y --no-install-recommends mesa-vulkan-drivers \ + && rm -rf /var/lib/apt/lists/* +CMD ["mise", "run", "test"] + +# --- server: release build of et-ws-server, the default image (final stage). --- +# A plain `docker build` produces this. The release binary is copied out and +# target/ dropped in the SAME layer so the build intermediates don't bloat the +# image; the binary finds its libs via baked rpaths and serves each module from +# its pkg/ (none of which live in target/). mise stays on PATH and MISE_ENV is +# set, so the server's `mise where` module-path lookups resolve. +# docker run --rm -p 8080:8080 edge-toolkit # then open http://localhost:8080 +FROM precompile AS server +RUN mise exec -- cargo build --release -p et-ws-server \ + && cp target/release/et-ws-server /usr/local/bin/et-ws-server \ + && rm -rf target +EXPOSE 8080 8443 +CMD ["et-ws-server"] diff --git a/Dockerfile.windows b/Dockerfile.windows new file mode 100644 index 0000000..a330bb4 --- /dev/null +++ b/Dockerfile.windows @@ -0,0 +1,53 @@ +# escape=` +# SKETCH (UNVERIFIED): reproduce the README's mise setup on a bare Windows box, +# to discover what a clean Windows machine actually needs. Windows containers +# build only on a Windows Docker host, so this is exercised solely by the +# `windows` job in .github/workflows/docker.yml -- it cannot be built on the +# Linux dev/CI machines, so treat it as a starting point to iterate on via CI, +# not a known-good image. `escape=`` ` `` above makes backtick the line +# continuation (matching PowerShell) and frees `\` so Windows paths are literal. +# +# Base: Windows Server Core, pinned to the runner's OS. Windows process isolation +# needs the container build == host build (ltsc2022 == the windows-2022 runner); +# nanoserver is too small to install toolchains. Server Core is multi-GB on its +# own, before anything below -- Windows images are large and slow to build. +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop';"] + +# A bare Windows has none of the build prereqs the README assumes (it documents +# only pipx). Install them with Chocolatey: +# git, python : cargo + the README's `python -m pip install pipx` (Windows has +# no aqua pipx build, so the `pipx:*` mise tools need pip's pipx). +# llvm : libclang.dll for bindgen (the deno web runner's libsqlite3-sys); +# lands in C:\Program Files\LLVM, which .mise/config.toml's +# LIBCLANG_PATH already points at. +# mise : the toolchain manager itself. +# visualstudio2022buildtools (VCTools) : MSVC cl/link + the Windows SDK that +# Rust's msvc toolchain links through (a clean Windows lacks them). +RUN Set-ExecutionPolicy Bypass -Scope Process -Force; ` + iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +RUN choco install -y --no-progress git python llvm mise +RUN choco install -y --no-progress visualstudio2022buildtools ` + --package-parameters '--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' + +WORKDIR C:\workspace +COPY . . + +# README "mise" setup, Windows flavour. pipx via pip per the README's Windows +# note, so the always-loaded `pipx:*` tools resolve. Then install the default +# toolchain and run the Rust check -- this is the basic smoke (and it exercises +# the Windows LIBCLANG_PATH fix, since the web runner's libsqlite3-sys runs +# bindgen). A full `install-all` / `test` across every guest language is far +# heavier on Windows containers; expand once these basics are green. +# +# Known unknowns to settle via CI: mise's PATH after the choco install; whether +# cargo finds the VS Build Tools without an explicit `vcvars` shell; and whether +# every always-loaded `pipx:*` tool (e.g. semgrep) installs cleanly on Windows. +RUN mise trust; ` + mise settings experimental=true; ` + mise settings set cargo.binstall true; ` + mise use -g cargo-binstall; ` + python -m pip install --no-warn-script-location pipx +RUN mise install; ` + mise run cargo-check diff --git a/README.md b/README.md index ef443c3..0e3a7df 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,64 @@ mise install-all Use `mise run fmt-all` and `mise run check-all` to run formatters and checkers. +## Building and running with Docker + +[`Dockerfile`](Dockerfile) reproduces the mise setup above on a clean, minimal +Ubuntu, in stages (`build` → `prefetch` → `precompile` → `test`/`server`). A +plain build produces the **server** image (the final stage): a release build of +`et-ws-server`, served automatically. `mise install-all` fetches many tools from +GitHub releases, so build with a GitHub token to avoid the anonymous +60-requests/hour limit (see [GitHub rate limits](#github-rate-limits)), passed as +a BuildKit secret so it never lands in an image layer: + +```bash +GITHUB_TOKEN="$(gh auth token)" DOCKER_BUILDKIT=1 \ + docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . +docker run --rm -p 8080:8080 edge-toolkit +``` + +Then open (add `-p 8443:8443` for TLS). The server needs +no GPU. OpenObserve/`o2` is optional — OTLP export is off when no collector is +configured. (Drop `--secret` to build without a token; `install-all` may then hit +rate limits.) + +The full test suite is a **separate, non-final stage**, so build it explicitly +with `--target test`. The WebGPU compute test needs a GPU, and `docker build` +can't attach one (no `--gpus` for build), so it runs at `docker run` time. The +`test` stage bundles `mesa-vulkan-drivers`, so passing the host DRI node gives +wgpu a real Intel/AMD GPU (and a software fallback if you pass nothing): + +```bash +docker build --target test -t edge-toolkit-test . +docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD GPU +``` + +NVIDIA via `--gpus all` (with the NVIDIA Container Toolkit) is wired but +**unverified** — its in-container Vulkan ICD doesn't initialize yet, so prefer a +DRI device. The image skips the `o2`/`ws-server` README steps (runtime services). + +### CI + +The [`docker`](.github/workflows/docker.yml) workflow rebuilds these images when +a `Dockerfile*` or `.dockerignore` changes (or on manual dispatch) — both builds +are too heavy for every push. A `linux` job builds the server image and curls +`/health`; a `windows` job builds the sketch below. + +### Windows setup sketch + +[`Dockerfile.windows`](Dockerfile.windows) is an **unverified sketch** of the +same setup on a bare Windows machine — useful for finding what a clean Windows +needs beyond the README. Windows containers build only on a Windows Docker host, +so the `windows` CI job is the only place it runs (not the Linux dev/CI boxes). +It starts from `windows/servercore` (pinned to the runner's OS so process +isolation works) and installs, via Chocolatey, what a clean Windows lacks: the +MSVC **Visual Studio Build Tools** (C++ + Windows SDK) that Rust's msvc toolchain +links through, **LLVM** for bindgen's `libclang.dll`, plus git and python — then +mise, the README setup, and `mise run cargo-check`. Those extra prereqs are +findings the README's "Windows only" section should eventually formalise. Expect +to iterate on it via CI; the `mise install` is kept basic (not `install-all`) +until the fundamentals pass. + ## Run ws agent in browser ### Build modules and run the WS server From 690567494e82bbe2227e604f9d7798226174b0a7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 12 Jun 2026 19:25:57 +0800 Subject: [PATCH 02/60] slimmer wind --- .github/workflows/docker.yml | 26 ++---- .mise/config.toml | 15 +++- Dockerfile.windows | 102 +++++++++++++---------- README.md | 33 ++++---- config/semgrep/no-trailing-backslash.yml | 2 +- 5 files changed, 100 insertions(+), 78 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 80364eb..9929527 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,7 +22,9 @@ concurrency: cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: - # Build the Linux image (default `server` stage) and confirm the server runs. + # Build the Linux `test` stage and run the suite. GitHub runners have no GPU, + # so the stage's bundled mesa drivers fall back to lavapipe (software Vulkan) + # for the WebGPU compute test. linux: runs-on: ubuntu-latest timeout-minutes: 120 @@ -43,25 +45,15 @@ jobs: # GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub limit during # install-all; passed as a BuildKit secret so it never lands in a layer. - - name: Build the edge-toolkit image (server stage) + - name: Build the edge-toolkit test image env: GITHUB_TOKEN: ${{ github.token }} - run: | - DOCKER_BUILDKIT=1 docker build \ - --secret id=gh_token,env=GITHUB_TOKEN \ - -t edge-toolkit . + run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . - - name: Smoke-test the ws-server - run: | - docker run -d --rm -p 8080:8080 --name et edge-toolkit - ok= - for _ in $(seq 1 30); do - if curl -fsS http://localhost:8080/health; then ok=1; break; fi - sleep 2 - done - docker logs et || true - docker stop et - test -n "$ok" || { echo "ws-server did not become healthy"; exit 1; } + # The `test` stage's CMD is `mise run test`; running the image runs the + # suite. No --device: with no GPU, mesa falls back to lavapipe. + - name: Run the test suite + run: docker run --rm edge-toolkit-test # Build the Windows setup sketch. Windows containers can only be built on a # Windows Docker host, so CI is the only place this is exercised (we can't diff --git a/.mise/config.toml b/.mise/config.toml index c2c7b23..9d5b575 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -36,9 +36,20 @@ cmake = "latest" "conda:lld" = "latest" "conda:openssl" = "3" # Ships the C `libclang.so` (+ clang resource headers) that bindgen needs to -# build the deno web runner's libsqlite3-sys. Linux-only: macOS/Windows use -# their system/Xcode libclang. Consumed by LIBCLANG_PATH / BINDGEN_EXTRA_CLANG_ARGS. +# build the deno web runner's libsqlite3-sys. Linux-only: macOS uses its +# system/Xcode libclang; Windows gets libclang from llvm-mingw (below). +# Consumed by LIBCLANG_PATH / BINDGEN_EXTRA_CLANG_ARGS. "conda:clangxx" = { version = "latest", os = ["linux"] } +# Windows toolchain, all mise-provided so a bare Windows box needs nothing +# preinstalled except mise itself (see Dockerfile.windows). The conda msys2 +# mirror packages give the `bash` the `bash -euo pipefail` tasks expect and a +# `git`; llvm-mingw bundles clang + lld + the mingw-w64 runtime/headers/libs +# (the `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen +# loads -- found via PATH once mise activates the tool. The MSVC CRT + Windows +# SDK can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. +"conda:m2-bash" = { version = "latest", os = ["windows"] } +"conda:m2-git" = { version = "latest", os = ["windows"] } +"ubi:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" diff --git a/Dockerfile.windows b/Dockerfile.windows index a330bb4..7ee3a8a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -1,53 +1,67 @@ # escape=` -# SKETCH (UNVERIFIED): reproduce the README's mise setup on a bare Windows box, -# to discover what a clean Windows machine actually needs. Windows containers -# build only on a Windows Docker host, so this is exercised solely by the -# `windows` job in .github/workflows/docker.yml -- it cannot be built on the -# Linux dev/CI machines, so treat it as a starting point to iterate on via CI, -# not a known-good image. `escape=`` ` `` above makes backtick the line -# continuation (matching PowerShell) and frees `\` so Windows paths are literal. +# SKETCH (UNVERIFIED): build edge-toolkit on a *bare* Windows box where mise +# supplies the entire toolchain -- the point is to prove the README's mise setup +# needs nothing hand-installed but mise itself. Windows containers build only on +# a Windows Docker host, so this runs solely in the `windows` job of +# .github/workflows/docker.yml; it can't be built on the Linux dev/CI machines. +# Treat it as a CI-iterated starting point, not a known-good image. # -# Base: Windows Server Core, pinned to the runner's OS. Windows process isolation -# needs the container build == host build (ltsc2022 == the windows-2022 runner); -# nanoserver is too small to install toolchains. Server Core is multi-GB on its -# own, before anything below -- Windows images are large and slow to build. -FROM mcr.microsoft.com/windows/servercore:ltsc2022 +# Base: Nano Server, the smallest Windows base (~120 MB vs Server Core's ~1.25 +# GB). It has no MSI/installer stack, no PowerShell, and runs unprivileged -- so +# the Visual Studio Build Tools installer can't run here at all. That's the +# point: nothing is installed the Windows way. We bootstrap mise (a single .exe) +# with the curl.exe + tar.exe built into Nano Server, and `mise install` pulls +# the rest -- rust, llvm-mingw (clang/lld/mingw-w64 + libclang), bash and git +# (see the os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows +# SDK can't come from mise, so the build targets x86_64-pc-windows-gnu. +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 -SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop';"] +# cmd is the only shell Nano Server has; ContainerAdministrator so we can write +# under C:\. `escape=`` ` `` (line 1) makes backtick the line continuation so the +# `\` in Windows paths stays literal. +USER ContainerAdministrator +SHELL ["cmd", "/S", "/C"] -# A bare Windows has none of the build prereqs the README assumes (it documents -# only pipx). Install them with Chocolatey: -# git, python : cargo + the README's `python -m pip install pipx` (Windows has -# no aqua pipx build, so the `pipx:*` mise tools need pip's pipx). -# llvm : libclang.dll for bindgen (the deno web runner's libsqlite3-sys); -# lands in C:\Program Files\LLVM, which .mise/config.toml's -# LIBCLANG_PATH already points at. -# mise : the toolchain manager itself. -# visualstudio2022buildtools (VCTools) : MSVC cl/link + the Windows SDK that -# Rust's msvc toolchain links through (a clean Windows lacks them). -RUN Set-ExecutionPolicy Bypass -Scope Process -Force; ` - iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) -RUN choco install -y --no-progress git python llvm mise -RUN choco install -y --no-progress visualstudio2022buildtools ` - --package-parameters '--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' +# Bootstrap mise from its CDN zip (curl.exe + tar.exe ship in Nano Server). +# Extract to C:\ and call mise by absolute path below, so we don't depend on +# Windows-container PATH propagation between RUN layers. (Zip layout assumed +# mise\bin\mise.exe -- one of the things to confirm via CI.) +RUN curl -fsSL -o mise.zip https://mise.jdx.dev/mise-latest-windows-x64.zip && ` + tar -xf mise.zip -C C:\ && ` + del mise.zip WORKDIR C:\workspace COPY . . -# README "mise" setup, Windows flavour. pipx via pip per the README's Windows -# note, so the always-loaded `pipx:*` tools resolve. Then install the default -# toolchain and run the Rust check -- this is the basic smoke (and it exercises -# the Windows LIBCLANG_PATH fix, since the web runner's libsqlite3-sys runs -# bindgen). A full `install-all` / `test` across every guest language is far -# heavier on Windows containers; expand once these basics are green. +# README "mise" setup. `mise install` pulls the whole toolchain declared in +# .mise/config.toml; experimental is on for the conda/ubi backends, and trust is +# needed for the copied-in config. +RUN C:\mise\bin\mise.exe trust && ` + C:\mise\bin\mise.exe settings experimental=true && ` + C:\mise\bin\mise.exe settings set cargo.binstall true && ` + C:\mise\bin\mise.exe install + +# mise's core rust installs the msvc-host toolchain by default, but Nano Server +# has no msvc linker -- so flip rustup (managed inside mise) to the gnu host and +# build the gnu target, which links via the mise-provided llvm-mingw. mise's +# core:rust has no host-triple knob, so we drive rustup directly. This seam is +# the main thing to confirm via CI. +RUN C:\mise\bin\mise.exe exec -- rustup set default-host x86_64-pc-windows-gnu && ` + C:\mise\bin\mise.exe exec -- rustup default stable-x86_64-pc-windows-gnu + +# Basic smoke: a workspace check against the gnu target (the linker is +# llvm-mingw's gcc wrapper, on PATH once mise activates the tool). Heavier work +# (install-all / test across every guest language) is a later step once these +# fundamentals are green. +ENV CARGO_BUILD_TARGET=x86_64-pc-windows-gnu ` + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc +RUN C:\mise\bin\mise.exe run cargo-check # -# Known unknowns to settle via CI: mise's PATH after the choco install; whether -# cargo finds the VS Build Tools without an explicit `vcvars` shell; and whether -# every always-loaded `pipx:*` tool (e.g. semgrep) installs cleanly on Windows. -RUN mise trust; ` - mise settings experimental=true; ` - mise settings set cargo.binstall true; ` - mise use -g cargo-binstall; ` - python -m pip install --no-warn-script-location pipx -RUN mise install; ` - mise run cargo-check +# Known unknowns to settle via CI: +# * whether the rustup host flip sticks (mise may re-pin its own toolchain), +# and whether a gnu-host proc-macro/build-script build is clean; +# * whether `ort` (ONNX Runtime, msvc-only prebuilts) breaks the gnu link -- +# cargo check shouldn't link it, but a later cargo build/test will; +# * whether the conda backend (micromamba) and the ubi llvm-mingw asset-match +# resolve cleanly on Nano Server; +# * whether mise finds bash (m2-bash) for the `bash -euo pipefail` tasks. diff --git a/README.md b/README.md index 0e3a7df..0708bea 100644 --- a/README.md +++ b/README.md @@ -120,23 +120,28 @@ DRI device. The image skips the `o2`/`ws-server` README steps (runtime services) The [`docker`](.github/workflows/docker.yml) workflow rebuilds these images when a `Dockerfile*` or `.dockerignore` changes (or on manual dispatch) — both builds -are too heavy for every push. A `linux` job builds the server image and curls -`/health`; a `windows` job builds the sketch below. +are too heavy for every push. A `linux` job builds the `test` stage and runs the +suite (lavapipe software Vulkan, since the runners have no GPU); a `windows` job +builds the sketch below. ### Windows setup sketch -[`Dockerfile.windows`](Dockerfile.windows) is an **unverified sketch** of the -same setup on a bare Windows machine — useful for finding what a clean Windows -needs beyond the README. Windows containers build only on a Windows Docker host, -so the `windows` CI job is the only place it runs (not the Linux dev/CI boxes). -It starts from `windows/servercore` (pinned to the runner's OS so process -isolation works) and installs, via Chocolatey, what a clean Windows lacks: the -MSVC **Visual Studio Build Tools** (C++ + Windows SDK) that Rust's msvc toolchain -links through, **LLVM** for bindgen's `libclang.dll`, plus git and python — then -mise, the README setup, and `mise run cargo-check`. Those extra prereqs are -findings the README's "Windows only" section should eventually formalise. Expect -to iterate on it via CI; the `mise install` is kept basic (not `install-all`) -until the fundamentals pass. +[`Dockerfile.windows`](Dockerfile.windows) is an **unverified sketch** that +proves the stronger claim: on a bare Windows box, mise supplies the _entire_ +toolchain and nothing is installed the Windows way. It starts from **Nano +Server** (the smallest Windows base, ~120 MB) — which has no installer stack, +PowerShell, or admin shell, so the Visual Studio Build Tools installer can't run +there at all. Instead it bootstraps just `mise.exe` (via the `curl`/`tar` built +into Nano Server) and lets `mise install` pull everything else: rust, an +`llvm-mingw` toolchain (clang + lld + the mingw-w64 runtime and `libclang.dll`), +plus `bash` and `git` from conda's msys2 packages — all declared os-guarded in +[`.mise/config.toml`](.mise/config.toml). Because the MSVC CRT + Windows SDK are +the one thing mise can't supply, the build targets `x86_64-pc-windows-gnu` +rather than `-msvc`. Windows containers build only on a Windows Docker host, so +the `windows` CI job is the only place it runs; expect to iterate there (the +rustup gnu-host flip and whether msvc-only prebuilts like `ort` link are the +open questions). Whatever it ends up needing are findings the README's "Windows +only" section should fold in. ## Run ws agent in browser diff --git a/config/semgrep/no-trailing-backslash.yml b/config/semgrep/no-trailing-backslash.yml index d30a60c..b035e9d 100644 --- a/config/semgrep/no-trailing-backslash.yml +++ b/config/semgrep/no-trailing-backslash.yml @@ -9,7 +9,7 @@ rules: - "/.mise/config.python.toml" - "/README.md" - "/utilities/cli/README.md" - - "/services/ws-server/Dockerfile" + - "**/Dockerfile*" # Generated deployment artifacts (regen-verification output). - "/verification/**/mise.toml" - "/verification/**/compose.yaml" From 9db27ee0d79ceb1a330b52ae94f899af31604fa4 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 12 Jun 2026 19:49:56 +0800 Subject: [PATCH 03/60] window --- Dockerfile.windows | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile.windows b/Dockerfile.windows index 7ee3a8a..3bb5cc1 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -14,7 +14,16 @@ # the rest -- rust, llvm-mingw (clang/lld/mingw-w64 + libclang), bash and git # (see the os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows # SDK can't come from mise, so the build targets x86_64-pc-windows-gnu. + +# Donor stage: Nano Server omits the VC++ runtime that msvc-built executables +# (mise.exe, and the rust/cargo mise installs) need just to *start* -- without it +# mise.exe dies with 0xC0000135 (DLL not found). It isn't a tool mise can install +# (mise needs it to run), so copy the DLL from Server Core, which ships it. This +# plus mise.exe are the only two non-mise bootstrap bits; everything else is mise. +FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS vcruntime + FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\. `escape=`` ` `` (line 1) makes backtick the line continuation so the From acf5b075586e13de92cb672e26b9ac7e67b8fcc7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 12 Jun 2026 19:56:18 +0800 Subject: [PATCH 04/60] win --- Dockerfile.windows | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 3bb5cc1..ebfd3ff 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -17,13 +17,24 @@ # Donor stage: Nano Server omits the VC++ runtime that msvc-built executables # (mise.exe, and the rust/cargo mise installs) need just to *start* -- without it -# mise.exe dies with 0xC0000135 (DLL not found). It isn't a tool mise can install -# (mise needs it to run), so copy the DLL from Server Core, which ships it. This -# plus mise.exe are the only two non-mise bootstrap bits; everything else is mise. +# mise.exe dies with 0xC0000135 (DLL not found). No base Windows image ships it +# (not even Server Core), and mise can't install it (mise needs it to run), so we +# install the official VC++ redistributable on a Server Core donor -- which has +# the installer stack Nano Server lacks -- and copy the runtime DLLs forward. +# This plus mise.exe are the only non-mise bootstrap bits; the rest is mise. FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS vcruntime +SHELL ["cmd", "/S", "/C"] +# `|| ver>nul` swallows the redistributable's 3010 ("reboot required") exit code. +RUN curl -fsSL -o vc_redist.exe https://aka.ms/vs/17/release/vc_redist.x64.exe && ` + (vc_redist.exe /install /quiet /norestart || ver>nul) && ` + del vc_redist.exe FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 -COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ +COPY --from=vcruntime ` + C:\Windows\System32\vcruntime140.dll ` + C:\Windows\System32\vcruntime140_1.dll ` + C:\Windows\System32\msvcp140.dll ` + C:\Windows\System32\ # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\. `escape=`` ` `` (line 1) makes backtick the line continuation so the From ec54f2770ce1b0f5d28db565968582cbb4d047ac Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 05:47:30 +0800 Subject: [PATCH 05/60] get correct windows mise --- .mise/config.toml | 2 +- Dockerfile.windows | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 9d5b575..01956c2 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -49,7 +49,7 @@ cmake = "latest" # SDK can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. "conda:m2-bash" = { version = "latest", os = ["windows"] } "conda:m2-git" = { version = "latest", os = ["windows"] } -"ubi:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } +"github:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" diff --git a/Dockerfile.windows b/Dockerfile.windows index ebfd3ff..639ec66 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -42,11 +42,15 @@ COPY --from=vcruntime ` USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Bootstrap mise from its CDN zip (curl.exe + tar.exe ship in Nano Server). -# Extract to C:\ and call mise by absolute path below, so we don't depend on -# Windows-container PATH propagation between RUN layers. (Zip layout assumed -# mise\bin\mise.exe -- one of the things to confirm via CI.) -RUN curl -fsSL -o mise.zip https://mise.jdx.dev/mise-latest-windows-x64.zip && ` +# Bootstrap mise from a *pinned* release. `mise-latest` drifts: a newer mise +# (2026.6.4) rejects this repo's `task.run_auto_install` setting, so keep +# MISE_VERSION in lockstep with the mise the repo targets. curl.exe + tar.exe +# ship in Nano Server; extract to C:\ and call mise by absolute path below, so we +# don't depend on Windows-container PATH propagation between RUN layers. (Zip +# layout is mise\bin\mise.exe.) +ARG MISE_VERSION=2026.6.2 +RUN curl -fsSL -o mise.zip ` + https://github.com/jdx/mise/releases/download/v${MISE_VERSION}/mise-v${MISE_VERSION}-windows-x64.zip && ` tar -xf mise.zip -C C:\ && ` del mise.zip @@ -82,6 +86,6 @@ RUN C:\mise\bin\mise.exe run cargo-check # and whether a gnu-host proc-macro/build-script build is clean; # * whether `ort` (ONNX Runtime, msvc-only prebuilts) breaks the gnu link -- # cargo check shouldn't link it, but a later cargo build/test will; -# * whether the conda backend (micromamba) and the ubi llvm-mingw asset-match -# resolve cleanly on Nano Server; +# * whether the conda backend (micromamba) and the github llvm-mingw asset +# match resolve cleanly on Nano Server; # * whether mise finds bash (m2-bash) for the `bash -euo pipefail` tasks. From f270b3e3e0d39b7d709db1cabef992bb095dd7a1 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 06:02:38 +0800 Subject: [PATCH 06/60] force builder to specify version --- .github/workflows/docker.yml | 16 +++++++++++++++- Dockerfile.windows | 28 ++++++++++++++++++---------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9929527..e39c068 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,9 +62,23 @@ jobs: windows: runs-on: windows-2022 timeout-minutes: 120 + env: + # The mise the repo targets. Supplied to the build explicitly (not via a + # Dockerfile ARG default) because the classic Windows builder only + # substitutes build-args passed on the command line. Bump in lockstep with + # the repo's mise; `mise-latest` drifts and breaks the config schema. + MISE_VERSION: "2026.6.2" steps: - name: Checkout uses: actions/checkout@v4 + # Both args are build-args: Windows containers use the classic builder, so + # no BuildKit secret. GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub + # limit during `mise install`; the image is throwaway and the token expires + # with the job, so its presence in `docker history` is moot. - name: Build the Windows setup image (sketch) - run: docker build -f Dockerfile.windows -t edge-toolkit-windows . + run: >- + docker build -f Dockerfile.windows + --build-arg MISE_VERSION=${{ env.MISE_VERSION }} + --build-arg GITHUB_TOKEN=${{ github.token }} + -t edge-toolkit-windows . diff --git a/Dockerfile.windows b/Dockerfile.windows index 639ec66..a6634df 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -42,13 +42,16 @@ COPY --from=vcruntime ` USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Bootstrap mise from a *pinned* release. `mise-latest` drifts: a newer mise -# (2026.6.4) rejects this repo's `task.run_auto_install` setting, so keep -# MISE_VERSION in lockstep with the mise the repo targets. curl.exe + tar.exe -# ship in Nano Server; extract to C:\ and call mise by absolute path below, so we -# don't depend on Windows-container PATH propagation between RUN layers. (Zip -# layout is mise\bin\mise.exe.) -ARG MISE_VERSION=2026.6.2 +# Bootstrap mise from a *pinned* release; the version is supplied by the build +# (docker.yml passes --build-arg MISE_VERSION) so there's one source of truth. +# `mise-latest` drifts -- a newer mise (2026.6.4) rejected this repo's +# `task.run_auto_install` setting -- so the workflow keeps MISE_VERSION in +# lockstep with the mise the repo targets. No ARG default: the classic Windows +# builder only substitutes build-args passed explicitly, not defaults, so an +# unset value fails loudly here rather than silently. curl.exe + tar.exe ship in +# Nano Server; extract to C:\ and call mise by absolute path below. (Zip is +# mise\bin\mise.exe.) +ARG MISE_VERSION RUN curl -fsSL -o mise.zip ` https://github.com/jdx/mise/releases/download/v${MISE_VERSION}/mise-v${MISE_VERSION}-windows-x64.zip && ` tar -xf mise.zip -C C:\ && ` @@ -58,9 +61,14 @@ WORKDIR C:\workspace COPY . . # README "mise" setup. `mise install` pulls the whole toolchain declared in -# .mise/config.toml; experimental is on for the conda/ubi backends, and trust is -# needed for the copied-in config. -RUN C:\mise\bin\mise.exe trust && ` +# .mise/config.toml; experimental is on for the conda/github backends, and trust +# is needed for the copied-in config. GITHUB_TOKEN (build-arg, set for this RUN +# only) lifts mise's 60-req/hr anonymous GitHub limit; the classic Windows +# builder has no BuildKit secret, so it's a build-arg -- fine for a throwaway +# image built with an ephemeral token. +ARG GITHUB_TOKEN +RUN set "GITHUB_TOKEN=${GITHUB_TOKEN}" && ` + C:\mise\bin\mise.exe trust && ` C:\mise\bin\mise.exe settings experimental=true && ` C:\mise\bin\mise.exe settings set cargo.binstall true && ` C:\mise\bin\mise.exe install From 83d464520803e3ce186923ec42fcacbd78124884 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 06:31:45 +0800 Subject: [PATCH 07/60] more windows --- .github/workflows/docker.yml | 13 +++++++++++-- .mise/config.toml | 2 ++ Dockerfile.windows | 24 ++++++++++++++---------- README.md | 4 ++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e39c068..1938041 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,13 +65,22 @@ jobs: env: # The mise the repo targets. Supplied to the build explicitly (not via a # Dockerfile ARG default) because the classic Windows builder only - # substitutes build-args passed on the command line. Bump in lockstep with - # the repo's mise; `mise-latest` drifts and breaks the config schema. + # substitutes build-args passed on the command line. (mise's prebuilt + # windows-x64 "latest" zip is stale at 2026.3.0 and can't read this repo's + # config -- hence a pinned recent release fetched from GitHub.) MISE_VERSION: "2026.6.2" steps: - name: Checkout uses: actions/checkout@v4 + # Hosted Windows runners don't reliably leave the Docker daemon running, so + # the build can fail connecting to the docker_engine pipe. Start it (no-op + # if already running) and confirm connectivity before building. + - name: Start the Docker daemon + run: | + if ((Get-Service docker).Status -ne 'Running') { Start-Service docker } + docker version + # Both args are build-args: Windows containers use the classic builder, so # no BuildKit secret. GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub # limit during `mise install`; the image is throwaway and the token expires diff --git a/.mise/config.toml b/.mise/config.toml index 01956c2..cf76658 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -16,6 +16,8 @@ # (`mise install` / `mise run install-all`, and CI installs up front). This # keeps cheap tasks like `print-all-langs`, and the pre-install `setup-aube` # step, from eagerly pulling the whole toolchain — on CI and the CLI alike. +# (Current mise accepts this nested form; very old builds like 2026.3 instead +# used the flat `task_run_auto_install`.) task.run_auto_install = false [tools] diff --git a/Dockerfile.windows b/Dockerfile.windows index a6634df..48a2e3c 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -1,4 +1,8 @@ # escape=` +# Parser directive (must be line 1): sets the line-continuation / escape +# character to a backtick instead of the default `\`, so backslashes in Windows +# paths (C:\..., trailing `C:\`) stay literal and lines continue with a backtick. + # SKETCH (UNVERIFIED): build edge-toolkit on a *bare* Windows box where mise # supplies the entire toolchain -- the point is to prove the README's mise setup # needs nothing hand-installed but mise itself. Windows containers build only on @@ -37,19 +41,19 @@ COPY --from=vcruntime ` C:\Windows\System32\ # cmd is the only shell Nano Server has; ContainerAdministrator so we can write -# under C:\. `escape=`` ` `` (line 1) makes backtick the line continuation so the -# `\` in Windows paths stays literal. +# under C:\ (the backtick line continuations below rely on the escape directive +# documented at the top of this file). USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Bootstrap mise from a *pinned* release; the version is supplied by the build -# (docker.yml passes --build-arg MISE_VERSION) so there's one source of truth. -# `mise-latest` drifts -- a newer mise (2026.6.4) rejected this repo's -# `task.run_auto_install` setting -- so the workflow keeps MISE_VERSION in -# lockstep with the mise the repo targets. No ARG default: the classic Windows -# builder only substitutes build-args passed explicitly, not defaults, so an -# unset value fails loudly here rather than silently. curl.exe + tar.exe ship in -# Nano Server; extract to C:\ and call mise by absolute path below. (Zip is +# Bootstrap mise from a *pinned* GitHub release supplied by the build (docker.yml +# passes --build-arg MISE_VERSION), for one source of truth. We avoid mise's +# `mise-latest-windows-x64.zip`: that prebuilt is stale (ships 2026.3.0, which +# predates the nested `task.run_auto_install` this repo's config uses and errors +# on it); a recent release reads the config fine. No ARG default: the classic +# Windows builder only substitutes build-args passed explicitly, not defaults, so +# an unset value fails loudly here rather than silently. curl.exe + tar.exe ship +# in Nano Server; extract to C:\ and call mise by absolute path below. (Zip is # mise\bin\mise.exe.) ARG MISE_VERSION RUN curl -fsSL -o mise.zip ` diff --git a/README.md b/README.md index 0708bea..36926c9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ mise use -g cargo-binstall ### Windows only +If `mise install` fails on Windows with `unknown field run_auto_install`, your +mise is outdated (e.g. 2026.3) — update to a current mise, which reads the +config fine. + On Windows only, `pipx` also needs to be pre-installed. See the Windows section of [pipx instructions](https://pipx.pypa.io/stable/how-to/install-pipx/). From 0d4e67a3c1eadf519994538966c2f6de28a9294f Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 06:52:05 +0800 Subject: [PATCH 08/60] avoid build args on Windows --- .github/workflows/docker.yml | 31 ++++++++++--------- Dockerfile | 52 +++++++++++++++---------------- Dockerfile.windows | 60 +++++++++++++++++------------------- README.md | 5 +-- 4 files changed, 73 insertions(+), 75 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1938041..3900006 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -58,16 +58,15 @@ jobs: # Build the Windows setup sketch. Windows containers can only be built on a # Windows Docker host, so CI is the only place this is exercised (we can't # build it on the Linux dev/CI boxes). Expect to iterate here -- see - # Dockerfile.windows and the README for the known unknowns. + # Dockerfile.windows for the known unknowns. windows: runs-on: windows-2022 timeout-minutes: 120 env: - # The mise the repo targets. Supplied to the build explicitly (not via a - # Dockerfile ARG default) because the classic Windows builder only - # substitutes build-args passed on the command line. (mise's prebuilt - # windows-x64 "latest" zip is stale at 2026.3.0 and can't read this repo's - # config -- hence a pinned recent release fetched from GitHub.) + # The mise version to install. The classic Windows builder can't substitute + # build-args into the Dockerfile's RUN, and mise's prebuilt "latest" zip is + # stale (2026.3.0, too old for the config), so the job stages this pinned + # release in the build context and Dockerfile.windows copies it in. MISE_VERSION: "2026.6.2" steps: - name: Checkout @@ -81,13 +80,15 @@ jobs: if ((Get-Service docker).Status -ne 'Running') { Start-Service docker } docker version - # Both args are build-args: Windows containers use the classic builder, so - # no BuildKit secret. GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub - # limit during `mise install`; the image is throwaway and the token expires - # with the job, so its presence in `docker history` is moot. + # Provide mise + the GitHub token through the build context (the classic + # Windows builder can't substitute build-args into RUN). The token lifts + # mise's 60-req/hr anonymous limit; it lands in the throwaway image and + # expires with the job. + - name: Stage mise and the token in the build context + run: | + $v = "${{ env.MISE_VERSION }}" + curl.exe -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" + Set-Content -Path gh_token -Value "${{ github.token }}" -NoNewline + - name: Build the Windows setup image (sketch) - run: >- - docker build -f Dockerfile.windows - --build-arg MISE_VERSION=${{ env.MISE_VERSION }} - --build-arg GITHUB_TOKEN=${{ github.token }} - -t edge-toolkit-windows . + run: docker build -f Dockerfile.windows -t edge-toolkit-windows . diff --git a/Dockerfile b/Dockerfile index 4b788df..71c3ec3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -# Verify the README's mise setup end-to-end from a clean, minimal Ubuntu, split -# into stages so each can be cached and targeted independently: +# Build, test, and serve edge-toolkit from a clean, minimal Ubuntu, split into +# stages so each can be cached and targeted independently: # -# build install mise + every toolchain (README setup only; no build/test) +# build install mise + every toolchain (toolchain setup only; no build/test) # prefetch download all dependencies + ONNX models # precompile build the WASM/JS modules (drops target/ to stay slim) # test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) @@ -10,20 +10,20 @@ # Each stage `FROM`s the previous, so installed tools, downloaded deps, and the # built module pkg/ carry forward. Stop early with `--target`. # -# It follows README.md's "mise" setup verbatim (install mise -> configure -> +# The build stage runs the mise setup verbatim (install mise -> configure -> # install conda:openssl -> install-all). The OpenObserve (`o2`/`open-o2`) and -# `ws-server` steps from the README's "Run ws agent" section are intentionally -# skipped -- they are runtime services, not build/test verification. +# `ws-server` runtime services are intentionally skipped -- they aren't +# build/test steps. # -# The goal is to catch README drift: if a documented step is missing or wrong, -# it fails. Anything language-specific must come from mise (per the README's -# "installed into the local workspace" promise); the only apt packages below are -# universal build prereqs a normal dev machine has, so a failure that needs -# another system lib is itself a finding to fold back into the README. +# It also catches setup drift: a missing or wrong step fails the build. Anything +# language-specific must come from mise (the "installed into the local +# workspace" promise); the only apt packages below are universal build prereqs a +# normal dev machine has, so a failure that needs another system lib is itself a +# finding worth documenting. # # A plain build produces the SERVER image (final stage): a release et-ws-server, # served automatically. A GitHub token avoids mise's 60-req/hr anonymous limit -# (README "GitHub rate limits"): +# during install-all: # DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . # docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 # (drop --secret to build tokenless; install-all may then hit rate limits) @@ -37,7 +37,7 @@ # NVIDIA via `--gpus all` is wired but UNVERIFIED (its in-container Vulkan ICD # doesn't initialize yet) -- prefer a DRI device. -# --- build: install mise + every language toolchain (README "mise" setup). --- +# --- build: install mise + every language toolchain (the mise setup). --- # No prefetch/build/test, so this layer is reused until the setup itself changes. FROM ubuntu:24.04 AS build @@ -50,9 +50,9 @@ FROM ubuntu:24.04 AS build # Without it the dotnet CLI FailFast-aborts at startup with # "Couldn't find a valid ICU package installed on the system" # (minimal Ubuntu ships no ICU). The "74" tracks the Ubuntu -# base (74 = 24.04) -- bump it alongside the FROM line. README -# gap: the README's setup should note that .NET needs libicu on -# minimal systems (or set System.Globalization.Invariant=true). +# base (74 = 24.04) -- bump it alongside the FROM line. (.NET +# needs libicu on minimal systems, else set +# System.Globalization.Invariant=true.) # libvulkan1 : Vulkan loader for the wgpu compute test (wasi-graphics-info). # Just the loader -- no software driver (lavapipe): the GPU's # real ICD is injected at `docker run` time by the NVIDIA @@ -64,14 +64,14 @@ RUN apt-get update \ libvulkan1 \ && rm -rf /var/lib/apt/lists/* -# README: "Please install mise, including the shell integration." In a -# non-interactive build, putting mise + its shims on PATH is the equivalent -- -# every `mise` / `mise run` below then resolves the workspace tools. +# Install mise and put it + its shims on PATH; in a non-interactive build that's +# the equivalent of the shell integration -- every `mise` / `mise run` below +# then resolves the workspace tools. RUN curl -fsSL https://mise.run | sh ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" -# README "Configure it" + "Pre-install cargo-install". A GitHub token (if -# provided) lifts the anonymous rate limit for the cargo-binstall release fetch. +# Configure mise + pre-install cargo-binstall. A GitHub token (if provided) +# lifts the anonymous rate limit for the cargo-binstall release fetch. RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ sh -c 'mise settings experimental=true \ @@ -82,13 +82,13 @@ WORKDIR /workspace COPY . . # A fresh checkout's config is untrusted and mise would prompt interactively -# (which a build can't answer). The README omits this because a dev's first -# `mise` command prompts once; trust it up front instead. +# (which a build can't answer) -- a dev just answers that prompt once, so trust +# it up front here instead. RUN mise trust -# README "All OS": openssl dev files, then every language toolchain. `mise -# install node` first per the README's note that mise may need node to install -# other tools. `install-all` == `MISE_ENV="$ALL_LANGS" mise install`. +# openssl dev files, then every language toolchain. `mise install node` first +# because mise may need node to install other tools. `install-all` == +# `MISE_ENV="$ALL_LANGS" mise install`. RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ sh -c 'mise install node \ diff --git a/Dockerfile.windows b/Dockerfile.windows index 48a2e3c..f8d04c6 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -4,20 +4,20 @@ # paths (C:\..., trailing `C:\`) stay literal and lines continue with a backtick. # SKETCH (UNVERIFIED): build edge-toolkit on a *bare* Windows box where mise -# supplies the entire toolchain -- the point is to prove the README's mise setup -# needs nothing hand-installed but mise itself. Windows containers build only on -# a Windows Docker host, so this runs solely in the `windows` job of -# .github/workflows/docker.yml; it can't be built on the Linux dev/CI machines. -# Treat it as a CI-iterated starting point, not a known-good image. +# supplies the entire toolchain -- the point is that nothing is hand-installed +# but mise itself. Windows containers build only on a Windows Docker host, so +# this runs solely in the `windows` job of .github/workflows/docker.yml; it +# can't be built on the Linux dev/CI machines. Treat it as a CI-iterated +# starting point, not a known-good image. # # Base: Nano Server, the smallest Windows base (~120 MB vs Server Core's ~1.25 # GB). It has no MSI/installer stack, no PowerShell, and runs unprivileged -- so # the Visual Studio Build Tools installer can't run here at all. That's the -# point: nothing is installed the Windows way. We bootstrap mise (a single .exe) -# with the curl.exe + tar.exe built into Nano Server, and `mise install` pulls -# the rest -- rust, llvm-mingw (clang/lld/mingw-w64 + libclang), bash and git -# (see the os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows -# SDK can't come from mise, so the build targets x86_64-pc-windows-gnu. +# point: nothing is installed the Windows way. mise (a single .exe the workflow +# stages in the build context) is copied in, and `mise install` pulls the rest -- +# rust, llvm-mingw (clang/lld/mingw-w64 + libclang), bash and git (see the +# os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows SDK +# can't come from mise, so the build targets x86_64-pc-windows-gnu. # Donor stage: Nano Server omits the VC++ runtime that msvc-built executables # (mise.exe, and the rust/cargo mise installs) need just to *start* -- without it @@ -25,7 +25,7 @@ # (not even Server Core), and mise can't install it (mise needs it to run), so we # install the official VC++ redistributable on a Server Core donor -- which has # the installer stack Nano Server lacks -- and copy the runtime DLLs forward. -# This plus mise.exe are the only non-mise bootstrap bits; the rest is mise. +# This plus mise are the only non-mise bootstrap bits; the rest is mise. FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS vcruntime SHELL ["cmd", "/S", "/C"] # `|| ver>nul` swallows the redistributable's 3010 ("reboot required") exit code. @@ -46,32 +46,28 @@ COPY --from=vcruntime ` USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Bootstrap mise from a *pinned* GitHub release supplied by the build (docker.yml -# passes --build-arg MISE_VERSION), for one source of truth. We avoid mise's -# `mise-latest-windows-x64.zip`: that prebuilt is stale (ships 2026.3.0, which -# predates the nested `task.run_auto_install` this repo's config uses and errors -# on it); a recent release reads the config fine. No ARG default: the classic -# Windows builder only substitutes build-args passed explicitly, not defaults, so -# an unset value fails loudly here rather than silently. curl.exe + tar.exe ship -# in Nano Server; extract to C:\ and call mise by absolute path below. (Zip is -# mise\bin\mise.exe.) -ARG MISE_VERSION -RUN curl -fsSL -o mise.zip ` - https://github.com/jdx/mise/releases/download/v${MISE_VERSION}/mise-v${MISE_VERSION}-windows-x64.zip && ` - tar -xf mise.zip -C C:\ && ` - del mise.zip +# mise.zip is staged in the build context by the windows job in +# .github/workflows/docker.yml (which pins the version) and copied in here. We +# don't curl a versioned URL inside the build: the classic Windows builder +# doesn't substitute build-args into RUN, and mise's `mise-latest-windows-x64.zip` +# prebuilt is stale (2026.3.0, which predates the nested `task.run_auto_install` +# in .mise/config.toml and errors on it). Extract with Nano Server's tar.exe and +# call mise by absolute path below. (Zip is mise\bin\mise.exe.) +COPY mise.zip C:\mise.zip +RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip WORKDIR C:\workspace COPY . . -# README "mise" setup. `mise install` pulls the whole toolchain declared in +# mise setup. `mise install` pulls the whole toolchain declared in # .mise/config.toml; experimental is on for the conda/github backends, and trust -# is needed for the copied-in config. GITHUB_TOKEN (build-arg, set for this RUN -# only) lifts mise's 60-req/hr anonymous GitHub limit; the classic Windows -# builder has no BuildKit secret, so it's a build-arg -- fine for a throwaway -# image built with an ephemeral token. -ARG GITHUB_TOKEN -RUN set "GITHUB_TOKEN=${GITHUB_TOKEN}" && ` +# is needed for the copied-in config. The windows job also stages the GitHub +# token as gh_token in the build context (lifting mise's 60-req/hr anonymous +# limit); we read it here rather than via a build-arg, which the classic Windows +# builder can't substitute into RUN. The token lands in this throwaway image and +# expires with the job. +COPY gh_token C:\gh_token +RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 07:23:06 +0800 Subject: [PATCH 09/60] fix windows and macos --- .mise/config.toml | 14 +++++++++++--- Dockerfile.windows | 29 ++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index cf76658..a8b6a24 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -99,6 +99,13 @@ clang_sysroot = "{{ vars.conda_clangxx }}/{{ vars.conda_arch }}-conda-linux-gnu/ # leaves the builtin type macros unset and fails in CI) + `--sysroot` so header # resolution is identical on every machine. bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_sysroot }}" +# macOS libclang dir: point clang-sys at Xcode's libclang (the lib dir beside +# `xcrun`'s clang); clang-sys doesn't always auto-find it on CI. Guarded so the +# exec runs only on macOS; the system libclang knows the SDK, no bindgen args. +mac_libclang = "{% if os() == 'macos' %}{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib{% endif %}" +# LIBCLANG_PATH per OS: Linux -> conda libclang; macOS -> Xcode libclang above; +# Windows -> empty (clang-sys finds llvm-mingw's libclang on PATH). +libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{{ vars.mac_libclang }}{% endif %}" # mise-managed CPython 3.13 interpreter, composed into PYO3_PYTHON below. Split # per-OS because the mise data dir differs (LOCALAPPDATA\mise on Windows, # HOME-based elsewhere). `get_env(..., default='')` guards the Windows-only @@ -164,9 +171,10 @@ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars # `bindgen_args`). Using conda's sysroot rather than the host's `/usr/include` # makes bindgen self-contained, so CI resolves headers identically to a local box # -- no environment-specific divergence. Same conda-managed, no-admin approach as -# OPENSSL_DIR; Linux-only (macOS/Windows use system/Xcode libclang). mise can't -# omit an env key, so off-Linux these render empty -> clang-sys ignores them. -LIBCLANG_PATH = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% endif %}" +# OPENSSL_DIR. macOS points LIBCLANG_PATH at Xcode's libclang (see mac_libclang); +# Windows leaves it empty and finds llvm-mingw's libclang on PATH. BINDGEN args +# stay Linux-only -- the macOS/Windows libclang each know their own headers. +LIBCLANG_PATH = "{{ vars.libclang_path }}" BINDGEN_EXTRA_CLANG_ARGS = "{% if os() == 'linux' %}{{ vars.bindgen_args }}{% endif %}" # pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from # PYO3_PYTHON, else the first python on PATH -- under mise that's the 32-bit diff --git a/Dockerfile.windows b/Dockerfile.windows index f8d04c6..7b3d816 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -46,6 +46,20 @@ COPY --from=vcruntime ` USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] +# Nano Server sets no HOME, so mise resolves its global config to an untrusted +# drive-root path (C:\.config\mise) -- which breaks the `mise settings` steps +# below. Point HOME at the admin profile (as a real Windows box has) so the +# global config lands in a normal, auto-trusted location. +ENV HOME=C:\Users\ContainerAdministrator + +# Fail fast if that HOME isn't the running user's real profile: it must exist and +# end with the current %USERNAME% (so mise's global config lands in a writable, +# user-owned dir). USERPROFILE is echoed for diagnostics. +RUN echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%] USERNAME=[%USERNAME%] & ` + if not exist "%HOME%\" exit /b 1 & ` + echo %HOME%| findstr /i /e /c:"%USERNAME%" >nul & ` + if errorlevel 1 exit /b 1 + # mise.zip is staged in the build context by the windows job in # .github/workflows/docker.yml (which pins the version) and copied in here. We # don't curl a versioned URL inside the build: the classic Windows builder @@ -59,18 +73,19 @@ RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip WORKDIR C:\workspace COPY . . -# mise setup. `mise install` pulls the whole toolchain declared in -# .mise/config.toml; experimental is on for the conda/github backends, and trust -# is needed for the copied-in config. The windows job also stages the GitHub -# token as gh_token in the build context (lifting mise's 60-req/hr anonymous -# limit); we read it here rather than via a build-arg, which the classic Windows -# builder can't substitute into RUN. The token lands in this throwaway image and -# expires with the job. +# mise setup: trust the copied-in workspace config, apply the standard settings +# (HOME is set above so `mise settings` writes a sane global config), explicitly +# trust that global config, then install. The windows job stages the GitHub token +# as gh_token in the build context (lifting mise's 60-req/hr anonymous limit); we +# read it here rather than via a build-arg, which the classic Windows builder +# can't substitute into RUN. The token lands in this throwaway image and expires +# with the job. COPY gh_token C:\gh_token RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 07:28:52 +0800 Subject: [PATCH 10/60] windows ci fix --- .mise/config.toml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index a8b6a24..738a155 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -103,9 +103,13 @@ bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_ # `xcrun`'s clang); clang-sys doesn't always auto-find it on CI. Guarded so the # exec runs only on macOS; the system libclang knows the SDK, no bindgen args. mac_libclang = "{% if os() == 'macos' %}{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib{% endif %}" -# LIBCLANG_PATH per OS: Linux -> conda libclang; macOS -> Xcode libclang above; -# Windows -> empty (clang-sys finds llvm-mingw's libclang on PATH). -libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{{ vars.mac_libclang }}{% endif %}" +# Windows libclang: the runner's system LLVM. (Docker's gnu build has no system +# LLVM but finds llvm-mingw's libclang on PATH, which clang-sys also searches.) +win_libclang = "C:\\Program Files\\LLVM\\bin" +nonlin_libclang = "{% if os() == 'macos' %}{{ vars.mac_libclang }}{% else %}{{ vars.win_libclang }}{% endif %}" +# LIBCLANG_PATH per OS: Linux -> conda libclang; macOS -> Xcode libclang; Windows +# -> system LLVM (clang-sys also searches PATH, so llvm-mingw works too). +libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{{ vars.nonlin_libclang }}{% endif %}" # mise-managed CPython 3.13 interpreter, composed into PYO3_PYTHON below. Split # per-OS because the mise data dir differs (LOCALAPPDATA\mise on Windows, # HOME-based elsewhere). `get_env(..., default='')` guards the Windows-only @@ -172,8 +176,9 @@ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars # makes bindgen self-contained, so CI resolves headers identically to a local box # -- no environment-specific divergence. Same conda-managed, no-admin approach as # OPENSSL_DIR. macOS points LIBCLANG_PATH at Xcode's libclang (see mac_libclang); -# Windows leaves it empty and finds llvm-mingw's libclang on PATH. BINDGEN args -# stay Linux-only -- the macOS/Windows libclang each know their own headers. +# Windows uses the runner's system LLVM (Docker's gnu build falls back to +# llvm-mingw on PATH). BINDGEN args stay Linux-only -- each libclang finds its +# own headers. LIBCLANG_PATH = "{{ vars.libclang_path }}" BINDGEN_EXTRA_CLANG_ARGS = "{% if os() == 'linux' %}{{ vars.bindgen_args }}{% endif %}" # pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from From 46d851b4a2daa0ea558ff5b21c3c0498c665e959 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 07:52:40 +0800 Subject: [PATCH 11/60] docker fixes --- .github/workflows/docker.yml | 22 +------------------ Dockerfile | 42 +++++++++++++++--------------------- Dockerfile.windows | 30 +++++++++++++------------- README.md | 3 +-- 4 files changed, 34 insertions(+), 63 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3900006..7452fcf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,17 +1,12 @@ --- name: docker -# Build the verification images. Only fires when a Dockerfile (or the build -# context's .dockerignore / this workflow) changes, or on manual dispatch -- -# these builds are heavy (every toolchain + a release compile), so they don't -# run on every push. "on": pull_request: paths: + - .github/workflows/docker.yml - Dockerfile - Dockerfile.windows - - .dockerignore - - .github/workflows/docker.yml workflow_dispatch: permissions: @@ -22,9 +17,6 @@ concurrency: cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: - # Build the Linux `test` stage and run the suite. GitHub runners have no GPU, - # so the stage's bundled mesa drivers fall back to lavapipe (software Vulkan) - # for the WebGPU compute test. linux: runs-on: ubuntu-latest timeout-minutes: 120 @@ -43,22 +35,14 @@ jobs: sudo systemctl start docker docker info --format 'Docker Root Dir: {{.DockerRootDir}}' - # GITHUB_TOKEN lifts mise's 60-req/hr anonymous GitHub limit during - # install-all; passed as a BuildKit secret so it never lands in a layer. - name: Build the edge-toolkit test image env: GITHUB_TOKEN: ${{ github.token }} run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . - # The `test` stage's CMD is `mise run test`; running the image runs the - # suite. No --device: with no GPU, mesa falls back to lavapipe. - name: Run the test suite run: docker run --rm edge-toolkit-test - # Build the Windows setup sketch. Windows containers can only be built on a - # Windows Docker host, so CI is the only place this is exercised (we can't - # build it on the Linux dev/CI boxes). Expect to iterate here -- see - # Dockerfile.windows for the known unknowns. windows: runs-on: windows-2022 timeout-minutes: 120 @@ -80,10 +64,6 @@ jobs: if ((Get-Service docker).Status -ne 'Running') { Start-Service docker } docker version - # Provide mise + the GitHub token through the build context (the classic - # Windows builder can't substitute build-args into RUN). The token lifts - # mise's 60-req/hr anonymous limit; it lands in the throwaway image and - # expires with the job. - name: Stage mise and the token in the build context run: | $v = "${{ env.MISE_VERSION }}" diff --git a/Dockerfile b/Dockerfile index 71c3ec3..c7ad339 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,15 +53,11 @@ FROM ubuntu:24.04 AS build # base (74 = 24.04) -- bump it alongside the FROM line. (.NET # needs libicu on minimal systems, else set # System.Globalization.Invariant=true.) -# libvulkan1 : Vulkan loader for the wgpu compute test (wasi-graphics-info). -# Just the loader -- no software driver (lavapipe): the GPU's -# real ICD is injected at `docker run` time by the NVIDIA -# Container Toolkit (`--gpus all`), so the test runs on actual -# hardware. (AMD/Intel: add mesa-vulkan-drivers + --device /dev/dri.) +# (Vulkan for the wgpu test -- libvulkan1 + mesa-vulkan-drivers -- is installed in +# the test stage, not here.) RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential ca-certificates curl git xz-utils unzip bzip2 libicu74 \ - libvulkan1 \ && rm -rf /var/lib/apt/lists/* # Install mise and put it + its shims on PATH; in a non-interactive build that's @@ -79,28 +75,23 @@ RUN --mount=type=secret,id=gh_token,required=false \ && mise use -g cargo-binstall' WORKDIR /workspace -COPY . . +# Only the mise config is needed to install the toolchain. +COPY .mise/ .mise/ -# A fresh checkout's config is untrusted and mise would prompt interactively -# (which a build can't answer) -- a dev just answers that prompt once, so trust -# it up front here instead. RUN mise trust -# openssl dev files, then every language toolchain. `mise install node` first -# because mise may need node to install other tools. `install-all` == -# `MISE_ENV="$ALL_LANGS" mise install`. +ENV MISE_ENV="dart,dotnet,java,python,rust,zig" + RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ sh -c 'mise install node \ && mise install conda:openssl \ && mise install-all' -# Mirrors what the *-all tasks set internally, so the bare task names in the -# stages below act on all languages, not just Rust. Inherited by every stage. -ENV MISE_ENV="dart,dotnet,java,python,rust,zig" - # --- prefetch: download all dependencies + ONNX models. --- +# The full source is needed from here on (module builds, cargo fetch, pnpm). FROM build AS prefetch +COPY . . RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ mise run prefetch @@ -111,16 +102,17 @@ RUN --mount=type=secret,id=gh_token,required=false \ # Dropping target/ here keeps it out of this layer and the stages built on it; # test and server recompile only what they need. FROM prefetch AS precompile -RUN mise run build-modules && rm -rf target +RUN mise run build-modules && rm -rf target/ # --- test: the full suite (Rust + web runner + every guest language). --- # Compiled AND run at `docker run` time (precompile keeps no target/), so the # multi-GB debug test binaries never bake into a layer -- they live in the -# ephemeral container and vanish when it exits. The wgpu compute test needs a -# Vulkan device; mesa-vulkan-drivers provides one -- a real Intel/AMD GPU when the -# host DRI node is passed with `--device /dev/dri`, else a CPU (lavapipe) fallback -# so the suite still runs. (NVIDIA's `--gpus` Vulkan path doesn't initialize in a -# container.) Installed here, not the build stage, to keep that layer cached. +# ephemeral container and vanish when it exits. The wgpu compute test needs +# Vulkan: libvulkan1 (the loader) + mesa-vulkan-drivers give a real Intel/AMD GPU +# when the host DRI node is passed with `--device /dev/dri`, else a CPU (lavapipe) +# fallback so the suite still runs. (NVIDIA's `--gpus` Vulkan path doesn't +# initialize in a container.) Both live here, not the build stage, to keep that +# layer cached. # docker build --target test -t edge-toolkit-test . # docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) # NVIDIA via `--gpus all` is wired (NVIDIA_DRIVER_CAPABILITIES=all below, needs @@ -129,7 +121,7 @@ RUN mise run build-modules && rm -rf target FROM precompile AS test ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=all RUN apt-get update \ - && apt-get install -y --no-install-recommends mesa-vulkan-drivers \ + && apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers \ && rm -rf /var/lib/apt/lists/* CMD ["mise", "run", "test"] @@ -143,6 +135,6 @@ CMD ["mise", "run", "test"] FROM precompile AS server RUN mise exec -- cargo build --release -p et-ws-server \ && cp target/release/et-ws-server /usr/local/bin/et-ws-server \ - && rm -rf target + && rm -rf target/ EXPOSE 8080 8443 CMD ["et-ws-server"] diff --git a/Dockerfile.windows b/Dockerfile.windows index 7b3d816..676aa8d 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -46,15 +46,14 @@ COPY --from=vcruntime ` USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Nano Server sets no HOME, so mise resolves its global config to an untrusted -# drive-root path (C:\.config\mise) -- which breaks the `mise settings` steps -# below. Point HOME at the admin profile (as a real Windows box has) so the -# global config lands in a normal, auto-trusted location. +# Set HOME to the admin profile so the container has a sane home (Nano Server +# leaves it unset). This also renders the config's `{{ env.HOME }}` paths; note +# mise still resolves its *global config* to C:\.config\mise on Windows +# regardless of HOME (trusted explicitly below). ENV HOME=C:\Users\ContainerAdministrator -# Fail fast if that HOME isn't the running user's real profile: it must exist and -# end with the current %USERNAME% (so mise's global config lands in a writable, -# user-owned dir). USERPROFILE is echoed for diagnostics. +# Sanity-check that HOME is the running user's real profile: it must exist and +# end with the current %USERNAME%. USERPROFILE is echoed for diagnostics. RUN echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%] USERNAME=[%USERNAME%] & ` if not exist "%HOME%\" exit /b 1 & ` echo %HOME%| findstr /i /e /c:"%USERNAME%" >nul & ` @@ -73,19 +72,20 @@ RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip WORKDIR C:\workspace COPY . . -# mise setup: trust the copied-in workspace config, apply the standard settings -# (HOME is set above so `mise settings` writes a sane global config), explicitly -# trust that global config, then install. The windows job stages the GitHub token -# as gh_token in the build context (lifting mise's 60-req/hr anonymous limit); we -# read it here rather than via a build-arg, which the classic Windows builder -# can't substitute into RUN. The token lands in this throwaway image and expires -# with the job. +# mise setup: trust the copied-in workspace config, apply the standard settings, +# then install. `mise settings` writes mise's global config; with no real home on +# Windows that resolves to C:\.config\mise\config.toml, which isn't auto-trusted, +# so trust it explicitly before `mise install` loads it. The windows job stages +# the GitHub token as gh_token in the build context (lifting mise's 60-req/hr +# anonymous limit); we read it here rather than via a build-arg, which the classic +# Windows builder can't substitute into RUN. The token lands in this throwaway +# image and expires with the job. COPY gh_token C:\gh_token RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 08:20:22 +0800 Subject: [PATCH 12/60] docker fixes --- .github/workflows/docker.yml | 12 ++++----- .mise/config.toml | 12 +++++---- Dockerfile | 8 ++++-- Dockerfile.windows | 49 +++++++++++++++++++++++------------- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7452fcf..08e61b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,9 +24,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # The image is large (every language toolchain + prefetched models + a - # release build); the runner's root disk (~14 GB free) can't hold it. Move - # Docker's storage to the ~70 GB /mnt scratch disk before building. + # The image is large (every language toolchain + prefetched models. + # The runner's root disk (~14 GB free) can't hold it. + # Move Docker's storage to the ~70 GB /mnt scratch disk. - name: Move Docker storage to /mnt run: | sudo systemctl stop docker docker.socket @@ -47,10 +47,8 @@ jobs: runs-on: windows-2022 timeout-minutes: 120 env: - # The mise version to install. The classic Windows builder can't substitute - # build-args into the Dockerfile's RUN, and mise's prebuilt "latest" zip is - # stale (2026.3.0, too old for the config), so the job stages this pinned - # release in the build context and Dockerfile.windows copies it in. + # The classic Windows builder can't substitute build-args into the Dockerfile's RUN, + # and mise's prebuilt "latest" zip is stale (2026.3.0, too old for the config). MISE_VERSION: "2026.6.2" steps: - name: Checkout diff --git a/.mise/config.toml b/.mise/config.toml index 738a155..edce502 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -44,13 +44,15 @@ cmake = "latest" "conda:clangxx" = { version = "latest", os = ["linux"] } # Windows toolchain, all mise-provided so a bare Windows box needs nothing # preinstalled except mise itself (see Dockerfile.windows). The conda msys2 -# mirror packages give the `bash` the `bash -euo pipefail` tasks expect and a -# `git`; llvm-mingw bundles clang + lld + the mingw-w64 runtime/headers/libs -# (the `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen -# loads -- found via PATH once mise activates the tool. The MSVC CRT + Windows -# SDK can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. +# mirror packages give the `bash` the `bash -euo pipefail` tasks expect, a `git`, +# and `gpg` (so mise verifies tool downloads instead of warning); llvm-mingw +# bundles clang + lld + the mingw-w64 runtime/headers/libs (the +# `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen loads +# -- found via PATH once mise activates the tool. The MSVC CRT + Windows SDK +# can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. "conda:m2-bash" = { version = "latest", os = ["windows"] } "conda:m2-git" = { version = "latest", os = ["windows"] } +"conda:m2-gnupg" = { version = "latest", os = ["windows"] } "github:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" diff --git a/Dockerfile b/Dockerfile index c7ad339..58dbd7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,9 +42,13 @@ FROM ubuntu:24.04 AS build # Universal prereqs a typical dev box already has; everything else is mise's job. -# build-essential : cc/ld to link Rust binaries + build C deps +# gcc/g++/libc6-dev : the C/C++ compiler + headers/crt that rustc links through +# (`cc`) and that C/C++ `-sys` crates build with. (Leaner than +# build-essential, which also pulls dpkg-dev + make + perl.) # curl + ca-certs : the mise installer and tool downloads # git : cargo + repo operations +# gpg : lets mise verify tool downloads (else "gpg not found, +# skipping verification") # xz-utils/unzip/bzip2 : mise unpacking tool archives (e.g. the pyodide .tar.bz2) # libicu74 : .NET runtime ICU, for the dotnet-data1 module's build/test. # Without it the dotnet CLI FailFast-aborts at startup with @@ -57,7 +61,7 @@ FROM ubuntu:24.04 AS build # the test stage, not here.) RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential ca-certificates curl git xz-utils unzip bzip2 libicu74 \ + bzip2 ca-certificates curl g++ gcc git gpg libc6-dev libicu74 unzip xz-utils \ && rm -rf /var/lib/apt/lists/* # Install mise and put it + its shims on PATH; in a non-interactive build that's diff --git a/Dockerfile.windows b/Dockerfile.windows index 676aa8d..8c0ad8a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -64,37 +64,50 @@ RUN echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%] USERNAME=[%USERNAME%] & ` # don't curl a versioned URL inside the build: the classic Windows builder # doesn't substitute build-args into RUN, and mise's `mise-latest-windows-x64.zip` # prebuilt is stale (2026.3.0, which predates the nested `task.run_auto_install` -# in .mise/config.toml and errors on it). Extract with Nano Server's tar.exe and -# call mise by absolute path below. (Zip is mise\bin\mise.exe.) +# in .mise/config.toml and errors on it). Extract with Nano Server's tar.exe (zip +# is mise\bin\mise.exe) and put that on PATH so the steps below can call `mise`. COPY mise.zip C:\mise.zip RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip +ENV PATH="C:\mise\bin;${PATH}" WORKDIR C:\workspace COPY . . -# mise setup: trust the copied-in workspace config, apply the standard settings, -# then install. `mise settings` writes mise's global config; with no real home on -# Windows that resolves to C:\.config\mise\config.toml, which isn't auto-trusted, -# so trust it explicitly before `mise install` loads it. The windows job stages -# the GitHub token as gh_token in the build context (lifting mise's 60-req/hr -# anonymous limit); we read it here rather than via a build-arg, which the classic -# Windows builder can't substitute into RUN. The token lands in this throwaway -# image and expires with the job. +# mise setup, following the README. `mise settings` writes mise's global config; +# with no real home on Windows that resolves to C:\.config\mise\config.toml, not +# auto-trusted -- so trust it after the settings writes and before any install +# loads it. aqua.github_attestations is turned off: the attestation's sigstore +# /TUF check has no cache dir here and hits GitHub's changed releases signing +# identity, failing every aqua tool. Then the README steps: pre-install +# cargo-binstall; gpg (conda:m2-gnupg, so node's download is verified not +# skipped); node + conda:openssl; and pipx via pip (the README's Windows-only +# step -- pipx isn't a mise tool here). `mise install` pulls the rest. The windows +# job stages the GitHub token as gh_token in the build context (lifting mise's +# 60-req/hr anonymous limit); we read it here rather than via a build-arg, which +# the classic Windows builder can't substitute into RUN. It lands in this +# throwaway image and expires with the job. COPY gh_token C:\gh_token RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 08:47:52 +0800 Subject: [PATCH 13/60] improve docker files --- .mise/config.toml | 5 +++-- Dockerfile | 53 +++++++++++++++++++++++++++++----------------- Dockerfile.windows | 31 +++++++++++++++++++++------ 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index edce502..1ffa762 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -67,8 +67,9 @@ osv-scanner = "latest" # still resolve through whatever pipx is already on PATH. pipx = { version = "latest", os = ["linux", "macos"] } # semgrep lints Cargo.toml style (see `semgrep-check`), so it stays in the -# always-loaded config alongside the other repo-wide linters. -"pipx:semgrep" = "latest" +# always-loaded config alongside the other repo-wide linters. Linux/macOS only: +# semgrep has no native Windows support, so its uv/pipx install fails there. +"pipx:semgrep" = { version = "latest", os = ["linux", "macos"] } # The default aqua backend has no darwin/amd64 prebuilt; fall back to # `npm:pnpm` on Intel Mac (node is already a mise tool above). "npm:pnpm" = { version = "latest", os = ["macos/x64"] } diff --git a/Dockerfile b/Dockerfile index 58dbd7f..0b40a5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,21 @@ # Build, test, and serve edge-toolkit from a clean, minimal Ubuntu, split into # stages so each can be cached and targeted independently: # -# build install mise + every toolchain (toolchain setup only; no build/test) -# prefetch download all dependencies + ONNX models -# precompile build the WASM/JS modules (drops target/ to stay slim) -# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) -# server release build of et-ws-server, served by default (final stage) +# build-minimal mise + the always-loaded toolchain (.mise/config.toml only) +# build + the guest-language toolchains (.mise/config..toml) +# prefetch download all dependencies + ONNX models +# precompile build the WASM/JS modules (drops target/ to stay slim) +# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) +# server release build of et-ws-server, served by default (final stage) # # Each stage `FROM`s the previous, so installed tools, downloaded deps, and the # built module pkg/ carry forward. Stop early with `--target`. # -# The build stage runs the mise setup verbatim (install mise -> configure -> -# install conda:openssl -> install-all). The OpenObserve (`o2`/`open-o2`) and -# `ws-server` runtime services are intentionally skipped -- they aren't -# build/test steps. +# The build stages run the mise setup verbatim (install mise -> configure -> +# install conda:openssl -> install), split so build-minimal (the always-loaded +# tools) caches separately from the guest languages (build). The OpenObserve +# (`o2`/`open-o2`) and `ws-server` runtime services are intentionally skipped -- +# they aren't build/test steps. # # It also catches setup drift: a missing or wrong step fails the build. Anything # language-specific must come from mise (the "installed into the local @@ -37,9 +39,10 @@ # NVIDIA via `--gpus all` is wired but UNVERIFIED (its in-container Vulkan ICD # doesn't initialize yet) -- prefer a DRI device. -# --- build: install mise + every language toolchain (the mise setup). --- -# No prefetch/build/test, so this layer is reused until the setup itself changes. -FROM ubuntu:24.04 AS build +# --- build-minimal: mise + the always-loaded toolchain (config.toml only). --- +# Copies just .mise/config.toml + installs the default tools, so this layer is +# reused until the always-loaded toolset changes -- not when a guest config does. +FROM ubuntu:24.04 AS build-minimal # Universal prereqs a typical dev box already has; everything else is mise's job. # gcc/g++/libc6-dev : the C/C++ compiler + headers/crt that rustc links through @@ -47,8 +50,8 @@ FROM ubuntu:24.04 AS build # build-essential, which also pulls dpkg-dev + make + perl.) # curl + ca-certs : the mise installer and tool downloads # git : cargo + repo operations -# gpg : lets mise verify tool downloads (else "gpg not found, -# skipping verification") +# gnupg : gpg + gpg-agent + dirmngr, so mise can verify tool +# downloads (bare `gpg` lacks the agent/dirmngr it needs) # xz-utils/unzip/bzip2 : mise unpacking tool archives (e.g. the pyodide .tar.bz2) # libicu74 : .NET runtime ICU, for the dotnet-data1 module's build/test. # Without it the dotnet CLI FailFast-aborts at startup with @@ -61,7 +64,7 @@ FROM ubuntu:24.04 AS build # the test stage, not here.) RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - bzip2 ca-certificates curl g++ gcc git gpg libc6-dev libicu74 unzip xz-utils \ + bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 unzip xz-utils \ && rm -rf /var/lib/apt/lists/* # Install mise and put it + its shims on PATH; in a non-interactive build that's @@ -79,18 +82,28 @@ RUN --mount=type=secret,id=gh_token,required=false \ && mise use -g cargo-binstall' WORKDIR /workspace -# Only the mise config is needed to install the toolchain. -COPY .mise/ .mise/ +# Only the always-loaded config is needed for the default tools; the +# guest-language configs come in the build stage below. +COPY .mise/config.toml .mise/config.toml RUN mise trust -ENV MISE_ENV="dart,dotnet,java,python,rust,zig" - RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ sh -c 'mise install node \ && mise install conda:openssl \ - && mise install-all' + && mise install' + +# --- build: add the guest-language toolchains (config..toml). --- +# install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are +# already installed by build-minimal, so this adds dart/dotnet/java/zig/etc. +FROM build-minimal AS build +COPY .mise/ .mise/ +RUN mise trust +ENV MISE_ENV="dart,dotnet,java,python,rust,zig" +RUN --mount=type=secret,id=gh_token,required=false \ + GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ + mise install-all # --- prefetch: download all dependencies + ONNX models. --- # The full source is needed from here on (module builds, cargo fetch, pnpm). diff --git a/Dockerfile.windows b/Dockerfile.windows index 8c0ad8a..7c613d3 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -59,6 +59,13 @@ RUN echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%] USERNAME=[%USERNAME%] & ` echo %HOME%| findstr /i /e /c:"%USERNAME%" >nul & ` if errorlevel 1 exit /b 1 +# Nano Server's default temp dir can be missing/unwritable, which fails rustup's +# "persist temporary file" step (Access denied) installing toolchains. Point +# TEMP/TMP at a writable dir on C: (same volume as mise's installs). +RUN if not exist C:\Temp mkdir C:\Temp +ENV TEMP=C:\Temp ` + TMP=C:\Temp + # mise.zip is staged in the build context by the windows job in # .github/workflows/docker.yml (which pins the version) and copied in here. We # don't curl a versioned URL inside the build: the classic Windows builder @@ -76,12 +83,13 @@ COPY . . # mise setup, following the README. `mise settings` writes mise's global config; # with no real home on Windows that resolves to C:\.config\mise\config.toml, not # auto-trusted -- so trust it after the settings writes and before any install -# loads it. aqua.github_attestations is turned off: the attestation's sigstore -# /TUF check has no cache dir here and hits GitHub's changed releases signing -# identity, failing every aqua tool. Then the README steps: pre-install +# loads it. aqua's signature checks (attestations, cosign, slsa) are off: their +# sigstore/TUF step can't determine a cache dir here, and attestation also hits +# GitHub's changed releases signing identity. Then the README steps: pre-install # cargo-binstall; gpg (conda:m2-gnupg, so node's download is verified not # skipped); node + conda:openssl; and pipx via pip (the README's Windows-only -# step -- pipx isn't a mise tool here). `mise install` pulls the rest. The windows +# step -- pipx isn't a mise tool here). The minimal `mise install` pulls the +# always-loaded tools (install-all below adds the guest languages). The windows # job stages the GitHub token as gh_token in the build context (lifting mise's # 60-req/hr anonymous limit); we read it here rather than via a build-arg, which # the classic Windows builder can't substitute into RUN. It lands in this @@ -92,15 +100,24 @@ RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 09:12:18 +0800 Subject: [PATCH 14/60] re-add make dep --- .mise/config.toml | 19 +++++++------ Dockerfile | 9 +++--- Dockerfile.windows | 70 ++++++++++++++++++++++++---------------------- README.md | 25 ++++++++++++++--- 4 files changed, 72 insertions(+), 51 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 1ffa762..1c5d26c 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -44,15 +44,17 @@ cmake = "latest" "conda:clangxx" = { version = "latest", os = ["linux"] } # Windows toolchain, all mise-provided so a bare Windows box needs nothing # preinstalled except mise itself (see Dockerfile.windows). The conda msys2 -# mirror packages give the `bash` the `bash -euo pipefail` tasks expect, a `git`, -# and `gpg` (so mise verifies tool downloads instead of warning); llvm-mingw -# bundles clang + lld + the mingw-w64 runtime/headers/libs (the -# `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen loads -# -- found via PATH once mise activates the tool. The MSVC CRT + Windows SDK -# can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. +# mirror packages give the `bash` the `bash -euo pipefail` tasks expect, `git`, +# `gpg` (so mise verifies tool downloads instead of warning), and `make` (some +# `-sys` build scripts shell out to it); llvm-mingw bundles clang + lld + the +# mingw-w64 runtime/headers/libs (the `x86_64-pc-windows-gnu` C/link toolchain) +# plus the `libclang.dll` bindgen loads -- found via PATH once mise activates the +# tool. The MSVC CRT + Windows SDK can't come from mise, so the Windows build +# targets `-gnu`, not `-msvc`. "conda:m2-bash" = { version = "latest", os = ["windows"] } "conda:m2-git" = { version = "latest", os = ["windows"] } "conda:m2-gnupg" = { version = "latest", os = ["windows"] } +"conda:m2-make" = { version = "latest", os = ["windows"] } "github:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" @@ -67,9 +69,8 @@ osv-scanner = "latest" # still resolve through whatever pipx is already on PATH. pipx = { version = "latest", os = ["linux", "macos"] } # semgrep lints Cargo.toml style (see `semgrep-check`), so it stays in the -# always-loaded config alongside the other repo-wide linters. Linux/macOS only: -# semgrep has no native Windows support, so its uv/pipx install fails there. -"pipx:semgrep" = { version = "latest", os = ["linux", "macos"] } +# always-loaded config alongside the other repo-wide linters. +"pipx:semgrep" = "latest" # The default aqua backend has no darwin/amd64 prebuilt; fall back to # `npm:pnpm` on Intel Mac (node is already a mise tool above). "npm:pnpm" = { version = "latest", os = ["macos/x64"] } diff --git a/Dockerfile b/Dockerfile index 0b40a5e..6a43696 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,9 +45,10 @@ FROM ubuntu:24.04 AS build-minimal # Universal prereqs a typical dev box already has; everything else is mise's job. -# gcc/g++/libc6-dev : the C/C++ compiler + headers/crt that rustc links through -# (`cc`) and that C/C++ `-sys` crates build with. (Leaner than -# build-essential, which also pulls dpkg-dev + make + perl.) +# gcc/g++/libc6-dev/make : the C/C++ toolchain rustc links through (`cc`) and +# that C/C++ `-sys` crates build with (make for build scripts +# that shell out to it). Leaner than build-essential, which +# also pulls dpkg-dev + perl. # curl + ca-certs : the mise installer and tool downloads # git : cargo + repo operations # gnupg : gpg + gpg-agent + dirmngr, so mise can verify tool @@ -64,7 +65,7 @@ FROM ubuntu:24.04 AS build-minimal # the test stage, not here.) RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 unzip xz-utils \ + bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils \ && rm -rf /var/lib/apt/lists/* # Install mise and put it + its shims on PATH; in a non-interactive build that's diff --git a/Dockerfile.windows b/Dockerfile.windows index 7c613d3..90af276 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -80,20 +80,21 @@ ENV PATH="C:\mise\bin;${PATH}" WORKDIR C:\workspace COPY . . -# mise setup, following the README. `mise settings` writes mise's global config; -# with no real home on Windows that resolves to C:\.config\mise\config.toml, not -# auto-trusted -- so trust it after the settings writes and before any install -# loads it. aqua's signature checks (attestations, cosign, slsa) are off: their -# sigstore/TUF step can't determine a cache dir here, and attestation also hits -# GitHub's changed releases signing identity. Then the README steps: pre-install -# cargo-binstall; gpg (conda:m2-gnupg, so node's download is verified not -# skipped); node + conda:openssl; and pipx via pip (the README's Windows-only -# step -- pipx isn't a mise tool here). The minimal `mise install` pulls the -# always-loaded tools (install-all below adds the guest languages). The windows -# job stages the GitHub token as gh_token in the build context (lifting mise's -# 60-req/hr anonymous limit); we read it here rather than via a build-arg, which -# the classic Windows builder can't substitute into RUN. It lands in this -# throwaway image and expires with the job. +# mise setup, following the README, but with the gnu toolchain installed and +# selected UP FRONT: Nano Server has no msvc link.exe, so any cargo: tool that +# lacks a prebuilt (and falls back to `cargo install`) must link via llvm-mingw, +# not msvc. So install gpg + llvm-mingw + rust first and flip rustup to the gnu +# host, then run the bulk installs. +# +# `mise settings` writes mise's global config; with no real home on Windows that +# resolves to C:\.config\mise\config.toml, not auto-trusted -- so trust it after +# the settings writes. aqua's signature checks (attestations, cosign, slsa) are +# off: their sigstore/TUF step can't determine a cache dir here, and attestation +# also hits GitHub's changed releases signing identity. The windows job stages +# the GitHub token as gh_token in the build context (lifting mise's 60-req/hr +# anonymous limit); we read it per-RUN rather than via a build-arg, which the +# classic Windows builder can't substitute into RUN. It lands in this throwaway +# image and expires with the job. COPY gh_token C:\gh_token RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 09:39:28 +0800 Subject: [PATCH 15/60] tidy Dockerfiles --- .dprint.jsonc | 4 ++++ Dockerfile | 60 ++++++++++++++++++++++------------------------ Dockerfile.windows | 22 ++++++++--------- 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/.dprint.jsonc b/.dprint.jsonc index 790dece..5e30b14 100644 --- a/.dprint.jsonc +++ b/.dprint.jsonc @@ -1,4 +1,7 @@ { + "dockerfile": { + "associations": ["**/Dockerfile", "**/*.dockerfile", "**/Dockerfile.windows"], + }, "java": { }, "json": { @@ -28,6 +31,7 @@ "https://plugins.dprint.dev/g-plane/malva-v0.15.2.wasm", "https://plugins.dprint.dev/g-plane/markup_fmt-v0.27.0.wasm", "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", + "https://plugins.dprint.dev/dockerfile-0.4.0.wasm", "https://plugins.dprint.dev/json-0.21.3.wasm", "https://plugins.dprint.dev/markdown-0.21.1.wasm", "https://plugins.dprint.dev/ruff-0.7.10.wasm", diff --git a/Dockerfile b/Dockerfile index 6a43696..0d4b0db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ # Build, test, and serve edge-toolkit from a clean, minimal Ubuntu, split into # stages so each can be cached and targeted independently: # -# build-minimal mise + the always-loaded toolchain (.mise/config.toml only) -# build + the guest-language toolchains (.mise/config..toml) -# prefetch download all dependencies + ONNX models -# precompile build the WASM/JS modules (drops target/ to stay slim) -# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) -# server release build of et-ws-server, served by default (final stage) +# build-minimal mise + the always-loaded toolchain (.mise/config.toml only) +# build + the guest-language toolchains (.mise/config..toml) +# prefetch download all dependencies + ONNX models +# precompile build the WASM/JS modules (drops target/ to stay slim) +# test compile + run the full suite (ephemeral, at `docker run`; needs a GPU) +# server release build of et-ws-server, served by default (final stage) # # Each stage `FROM`s the previous, so installed tools, downloaded deps, and the # built module pkg/ carry forward. Stop early with `--target`. @@ -26,16 +26,16 @@ # A plain build produces the SERVER image (final stage): a release et-ws-server, # served automatically. A GitHub token avoids mise's 60-req/hr anonymous limit # during install-all: -# DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . -# docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 -# (drop --secret to build tokenless; install-all may then hit rate limits) +# DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . +# docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 +# (drop --secret to build tokenless; install-all may then hit rate limits) # # To run the verification suite, target the non-final `test` stage and pass the # host GPU (`docker build` can't attach one). The stage bundles mesa-vulkan- # drivers, so the wgpu test gets a real Intel/AMD GPU via the DRI node (or a # software fallback if none is passed): -# docker build --target test -t edge-toolkit-test . -# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) # NVIDIA via `--gpus all` is wired but UNVERIFIED (its in-container Vulkan ICD # doesn't initialize yet) -- prefer a DRI device. @@ -45,24 +45,20 @@ FROM ubuntu:24.04 AS build-minimal # Universal prereqs a typical dev box already has; everything else is mise's job. -# gcc/g++/libc6-dev/make : the C/C++ toolchain rustc links through (`cc`) and -# that C/C++ `-sys` crates build with (make for build scripts -# that shell out to it). Leaner than build-essential, which -# also pulls dpkg-dev + perl. -# curl + ca-certs : the mise installer and tool downloads -# git : cargo + repo operations -# gnupg : gpg + gpg-agent + dirmngr, so mise can verify tool -# downloads (bare `gpg` lacks the agent/dirmngr it needs) -# xz-utils/unzip/bzip2 : mise unpacking tool archives (e.g. the pyodide .tar.bz2) -# libicu74 : .NET runtime ICU, for the dotnet-data1 module's build/test. -# Without it the dotnet CLI FailFast-aborts at startup with -# "Couldn't find a valid ICU package installed on the system" -# (minimal Ubuntu ships no ICU). The "74" tracks the Ubuntu -# base (74 = 24.04) -- bump it alongside the FROM line. (.NET -# needs libicu on minimal systems, else set -# System.Globalization.Invariant=true.) -# (Vulkan for the wgpu test -- libvulkan1 + mesa-vulkan-drivers -- is installed in -# the test stage, not here.) +# gcc, g++, libc6-dev and make are the C/C++ toolchain rustc links through (`cc`) +# and that C/C++ `-sys` crates build with (make for build scripts that shell out +# to it) -- leaner than build-essential, which also pulls dpkg-dev + perl. +# curl + ca-certificates fetch the mise installer and tool downloads; git is for +# cargo + repo operations; gnupg (gpg + gpg-agent + dirmngr) lets mise verify +# downloads (bare `gpg` lacks the agent/dirmngr it needs); xz-utils, unzip and +# bzip2 unpack mise's tool archives (e.g. the pyodide .tar.bz2). libicu74 is .NET +# runtime ICU for the dotnet-data1 module -- without it the dotnet CLI +# FailFast-aborts at startup ("Couldn't find a valid ICU package installed on the +# system"; minimal Ubuntu ships no ICU). The "74" tracks the Ubuntu base +# (74 = 24.04) -- bump it alongside the FROM line; .NET needs libicu on minimal +# systems (else set System.Globalization.Invariant=true). +# Vulkan for the wgpu test (libvulkan1 + mesa-vulkan-drivers) is installed in the +# test stage, not here. RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils \ @@ -131,8 +127,8 @@ RUN mise run build-modules && rm -rf target/ # fallback so the suite still runs. (NVIDIA's `--gpus` Vulkan path doesn't # initialize in a container.) Both live here, not the build stage, to keep that # layer cached. -# docker build --target test -t edge-toolkit-test . -# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) +# docker build --target test -t edge-toolkit-test . +# docker run --rm --device /dev/dri edge-toolkit-test # Intel/AMD (verified) # NVIDIA via `--gpus all` is wired (NVIDIA_DRIVER_CAPABILITIES=all below, needs # the NVIDIA Container Toolkit) but UNVERIFIED -- its in-container Vulkan ICD # doesn't initialize yet, so prefer a DRI device for now. @@ -149,7 +145,7 @@ CMD ["mise", "run", "test"] # image; the binary finds its libs via baked rpaths and serves each module from # its pkg/ (none of which live in target/). mise stays on PATH and MISE_ENV is # set, so the server's `mise where` module-path lookups resolve. -# docker run --rm -p 8080:8080 edge-toolkit # then open http://localhost:8080 +# docker run --rm -p 8080:8080 edge-toolkit # then open http://localhost:8080 FROM precompile AS server RUN mise exec -- cargo build --release -p et-ws-server \ && cp target/release/et-ws-server /usr/local/bin/et-ws-server \ diff --git a/Dockerfile.windows b/Dockerfile.windows index 90af276..2d8801f 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -34,11 +34,9 @@ RUN curl -fsSL -o vc_redist.exe https://aka.ms/vs/17/release/vc_redist.x64.exe & del vc_redist.exe FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 -COPY --from=vcruntime ` - C:\Windows\System32\vcruntime140.dll ` - C:\Windows\System32\vcruntime140_1.dll ` - C:\Windows\System32\msvcp140.dll ` - C:\Windows\System32\ +COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ +COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ +COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive @@ -137,10 +135,10 @@ RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 10:08:01 +0800 Subject: [PATCH 16/60] windows docker fix --- .github/workflows/dependencies.yml | 16 ++++++++++------ .hadolint.yaml | 18 ++++++++++++++++++ .mise/config.python.toml | 2 -- .mise/config.toml | 11 +++++++++++ .taplo.toml | 11 ++++++----- Cargo.toml | 2 +- Dockerfile.windows | 23 +++++++++++++++++------ README.md | 26 +++++++++++++++++++++++++- osv-scanner.toml | 6 +----- 9 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 .hadolint.yaml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index d86a56b..15f0bf4 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -8,6 +8,7 @@ name: dependencies - Cargo.toml - "**/Cargo.toml" - deny.toml + - osv-scanner.toml - .github/workflows/dependencies.yml workflow_dispatch: @@ -22,11 +23,6 @@ defaults: run: shell: bash -# Deliberately mise-free: the only tools this job needs are the three -# dep-audit binaries, all of which taiki-e/install-action ships -# prebuilt. Skipping mise also skips the conda:openssl + workspace -# tool install path that the main CI flows take ~3 min on, keeping -# this check fast (~30 s typical). jobs: dependencies: runs-on: ubuntu-latest @@ -38,7 +34,12 @@ jobs: - name: Install dep-audit tools uses: taiki-e/install-action@v2 with: - tool: cargo-deny,cargo-unmaintained,osv-scanner + tool: cargo-deny,cargo-unmaintained,mise,osv-scanner + + - name: Generate osv-scanner.toml from deny.toml + run: | + mise trust + mise run gen:osv-scanner - name: cargo deny check run: cargo deny check @@ -69,3 +70,6 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: cargo unmaintained + + - name: Check osv-scanner.toml is committed + run: git diff --exit-code -- osv-scanner.toml diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..7218a6c --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,18 @@ +# hadolint config, consumed by `mise run hadolint-check`. Each entry is a +# best-practice rule we consciously skip, with the reason; drop one once it no +# longer applies. (Dockerfile.windows is not linted -- see the hadolint-check +# task in .mise/config.toml.) +ignored: + # Pinning apt package versions is brittle: they float with the Ubuntu base + # image + security updates, so an exact pin breaks on every base bump. We + # install distro defaults deliberately -- the base FROM line is the version pin. + - DL3008 + # Same for npm: `npm install -g pnpm` tracks latest; we don't pin it here. + - DL3016 + # `curl ... | sh` (the mise installer): if curl fails, sh gets empty input and + # the next `mise` command then fails anyway, so the pipe failure is caught + # downstream -- no need to set SHELL -o pipefail just for that one line. + - DL4006 + # Consecutive RUNs are kept separate on purpose -- for layer caching and so a + # change to one step doesn't bust the others. + - DL3059 diff --git a/.mise/config.python.toml b/.mise/config.python.toml index e95af62..d8fd217 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -8,9 +8,7 @@ "pipx:datamodel-code-generator" = { version = "latest", extras = "ruff" } "pipx:openapi-python-client" = "0.29.0" "pipx:pytest" = "latest" -python = "3.13" ruff = "latest" -uv = "0.11.8" # Use the GitHub release tarball, not `npm:pyodide`. The npm package is only # the runtime (pyodide.js, .wasm, stdlib, micropip) ~5 MB — it's designed for diff --git a/.mise/config.toml b/.mise/config.toml index 1c5d26c..e612952 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -60,6 +60,7 @@ dprint = "latest" editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.114" +hadolint = "latest" mprocs = "latest" node = "22" "npm:onnxruntime-web" = "latest" @@ -86,6 +87,7 @@ typos = "latest" uv = "0.11.8" wasm-tools = "latest" yq = "latest" +gh = "latest" [vars] # mise's conda tool install dirs, reused below by OPENSSL_DIR and the @@ -255,6 +257,7 @@ depends = [ "dprint-check", "editorconfig-check", "gen-specs-check", + "hadolint-check", "semgrep-check", "taplo-check", "typos", @@ -283,6 +286,14 @@ run = "ast-grep scan --error -c config/ast-grep/sgconfig.yml" description = "Run Semgrep generic-mode rules (e.g. Cargo.toml style)" run = "semgrep scan --config config/semgrep --error --metrics=off ." +# hadolint lints the Linux Dockerfiles. Dockerfile.windows is excluded -- its +# backtick-escape continuations + cmd/PowerShell RUNs aren't hadolint-friendly +# (hadolint's ShellCheck pass assumes POSIX sh). Ignored rules live in +# .hadolint.yaml. +[tasks.hadolint-check] +description = "Lint the Linux Dockerfiles with hadolint" +run = "hadolint Dockerfile services/ws-server/Dockerfile" + [tasks.cargo-check] run = "cargo check --workspace" diff --git a/.taplo.toml b/.taplo.toml index 4247c12..f986a73 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -1,8 +1,9 @@ -# Build output is off-limits: it's gitignored and (per CLAUDE.md) holds agent -# scratch files under target/scratch/. taplo's baseline `taplo lint` / `format` -# don't honour .gitignore, and they auto-associate any stray `Cargo.toml` with -# the SchemaStore Cargo schema — so a scratch fixture there breaks the check. -exclude = ["target/**"] +# Build output and external checkouts are off-limits. Both are gitignored, but +# taplo's baseline `taplo lint` / `format` don't honour .gitignore: `target/` +# holds scratch + build artifacts (and auto-associates any stray `Cargo.toml` +# with the SchemaStore Cargo schema), and `data/` holds external upstream repo +# clones whose TOML files we must never reformat. +exclude = ["target/**", "data/**"] [formatting] column_width = 120 diff --git a/Cargo.toml b/Cargo.toml index c36e4bd..19311fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ unsafe_op_in_unsafe_fn = "deny" # sibling test crates that don't import them. Tracked at rust-lang/rust#95513; re-enable when # rustc gains a workspace-aware view. # unused_crate_dependencies = "deny" -unused_results = "warn" +unused_results = "deny" [workspace.lints.rustdoc] private_intra_doc_links = { level = "deny", priority = 8 } diff --git a/Dockerfile.windows b/Dockerfile.windows index 2d8801f..6cb297a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -88,7 +88,11 @@ COPY . . # resolves to C:\.config\mise\config.toml, not auto-trusted -- so trust it after # the settings writes. aqua's signature checks (attestations, cosign, slsa) are # off: their sigstore/TUF step can't determine a cache dir here, and attestation -# also hits GitHub's changed releases signing identity. The windows job stages +# also hits GitHub's changed releases signing identity. pipx.uvx=false makes the +# pipx: backend install via pip, not uv: uv builds console-script .exe shims by +# editing PE resources -- a Win32 call Nano Server doesn't implement (error 120), +# so the semgrep install dies; pip's distlib launchers don't touch PE resources. +# The windows job stages # the GitHub token as gh_token in the build context (lifting mise's 60-req/hr # anonymous limit); we read it per-RUN rather than via a build-arg, which the # classic Windows builder can't substitute into RUN. It lands in this throwaway @@ -101,6 +105,7 @@ RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 10:27:42 +0800 Subject: [PATCH 17/60] move files under config/ --- .github/workflows/dependencies.yml | 31 ++++++++----- .github/workflows/docker.yml | 2 +- .mise/config.toml | 48 ++++++++++++++------- deny.toml => config/deny.toml | 0 .hadolint.yaml => config/hadolint.yaml | 0 osv-scanner.toml => config/osv-scanner.toml | 2 +- .taplo.toml => config/taplo.toml | 0 config/taplo/no-banned-deps.schema.json | 2 +- libs/edge-toolkit/src/config.rs | 2 +- ruff.toml | 8 ++++ 10 files changed, 65 insertions(+), 30 deletions(-) rename deny.toml => config/deny.toml (100%) rename .hadolint.yaml => config/hadolint.yaml (100%) rename osv-scanner.toml => config/osv-scanner.toml (85%) rename .taplo.toml => config/taplo.toml (100%) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 15f0bf4..6019aea 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -7,8 +7,8 @@ name: dependencies - Cargo.lock - Cargo.toml - "**/Cargo.toml" - - deny.toml - - osv-scanner.toml + - config/deny.toml + - config/osv-scanner.toml - .github/workflows/dependencies.yml workflow_dispatch: @@ -31,21 +31,30 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # The audit binaries come from taiki-e (fast prebuilts); the checks below + # are driven through `mise run` so the commands + config paths have one + # source of truth in .mise/config.toml. mise's `task.run_auto_install = + # false` means `mise run` won't drag in the whole toolchain just to run a + # task, so this stays a lightweight, audit-only job. - name: Install dep-audit tools uses: taiki-e/install-action@v2 with: tool: cargo-deny,cargo-unmaintained,mise,osv-scanner - - name: Generate osv-scanner.toml from deny.toml - run: | - mise trust - mise run gen:osv-scanner + - name: Trust mise config + run: mise trust + + - name: Generate config/osv-scanner.toml from config/deny.toml + run: mise run gen:osv-scanner - name: cargo deny check - run: cargo deny check + run: mise run cargo-deny-check + # osv-scanner runs directly, not via `mise run osv-scanner`: that task + # depends on `cargo-check` (a full workspace build) for local use, which + # this job intentionally skips -- it only scans the committed lockfile. - name: osv-scanner - run: osv-scanner --lockfile Cargo.lock + run: osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml # `cargo unmaintained` persists per-repository archival/last-commit # lookups under `$XDG_CACHE_HOME/cargo-unmaintained` (default @@ -69,7 +78,7 @@ jobs: - name: cargo unmaintained env: GITHUB_TOKEN: ${{ github.token }} - run: cargo unmaintained + run: mise run cargo-unmaintained-check - - name: Check osv-scanner.toml is committed - run: git diff --exit-code -- osv-scanner.toml + - name: Check config/osv-scanner.toml is committed + run: git diff --exit-code -- config/osv-scanner.toml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 08e61b4..9c74e8d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,5 +68,5 @@ jobs: curl.exe -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" Set-Content -Path gh_token -Value "${{ github.token }}" -NoNewline - - name: Build the Windows setup image (sketch) + - name: Build the Windows setup image run: docker build -f Dockerfile.windows -t edge-toolkit-windows . diff --git a/.mise/config.toml b/.mise/config.toml index e612952..f48e572 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -150,6 +150,10 @@ mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" # no MISE_ENV=all). Hardcoded rather than shell-discovered so it works on Windows # too — keep in sync when adding/removing a config..toml. ALL_LANGS = "dart,dotnet,java,python,rust,zig" +# taplo's config now lives under config/ instead of a root .taplo.toml, which it +# no longer auto-discovers; TAPLO_CONFIG points every `taplo lint`/`format` at +# it. config_root keeps the path valid regardless of the invocation's cwd. +TAPLO_CONFIG = "{{ config_root }}/config/taplo.toml" # Use the conda:openssl install for Rust's OPENSSL_DIR so openssl-sys crate builds. OPENSSL_DIR = "{{ vars.conda_openssl }}" # Use lld on macos as documented at https://github.com/cameron1024/dart-typegen @@ -235,11 +239,12 @@ description = "Run all formatters: fmt:rust + any loaded guest fmt:" [tasks."check:rust"] # The dep-audit tools (osv-scanner, cargo-deny, cargo-unmaintained) run # in the standalone `.github/workflows/dependencies.yml` workflow so they -# only fire when Cargo.lock / Cargo.toml / deny.toml change, not on every +# only fire when Cargo.lock / Cargo.toml / config/deny.toml change, not on every # `mise run check`. `osv-scanner` and `cargo-deny` are kept in `[tools]` -# for ad-hoc local invocation (e.g. `mise run osv-scanner`, -# `cargo deny check`); `cargo-unmaintained` is installed only by the -# workflow via `taiki-e/install-action` since it isn't useful enough +# for ad-hoc local invocation (`mise run osv-scanner`, or +# `cargo deny check --config config/deny.toml` -- deny no longer auto-finds the +# config now that it lives under config/); `cargo-unmaintained` is installed only +# by the workflow via `taiki-e/install-action` since it isn't useful enough # locally to justify the cargo-build cost on every `mise install`. # # Rust + universal repo-wide checks. Lives in the always-loaded default config, @@ -289,10 +294,10 @@ run = "semgrep scan --config config/semgrep --error --metrics=off ." # hadolint lints the Linux Dockerfiles. Dockerfile.windows is excluded -- its # backtick-escape continuations + cmd/PowerShell RUNs aren't hadolint-friendly # (hadolint's ShellCheck pass assumes POSIX sh). Ignored rules live in -# .hadolint.yaml. +# config/hadolint.yaml. [tasks.hadolint-check] description = "Lint the Linux Dockerfiles with hadolint" -run = "hadolint Dockerfile services/ws-server/Dockerfile" +run = "hadolint --config config/hadolint.yaml Dockerfile services/ws-server/Dockerfile" [tasks.cargo-check] run = "cargo check --workspace" @@ -318,7 +323,7 @@ run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace run = "taplo format" [tasks.taplo-check] -# `taplo lint` reads .taplo.toml's `[[rule]] schema` entries for editor / +# `taplo lint` reads config/taplo.toml's `[[rule]] schema` entries for editor / # LSP-style validation, but silently ignores their nested constraints in # CLI mode (verified: injecting `path = "../foo"` into a member Cargo.toml # doesn't fire `no-path-deps` via the rule alone). We work around it by @@ -395,23 +400,36 @@ mise install aube@latest [tasks.osv-scanner] depends = ["cargo-check"] -run = "osv-scanner --lockfile Cargo.lock" +run = "osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml" [tasks."gen:osv-scanner"] -description = "Regenerate osv-scanner.toml from deny.toml's [advisories].ignore list" -# osv-scanner and cargo-deny must ignore the same advisory IDs; deny.toml is the -# source of truth (it carries the per-ID rationale). grep/sort/sed only, so the -# `dependencies` workflow can run it (via mise) next to the audit binaries. +description = "Regenerate config/osv-scanner.toml from config/deny.toml's [advisories].ignore list" +# osv-scanner and cargo-deny must ignore the same advisory IDs; config/deny.toml +# is the source of truth (it carries the per-ID rationale). grep/sort/sed only, so +# the `dependencies` workflow can run it (via mise) next to the audit binaries. run = """ { - echo '# AUTO-GENERATED from deny.toml by `mise run gen:osv-scanner`.' + echo '# AUTO-GENERATED from config/deny.toml by `mise run gen:osv-scanner`.' echo 'IgnoredVulns = [' - grep -oE '"RUSTSEC-[0-9]{4}-[0-9]{4}"' deny.toml | sort -u | sed 's/.*/ { id = & },/' + grep -oE '"RUSTSEC-[0-9]{4}-[0-9]{4}"' config/deny.toml | sort -u | sed 's/.*/ { id = & },/' echo ']' -} >osv-scanner.toml +} > config/osv-scanner.toml """ shell = "bash -euo pipefail -c" +[tasks.cargo-deny-check] +description = "Audit dependencies with cargo-deny (advisories, bans, licenses, sources)" +run = "cargo deny check --config config/deny.toml" + +[tasks.cargo-unmaintained-check] +description = "Flag dependencies whose upstream repo is archived or unmaintained" +# cargo-unmaintained is deliberately NOT a mise [tool]: it has no prebuilt and a +# from-source `cargo install` is too heavy to pay on every `mise install`. The +# `dependencies` workflow installs it via taiki-e/install-action; this task just +# runs the binary so the audit command lives beside the others. A GITHUB_TOKEN in +# the env lets it check each upstream repo's archival status. +run = "cargo unmaintained" + [tasks."gen:dockerignore"] description = "Regenerate .dockerignore from .gitignore + Docker-only excludes" # Docker reads only the root .dockerignore (never .gitignore or nested ignore diff --git a/deny.toml b/config/deny.toml similarity index 100% rename from deny.toml rename to config/deny.toml diff --git a/.hadolint.yaml b/config/hadolint.yaml similarity index 100% rename from .hadolint.yaml rename to config/hadolint.yaml diff --git a/osv-scanner.toml b/config/osv-scanner.toml similarity index 85% rename from osv-scanner.toml rename to config/osv-scanner.toml index 82a441e..25a1e54 100644 --- a/osv-scanner.toml +++ b/config/osv-scanner.toml @@ -1,4 +1,4 @@ -# AUTO-GENERATED from deny.toml by `mise run gen:osv-scanner`. +# AUTO-GENERATED from config/deny.toml by `mise run gen:osv-scanner`. IgnoredVulns = [ { id = "RUSTSEC-2023-0071" }, { id = "RUSTSEC-2024-0436" }, diff --git a/.taplo.toml b/config/taplo.toml similarity index 100% rename from .taplo.toml rename to config/taplo.toml diff --git a/config/taplo/no-banned-deps.schema.json b/config/taplo/no-banned-deps.schema.json index a74bfd9..8d06166 100644 --- a/config/taplo/no-banned-deps.schema.json +++ b/config/taplo/no-banned-deps.schema.json @@ -10,7 +10,7 @@ "const": "forbidden; define a thiserror enum instead" }, "ring": { - "const": "forbidden; use aws-lc-rs. Transitive via rcgen only (gated in deny.toml's bans.deny wrappers)." + "const": "forbidden; use aws-lc-rs. Transitive via rcgen only (gated in config/deny.toml's bans.deny)." }, "ureq": { "const": "forbidden; use reqwest::blocking (sync) or reqwest (async). One HTTPS stack only." diff --git a/libs/edge-toolkit/src/config.rs b/libs/edge-toolkit/src/config.rs index 295b88c..86e48c9 100644 --- a/libs/edge-toolkit/src/config.rs +++ b/libs/edge-toolkit/src/config.rs @@ -15,7 +15,7 @@ pub const LOCALHOST: &str = "127.0.0.1"; #[expect(clippy::missing_panics_doc, clippy::unwrap_used)] #[must_use] pub fn get_project_root() -> PathBuf { - match lets_find_up::find_up(".taplo.toml") { + match lets_find_up::find_up(".dprint.jsonc") { Ok(Some(mut path)) => { assert!(path.pop(), "Failed to drop the filename"); path diff --git a/ruff.toml b/ruff.toml index 8311165..32defa1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,11 @@ +# Kept at the repo root (unlike config/deny.toml, config/osv-scanner.toml and +# config/taplo.toml) on purpose: editors/IDEs (the VS Code Ruff extension, +# PyCharm) and pre-commit auto-discover ruff config only as a root ruff.toml / +# .ruff.toml / pyproject.toml. ruff has no config-path env var, so moving this +# under config/ would silently drop every editor back to ruff's defaults +# (line-length 88, no isort). The mise tasks could pass --config, but the live +# editor experience can't, so it stays here. +# # Match the repo-wide 120 line-length set in .editorconfig. Without this, ruff # would use its default of 88, so files that ruff considered "already # formatted" could still trip the editorconfig-check. From 691f4f6c51040645b33bc10ddbdc852a83c784ca Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 10:30:34 +0800 Subject: [PATCH 18/60] disable another auto install setting --- .mise/config.toml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index f48e572..a6848dd 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -12,13 +12,17 @@ # Make a selection sticky: export MISE_ENV=dart [settings] -# Never auto-install tools before a `mise run`: installs are explicit -# (`mise install` / `mise run install-all`, and CI installs up front). This -# keeps cheap tasks like `print-all-langs`, and the pre-install `setup-aube` -# step, from eagerly pulling the whole toolchain — on CI and the CLI alike. -# (Current mise accepts this nested form; very old builds like 2026.3 instead -# used the flat `task_run_auto_install`.) +# Never auto-install tools implicitly: installs are explicit (`mise install` / +# `mise run install-all`, and CI/Docker install up front). This keeps cheap tasks +# like `print-all-langs`, and the pre-install `setup-aube` step, from eagerly +# pulling the whole toolchain — on CI and the CLI alike. task.run_auto_install +# covers `mise run`; exec_auto_install covers `mise exec` (its OWN knob) — +# without it, `mise exec -- python …` in Dockerfile.windows installs *every* +# configured tool before running python. Turn both off so the policy holds. +# (Current mise accepts the nested task.run_auto_install form; very old builds +# like 2026.3 used the flat `task_run_auto_install`.) task.run_auto_install = false +exec_auto_install = false [tools] action-validator = "latest" From f956319a422771f8c0a9012eecd0a1c913072d59 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 11:11:56 +0800 Subject: [PATCH 19/60] various improvements --- .github/workflows/docker.yml | 7 +- .github/workflows/test.yml | 6 +- .mise/config.toml | 35 +++++++ Dockerfile.windows | 192 +++++++++++++++++++++++------------ README.md | 24 ++--- 5 files changed, 179 insertions(+), 85 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9c74e8d..02b9654 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,5 +68,10 @@ jobs: curl.exe -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" Set-Content -Path gh_token -Value "${{ github.token }}" -NoNewline + # Build only through the `build` stage (mise + the full toolchain): that's + # the current frontier -- the install pipeline is what we're proving on a + # bare Nano Server. The later stages (prefetch/precompile/test/server) + # mirror the Linux Dockerfile but aren't reached yet; bump --target as they + # come green. - name: Build the Windows setup image - run: docker build -f Dockerfile.windows -t edge-toolkit-windows . + run: docker build -f Dockerfile.windows --target build -t edge-toolkit-windows . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index acd1d03..038f0a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,13 +45,13 @@ jobs: github.event.inputs.os == 'all' && format( '[{0},{1},{2},{3},{4}]', '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":25}', + '{"os":"ubuntu-24.04-arm","timeout":30}', '{"os":"macos-latest","timeout":45}', '{"os":"macos-26-intel","timeout":60}', '{"os":"windows-latest","timeout":60}' ) || github.event.inputs.os == 'ubuntu-latest' && '[{"os":"ubuntu-latest","timeout":30}]' - || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":25}]' + || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":30}]' || github.event.inputs.os == 'macos-latest' && '[{"os":"macos-latest","timeout":45}]' || github.event.inputs.os == 'macos-26-intel' && '[{"os":"macos-26-intel","timeout":60}]' || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":60}]' @@ -59,7 +59,7 @@ jobs: || format( '[{0},{1},{2}]', '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":25}', + '{"os":"ubuntu-24.04-arm","timeout":30}', '{"os":"macos-latest","timeout":45}' ) ) }} diff --git a/.mise/config.toml b/.mise/config.toml index a6848dd..4750829 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -402,6 +402,41 @@ mise install aube@latest {% endif %} """ +[tasks.setup-all] +# Cross-platform preinstall, run first on every OS (and by both Dockerfiles): +# enable experimental + the cargo-binstall backend, then install cargo-binstall +# so later installs of cargo: tools fetch prebuilts instead of building from +# source. Mirrors the README's "Configure it with" + cargo-binstall steps. +description = "Cross-platform preinstall: enable experimental + cargo.binstall, install cargo-binstall" +run = """ +mise settings experimental=true +mise settings set cargo.binstall true +mise install cargo-binstall +""" + +[tasks.setup-macos] +# macOS preinstall (after Xcode's command-line tools, which mise can't supply): +# install conda:lld, the linker the darwin CARGO_TARGET_* RUSTFLAGS use. Mirrors +# the README's "MacOS only" step. +description = "macOS preinstall: install conda:lld (the linker the darwin build uses)" +run = "mise install conda:lld" + +[tasks.setup-windows] +# Windows preinstall, shared by Dockerfile.windows and a real workstation, run +# after setup-all. Installs the Windows toolchain mise CAN supply -- gpg +# (conda:m2-gnupg, so mise verifies later downloads), the llvm-mingw gnu C/link +# toolchain (+ libclang), make and rust -- then flips rustup to the gnu host so +# cargo links via llvm-mingw, not MSVC link.exe (no Visual Studio Build Tools +# needed). The prereqs mise CAN'T supply (a recent mise + pipx) are in the +# README's "Windows only" section. Windows-only; errors if run on another OS. +description = "Windows preinstall: install the gnu toolchain (gpg, llvm-mingw, make, rust) + flip rustup to gnu" +run = """ +mise install conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust +mise exec -- rustup set default-host x86_64-pc-windows-gnu +mise exec -- rustup toolchain install stable-x86_64-pc-windows-gnu +mise exec -- rustup default stable-x86_64-pc-windows-gnu +""" + [tasks.osv-scanner] depends = ["cargo-check"] run = "osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml" diff --git a/Dockerfile.windows b/Dockerfile.windows index 6cb297a..3ac80a7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -10,6 +10,15 @@ # can't be built on the Linux dev/CI machines. Treat it as a CI-iterated # starting point, not a known-good image. # +# Staged to mirror the Linux Dockerfile so the layers cache + target the same +# way: vcruntime -> build-minimal -> build -> prefetch -> precompile -> test / +# server. Each stage `FROM`s the previous, so installed tools, fetched deps and +# built module pkg/ carry forward; `--target ` stops early. The +# Windows-only bootstrap (VC++ runtime, mise.exe, HOME/TEMP, the gnu-host rust +# flip) lives in build-minimal. Where Linux stops at a clean build/test/serve, +# Windows is still proving the toolchain installs at all -- the stages past +# `build` are the CI frontier (see "Known unknowns" at the foot). +# # Base: Nano Server, the smallest Windows base (~120 MB vs Server Core's ~1.25 # GB). It has no MSI/installer stack, no PowerShell, and runs unprivileged -- so # the Visual Studio Build Tools installer can't run here at all. That's the @@ -19,13 +28,14 @@ # os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows SDK # can't come from mise, so the build targets x86_64-pc-windows-gnu. -# Donor stage: Nano Server omits the VC++ runtime that msvc-built executables -# (mise.exe, and the rust/cargo mise installs) need just to *start* -- without it -# mise.exe dies with 0xC0000135 (DLL not found). No base Windows image ships it -# (not even Server Core), and mise can't install it (mise needs it to run), so we -# install the official VC++ redistributable on a Server Core donor -- which has -# the installer stack Nano Server lacks -- and copy the runtime DLLs forward. -# This plus mise are the only non-mise bootstrap bits; the rest is mise. +# --- vcruntime: donor stage for the VC++ runtime DLLs. --- +# Nano Server omits the VC++ runtime that msvc-built executables (mise.exe, and +# the rust/cargo mise installs) need just to *start* -- without it mise.exe dies +# with 0xC0000135 (DLL not found). No base Windows image ships it (not even +# Server Core), and mise can't install it (mise needs it to run), so we install +# the official VC++ redistributable on a Server Core donor -- which has the +# installer stack Nano Server lacks -- and copy the runtime DLLs forward. This +# plus mise are the only non-mise bootstrap bits; the rest is mise. FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS vcruntime SHELL ["cmd", "/S", "/C"] # `|| ver>nul` swallows the redistributable's 3010 ("reboot required") exit code. @@ -33,7 +43,13 @@ RUN curl -fsSL -o vc_redist.exe https://aka.ms/vs/17/release/vc_redist.x64.exe & (vc_redist.exe /install /quiet /norestart || ver>nul) && ` del vc_redist.exe -FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +# --- build-minimal: mise + the always-loaded toolchain (config.toml only). --- +# Mirrors the Linux build-minimal: copies just .mise/config.toml + installs the +# default tools, so this layer is reused until the always-loaded toolset changes, +# not when a guest config or the source does. All the Windows-only bootstrap +# (VC++ DLLs, mise.exe, HOME, TEMP) and the gnu-host rust flip live here too, +# since every later stage builds on it. +FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 AS build-minimal COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ @@ -44,6 +60,14 @@ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] +# Diagnostic (informational -- never fails the build): does this base image ship +# a bash? The `bash -euo pipefail` mise tasks need one; Nano Server has none, so +# mise pulls conda:m2-bash (see the os-guarded Windows tools in .mise/config.toml). +# Record which it is in the build log so the "does mise find bash" unknown is +# answered up front. +RUN where bash >nul 2>&1 && (echo [bash-check] pre-installed: & where bash) || ` + echo [bash-check] not pre-installed -- mise provides conda:m2-bash for tasks + # Set HOME to the admin profile so the container has a sane home (Nano Server # leaves it unset). This also renders the config's `{{ env.HOME }}` paths; note # mise still resolves its *global config* to C:\.config\mise on Windows @@ -64,6 +88,14 @@ RUN if not exist C:\Temp mkdir C:\Temp ENV TEMP=C:\Temp ` TMP=C:\Temp +# Serialize mise's installs (MISE_JOBS=1). On Nano Server the parallel cargo: +# source builds -- no gnu binstall prebuilts exist, so they fall back to `cargo +# install` -- race on rustup's shared .rustup\downloads dir: one build's +# component .partial vanishes before another rustup can rename it ("cannot find +# the file", os error 2). Windows CI doesn't hit this (msvc binstall prebuilts +# mean no source builds); the gnu Docker path does. One job at a time avoids it. +ENV MISE_JOBS=1 + # mise.zip is staged in the build context by the windows job in # .github/workflows/docker.yml (which pins the version) and copied in here. We # don't curl a versioned URL inside the build: the classic Windows builder @@ -76,80 +108,106 @@ RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip ENV PATH="C:\mise\bin;${PATH}" WORKDIR C:\workspace -COPY . . +# Only the always-loaded config is needed for the default tools; the +# guest-language configs come in the build stage below. +COPY .mise/config.toml .mise/config.toml -# mise setup, following the README, but with the gnu toolchain installed and -# selected UP FRONT: Nano Server has no msvc link.exe, so any cargo: tool that -# lacks a prebuilt (and falls back to `cargo install`) must link via llvm-mingw, -# not msvc. So install gpg + llvm-mingw + rust first and flip rustup to the gnu -# host, then run the bulk installs. -# -# `mise settings` writes mise's global config; with no real home on Windows that -# resolves to C:\.config\mise\config.toml, not auto-trusted -- so trust it after -# the settings writes. aqua's signature checks (attestations, cosign, slsa) are -# off: their sigstore/TUF step can't determine a cache dir here, and attestation -# also hits GitHub's changed releases signing identity. pipx.uvx=false makes the -# pipx: backend install via pip, not uv: uv builds console-script .exe shims by -# editing PE resources -- a Win32 call Nano Server doesn't implement (error 120), -# so the semgrep install dies; pip's distlib launchers don't touch PE resources. -# The windows job stages -# the GitHub token as gh_token in the build context (lifting mise's 60-req/hr -# anonymous limit); we read it per-RUN rather than via a build-arg, which the -# classic Windows builder can't substitute into RUN. It lands in this throwaway -# image and expires with the job. +# The windows job stages the GitHub token as gh_token in the build context +# (lifting mise's 60-req/hr anonymous limit); we read it per-RUN rather than via +# a build-arg, which the classic Windows builder can't substitute into RUN. It +# carries forward into the later stages that also hit GitHub and expires with the +# throwaway job image. COPY gh_token C:\gh_token + +# mise setup via the shared tasks (the same a workstation runs -- see the +# README): setup-all enables experimental + cargo.binstall and installs +# cargo-binstall; setup-windows installs the gnu toolchain (gpg, llvm-mingw, +# make, rust) and flips rustup to the gnu host, so cargo links via llvm-mingw +# rather than the absent MSVC link.exe. The Docker-only settings stay inline: +# `mise settings` writes the global config (C:\.config\mise on Windows, not +# auto-trusted -- hence the explicit trust); aqua's signature checks +# (attestations/cosign/slsa) are off because their sigstore/TUF step can't find a +# cache dir here; pipx.uvx=false makes the pipx: backend use pip, not uv (uv's +# PE-resource trampoline isn't implemented on Nano Server, error 120). RUN set /p GITHUB_TOKEN=.toml). --- +# install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are +# already installed by build-minimal, so this adds dart/dotnet/java/zig/etc. +# MISE_ENV is set persistently so the later stages act on every language too. +FROM build-minimal AS build +COPY .mise/ .mise/ +RUN mise trust +ENV MISE_ENV=dart,dotnet,java,python,rust,zig RUN set /p GITHUB_TOKEN=\release\; copy it onto PATH (C:\mise\bin) and drop target/ in +# the same layer so the build intermediates don't bloat the image. The server +# serves each module from its pkg/ (none of which live in target/). +FROM precompile AS server +RUN mise exec -- cargo build --release -p et-ws-server && ` + copy target\x86_64-pc-windows-gnu\release\et-ws-server.exe C:\mise\bin\ && ` + if exist target rmdir /s /q target +EXPOSE 8080 8443 +CMD ["et-ws-server"] + +# Known unknowns to settle via CI, roughly in the order the stages reach them. +# build-minimal / build: whether the rustup host flip sticks (mise may re-pin its +# own toolchain) and a gnu-host proc-macro/build-script build is clean; whether +# the conda backend (micromamba) and the github llvm-mingw asset match resolve +# cleanly on Nano Server; whether mise finds bash (m2-bash) for the `bash -euo +# pipefail` tasks. prefetch onward (not yet reached -- the frontier): whether +# `ort` (ONNX Runtime, msvc-only prebuilts) breaks the gnu link once a real cargo +# build/test links it; whether build-modules (wasm-pack + the guest toolchains) +# and a release et-ws-server build work at all under the gnu target on Nano Server. diff --git a/README.md b/README.md index f7575c7..a4b91f5 100644 --- a/README.md +++ b/README.md @@ -28,17 +28,11 @@ all of them at once. The following works for Linux, macOS and Windows, and all tools "installed" are only installed into the local workspace, so no need for admin/root privileges. -Configure it with: +Configure it and pre-install `cargo-binstall` (so later installs of `cargo:` +tools fetch prebuilts instead of building from source): ```bash -mise settings experimental=true -mise settings set cargo.binstall true -``` - -Pre-install `cargo-install`, which can be done using: - -```bash -mise install cargo-binstall +mise run setup-all ``` ### GitHub rate limits @@ -82,12 +76,14 @@ be installed first — mise can't supply them: [pipx Windows instructions](https://pipx.pypa.io/stable/how-to/install-pipx/)); mise's `pipx:*` tools (e.g. semgrep) then resolve through it. -mise verifies tool downloads with gpg, which it can install itself on Windows. -Do that first — before the "All OS" steps below — so the verification is in -place: +Then run `setup-windows` — it installs the rest mise can supply (gpg so mise +verifies downloads, the llvm-mingw gnu toolchain, make and rust) and flips rust +to the `x86_64-pc-windows-gnu` host, so cargo links via llvm-mingw. (That gnu +path is why the Build Tools / LLVM above are only needed if you instead build +for the default msvc target.) Run it before the "All OS" steps below: ```bash -mise install conda:m2-gnupg +mise run setup-windows ``` ### MacOS only @@ -102,7 +98,7 @@ xcode-select --install We also need to install a better linker into the workspace. ```bash -mise install conda:lld +mise run setup-macos ``` ### All OS From b96905ffab962c55cf1033a17f176f7b68d16909 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 11:28:32 +0800 Subject: [PATCH 20/60] setup-foo tasks --- .mise/config.toml | 46 +++++++++++++++++++++++++++++----------------- Dockerfile | 20 ++++++++------------ Dockerfile.windows | 35 ++++++++++++++++------------------- README.md | 30 +++++++++++++++--------------- 4 files changed, 68 insertions(+), 63 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 4750829..8e72468 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -403,33 +403,45 @@ mise install aube@latest """ [tasks.setup-all] -# Cross-platform preinstall, run first on every OS (and by both Dockerfiles): -# enable experimental + the cargo-binstall backend, then install cargo-binstall -# so later installs of cargo: tools fetch prebuilts instead of building from -# source. Mirrors the README's "Configure it with" + cargo-binstall steps. -description = "Cross-platform preinstall: enable experimental + cargo.binstall, install cargo-binstall" +# Shared cross-platform preinstall -- not run directly; the per-OS setup tasks +# (setup-linux / setup-macos / setup-windows) depend on it, so each controls its +# own ordering. Enables experimental + cargo.binstall, then installs +# cargo-binstall (a prebuilt binary, safe to preinstall anywhere), node (mise +# uses it for the npm: tools) and conda:openssl (for openssl-sys) -- all needed +# on every platform. +description = "Shared cross-platform preinstall (invoked via setup-linux/macos/windows, not run directly)" run = """ mise settings experimental=true mise settings set cargo.binstall true mise install cargo-binstall +mise install node +mise install conda:openssl """ +[tasks.setup-linux] +# Linux preinstall: just the shared cross-platform base -- the C toolchain and +# gpg come from the distro (apt/system), not mise. +description = "Linux preinstall (the shared cross-platform base)" +depends = ["setup-all"] + [tasks.setup-macos] -# macOS preinstall (after Xcode's command-line tools, which mise can't supply): -# install conda:lld, the linker the darwin CARGO_TARGET_* RUSTFLAGS use. Mirrors -# the README's "MacOS only" step. -description = "macOS preinstall: install conda:lld (the linker the darwin build uses)" +# macOS preinstall, run after Xcode's command-line tools (which mise can't +# supply): the shared base, then conda:lld -- the linker the darwin +# CARGO_TARGET_* RUSTFLAGS use. +description = "macOS preinstall: shared base + conda:lld (the linker the darwin build uses)" +depends = ["setup-all"] run = "mise install conda:lld" [tasks.setup-windows] -# Windows preinstall, shared by Dockerfile.windows and a real workstation, run -# after setup-all. Installs the Windows toolchain mise CAN supply -- gpg -# (conda:m2-gnupg, so mise verifies later downloads), the llvm-mingw gnu C/link -# toolchain (+ libclang), make and rust -- then flips rustup to the gnu host so -# cargo links via llvm-mingw, not MSVC link.exe (no Visual Studio Build Tools -# needed). The prereqs mise CAN'T supply (a recent mise + pipx) are in the -# README's "Windows only" section. Windows-only; errors if run on another OS. -description = "Windows preinstall: install the gnu toolchain (gpg, llvm-mingw, make, rust) + flip rustup to gnu" +# Windows preinstall, shared by Dockerfile.windows and a real workstation: the +# shared base runs first, then this installs the Windows toolchain mise CAN +# supply -- gpg (conda:m2-gnupg, so mise verifies downloads), the llvm-mingw gnu +# C/link toolchain (+ libclang), make and rust -- then flips rustup to the gnu +# host so cargo links via llvm-mingw, not MSVC link.exe (no Visual Studio Build +# Tools needed). The prereqs mise CAN'T supply (a recent mise + pipx) are in the +# README's "Windows only" section. +description = "Windows preinstall: shared base + gnu toolchain (gpg, llvm-mingw, make, rust) + rustup gnu flip" +depends = ["setup-all"] run = """ mise install conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust mise exec -- rustup set default-host x86_64-pc-windows-gnu diff --git a/Dockerfile b/Dockerfile index 0d4b0db..0ae239d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,26 +70,22 @@ RUN apt-get update \ RUN curl -fsSL https://mise.run | sh ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" -# Configure mise + pre-install cargo-binstall. A GitHub token (if provided) -# lifts the anonymous rate limit for the cargo-binstall release fetch. -RUN --mount=type=secret,id=gh_token,required=false \ - GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ - sh -c 'mise settings experimental=true \ - && mise settings set cargo.binstall true \ - && mise use -g cargo-binstall' - WORKDIR /workspace # Only the always-loaded config is needed for the default tools; the -# guest-language configs come in the build stage below. +# guest-language configs come in the build stage below. setup-linux is a repo +# task, so the config has to be copied + trusted before it can run. COPY .mise/config.toml .mise/config.toml RUN mise trust +# Preinstall via the shared setup-linux task (the same a Linux workstation runs): +# its setup-all base enables experimental + cargo.binstall and installs +# cargo-binstall, node and conda:openssl; then `mise install` adds the rest of +# the always-loaded tools. A GitHub token (if provided) lifts the anonymous rate +# limit for the release fetches. RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ - sh -c 'mise install node \ - && mise install conda:openssl \ - && mise install' + sh -c 'mise run setup-linux && mise install' # --- build: add the guest-language toolchains (config..toml). --- # install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are diff --git a/Dockerfile.windows b/Dockerfile.windows index 3ac80a7..42bb0ad 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -119,16 +119,16 @@ COPY .mise/config.toml .mise/config.toml # throwaway job image. COPY gh_token C:\gh_token -# mise setup via the shared tasks (the same a workstation runs -- see the -# README): setup-all enables experimental + cargo.binstall and installs -# cargo-binstall; setup-windows installs the gnu toolchain (gpg, llvm-mingw, -# make, rust) and flips rustup to the gnu host, so cargo links via llvm-mingw -# rather than the absent MSVC link.exe. The Docker-only settings stay inline: -# `mise settings` writes the global config (C:\.config\mise on Windows, not -# auto-trusted -- hence the explicit trust); aqua's signature checks -# (attestations/cosign/slsa) are off because their sigstore/TUF step can't find a -# cache dir here; pipx.uvx=false makes the pipx: backend use pip, not uv (uv's -# PE-resource trampoline isn't implemented on Nano Server, error 120). +# mise setup via the same task a workstation runs (see the README): setup-windows +# depends on setup-all (experimental + cargo.binstall, cargo-binstall, node, +# openssl), then installs the gnu toolchain (gpg, llvm-mingw, make, rust) and +# flips rustup to the gnu host, so cargo links via llvm-mingw rather than the +# absent MSVC link.exe. The Docker-only settings stay inline: `mise settings` +# writes the global config (C:\.config\mise on Windows, not auto-trusted -- hence +# the explicit trust); aqua's signature checks (attestations/cosign/slsa) are off +# because their sigstore/TUF step can't find a cache dir here; pipx.uvx=false +# makes the pipx: backend use pip, not uv (uv's PE-resource trampoline isn't +# implemented on Nano Server, error 120). RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 13:11:24 +0800 Subject: [PATCH 21/60] windows fix and rules --- .github/workflows/check.yml | 18 +++-- .github/workflows/docker.yml | 11 ++- .github/workflows/test.yml | 27 ++++--- .mise/config.dart.toml | 2 +- .mise/config.toml | 55 ++++++++++---- CLAUDE.md | 74 +++++++++++++++++++ Dockerfile.windows | 70 +++++++++++------- README.md | 49 +++++++----- .../ast-grep/rules/gha-default-shell-bash.yml | 19 +++++ .../ast-grep/rules/gha-no-step-shell-bash.yml | 14 ++++ config/semgrep/mise-config.yml | 14 ++++ config/taplo.toml | 5 +- config/taplo/mise-run-not-array.schema.json | 19 +++++ 13 files changed, 290 insertions(+), 87 deletions(-) create mode 100644 config/ast-grep/rules/gha-default-shell-bash.yml create mode 100644 config/ast-grep/rules/gha-no-step-shell-bash.yml create mode 100644 config/taplo/mise-run-not-array.schema.json diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6013369..55bedeb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -54,13 +54,17 @@ jobs: - name: Install mise tools run: | mise settings add idiomatic_version_file_enable_tools "[]" - mise settings experimental=true - mise settings set cargo.binstall true - # See test.yml for notes on why conda:openssl is installed up front. - mise install conda:openssl - # On macOS, lld is needed to compile Rust binary tools from source - # (e.g. `cargo:taplo-cli`, see CARGO_TARGET_*_APPLE_DARWIN_RUSTFLAGS). - mise install conda:lld + # Platform preinstall via the shared setup task (see .mise/config.toml): + # setup-all does settings + cargo-binstall + node + conda:openssl; + # setup-macos adds conda:lld. Running it before the main `mise install` + # keeps conda:openssl ahead of any parallel cargo:* openssl-sys build. + # Windows CI uses its native msvc toolchain, so it runs setup-all (the + # base), not setup-windows whose gnu-host flip is for the bare-box path. + case "${{ runner.os }}" in + Linux) mise run setup-linux ;; + macOS) mise run setup-macos ;; + Windows) mise run setup-all ;; + esac mise install env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 02b9654..8e76cd7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,6 +16,10 @@ concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +defaults: + run: + shell: bash + jobs: linux: runs-on: ubuntu-latest @@ -58,15 +62,16 @@ jobs: # the build can fail connecting to the docker_engine pipe. Start it (no-op # if already running) and confirm connectivity before building. - name: Start the Docker daemon + shell: pwsh run: | if ((Get-Service docker).Status -ne 'Running') { Start-Service docker } docker version - name: Stage mise and the token in the build context run: | - $v = "${{ env.MISE_VERSION }}" - curl.exe -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" - Set-Content -Path gh_token -Value "${{ github.token }}" -NoNewline + v="${{ env.MISE_VERSION }}" + curl -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" + printf '%s' "${{ github.token }}" > gh_token # Build only through the `build` stage (mise + the full toolchain): that's # the current frontier -- the install pipeline is what we're proving on a diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 038f0a5..f1dc7a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,21 +108,20 @@ jobs: - name: Install mise tools run: | mise settings add idiomatic_version_file_enable_tools "[]" - mise settings experimental=true - mise settings set cargo.binstall true - - # Install conda:openssl up front — the project's OPENSSL_DIR points - # at its install dir, and any `cargo:*` tool that pulls openssl-sys - # reads that during its own `cargo install`. With the default - # parallel `mise install`, openssl-sys can panic with "OpenSSL - # include directory does not exist" if conda:openssl hasn't - # finished populating `include/` yet. - - mise install conda:openssl - - # And on macos, lld is needed to compile Rust binary tools from source. - mise install conda:lld + # Platform preinstall via the shared setup task (see .mise/config.toml). + # setup-all does settings + cargo-binstall + node + conda:openssl; + # setup-macos adds conda:lld. Running it before the main `mise install` + # keeps conda:openssl ahead of any parallel `cargo:*` openssl-sys build + # (which reads the project's OPENSSL_DIR and panics if include/ isn't + # populated yet). Windows CI uses its native msvc toolchain, so it runs + # setup-all (the base), not setup-windows whose gnu-host flip is for the + # bare-box/Docker path. + case "${{ runner.os }}" in + Linux) mise run setup-linux ;; + macOS) mise run setup-macos ;; + Windows) mise run setup-all ;; + esac mise install env: diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 69c2c54..23304a4 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -75,7 +75,6 @@ run = "dart pub get --directory services/ws-modules/dart-comm1" [tasks."gen:dart-ws"] depends = ["gen:ws-spec"] description = "Emit the Dart sealed-class WS client via dart-typegen (consumes generated/specs/ws.kdl from gen:ws-spec)" -shell = "bash -euo pipefail -c" # dart-typegen exits 0 on Windows but writes an empty .dart file, which # would clobber the committed source and fail gen-specs-check. Skip the # regen there; the committed file (regenerated on Linux/macOS) stays @@ -86,3 +85,4 @@ mkdir -p generated/dart-ws/lib dart-typegen generate -i generated/specs/ws.kdl -o generated/dart-ws/lib/ws_messages.dart dart format generated/dart-ws/lib/ws_messages.dart """ +shell = "bash -euo pipefail -c" diff --git a/.mise/config.toml b/.mise/config.toml index 8e72468..5abfe94 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -182,6 +182,16 @@ CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_ # linker on Linux accepts `-Wl,-rpath` directly, unlike Apple clang's lld dance. CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +# Windows targets x86_64-pc-windows-gnu (llvm-mingw -- see Dockerfile.windows / +# the README). cargo reads these CARGO_TARGET__* only when building for +# that triple, so they're inert on Linux/macOS and need no os() guard. The linker +# is llvm-mingw's gcc; -Cdlltool names llvm-mingw's dlltool for the gnu raw-dylib +# import libs (rust-lang/rust#103939). The vars that force the gnu host/target +# (RUSTUP_TOOLCHAIN, CARGO_BUILD_TARGET) can't live here -- they're not +# triple-scoped, so an empty off-Windows value breaks Linux cargo ("target was +# empty"); Dockerfile.windows sets those. +CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER = "x86_64-w64-mingw32-gcc" +CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS = "-Cdlltool=x86_64-w64-mingw32-dlltool" # bindgen (pulled in transitively by the deno web runner) needs the C # `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ # API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys @@ -288,8 +298,10 @@ description = "Validate GitHub Actions workflow YAML" run = "action-validator .github/workflows/*.yml" [tasks.ast-grep-check] +# --no-ignore hidden so the gha-* YAML rules reach .github/workflows (ast-grep +# skips dot-dirs by default); gitignored paths like target/ stay skipped. description = "Run ast-grep structural-search rules" -run = "ast-grep scan --error -c config/ast-grep/sgconfig.yml" +run = "ast-grep scan --error --no-ignore hidden -c config/ast-grep/sgconfig.yml" [tasks.semgrep-check] description = "Run Semgrep generic-mode rules (e.g. Cargo.toml style)" @@ -373,6 +385,11 @@ for f in "${members[@]}"; do lint_inheritable+=("$f") done taplo lint --schema "file://$PWD/config/taplo/require-lints-section.schema.json" "${lint_inheritable[@]}" + +# mise task `run` must be a string, not an array: taplo's reorder_arrays would +# re-sort an array and scramble an ordered command sequence (use a multiline +# string, one command per line). Applies to every .mise/config*.toml. +taplo lint --schema "file://$PWD/config/taplo/mise-run-not-array.schema.json" .mise/config*.toml """ shell = "bash -euo pipefail -c" @@ -391,8 +408,7 @@ run = "typos" # # Platform split: aqua ships aube for linux/macos-arm64/windows; macos/x64 has # no aqua asset, so use the cargo backend there. The branch is rendered by -# mise's tera templating, so it works regardless of the task shell (cmd on -# Windows). +# mise's tera templating to pick the right backend per platform. description = "Install the optional aube npm backend (platform-specific, best-effort)" run = """ {% if os() == "macos" and arch() == "x64" %} @@ -401,6 +417,7 @@ mise install cargo:aube@latest mise install aube@latest {% endif %} """ +shell = "bash -euo pipefail -c" [tasks.setup-all] # Shared cross-platform preinstall -- not run directly; the per-OS setup tasks @@ -417,6 +434,7 @@ mise install cargo-binstall mise install node mise install conda:openssl """ +shell = "bash -euo pipefail -c" [tasks.setup-linux] # Linux preinstall: just the shared cross-platform base -- the C toolchain and @@ -433,21 +451,30 @@ depends = ["setup-all"] run = "mise install conda:lld" [tasks.setup-windows] -# Windows preinstall, shared by Dockerfile.windows and a real workstation: the -# shared base runs first, then this installs the Windows toolchain mise CAN -# supply -- gpg (conda:m2-gnupg, so mise verifies downloads), the llvm-mingw gnu -# C/link toolchain (+ libclang), make and rust -- then flips rustup to the gnu -# host so cargo links via llvm-mingw, not MSVC link.exe (no Visual Studio Build -# Tools needed). The prereqs mise CAN'T supply (a recent mise + pipx) are in the -# README's "Windows only" section. -description = "Windows preinstall: shared base + gnu toolchain (gpg, llvm-mingw, make, rust) + rustup gnu flip" +# Windows preinstall, shared by Dockerfile.windows and a real workstation. bash +# (conda:m2-bash) is installed by the bootstrap that runs before any mise task on +# Windows (see Dockerfile.windows / the README), so this task's `bash -euo +# pipefail` shell works. The shared base runs first, then this installs git +# (conda:m2-git, for cargo git deps), gpg (conda:m2-gnupg, so mise verifies +# downloads), the llvm-mingw gnu C/link toolchain (+ libclang), make and rust -- +# then flips rustup to the gnu host so cargo links via llvm-mingw, not MSVC +# link.exe (no Visual Studio Build Tools needed). The prereqs mise CAN'T supply +# (a recent mise + the VC++ runtime it needs to start) are in the README's +# "Windows only" section. +description = "Windows preinstall: install the Windows toolchain (git, gpg, llvm-mingw, rust) + flip rustup to gnu" depends = ["setup-all"] run = """ -mise install conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust +mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust +# Install + default to the gnu build of the EXACT rust version mise pinned (not +# the "stable" channel, which only happens to match mise's "latest"). mise sets +# RUSTUP_TOOLCHAIN to that bare version, so once the default host is gnu it +# resolves to this -gnu toolchain. +ver="$(mise exec -- rustc --version | awk '{print $2}')" mise exec -- rustup set default-host x86_64-pc-windows-gnu -mise exec -- rustup toolchain install stable-x86_64-pc-windows-gnu -mise exec -- rustup default stable-x86_64-pc-windows-gnu +mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnu" +mise exec -- rustup default "${ver}-x86_64-pc-windows-gnu" """ +shell = "bash -euo pipefail -c" [tasks.osv-scanner] depends = ["cargo-check"] diff --git a/CLAUDE.md b/CLAUDE.md index 9b90d79..85b0233 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,12 +236,86 @@ If a function is private but needs testing, add a `[lib]` target to the crate an Every file under `tests/` must start with `#![cfg(test)]` (placed after the file's `//!` doc comment, if any). +## Linting + +Lint checks must be expressed through one of the repo's linters — **never** as a +bespoke shell script, whether a standalone file or a mise task `run`. The +available linters: + +- **ast-grep** (`config/ast-grep/rules/`) — structural rules for code **and + YAML** (e.g. GitHub Actions workflows). +- **semgrep** (`config/semgrep/`) — incl. `languages: [generic]`, which works on + TOML/text (e.g. `mise-config.yml` lints `.mise/config*.toml`). +- **taplo** JSON-schemas (`config/taplo/`) — TOML structure, applied via + `taplo lint --schema` in `taplo-check`. +- plus hadolint, editorconfig-checker, typos, and action-validator for their + domains. + +ast-grep has no TOML grammar, so it **cannot** lint TOML — use a taplo schema or +a semgrep `generic` rule there. If none of the above can express a check, +propose adding a new mise-installable linter rather than scripting it by hand. + +## No `scripts/` directory + +Do not create a `scripts/` directory or drop loose shell/Python scripts in the +repo. Every script belongs in one of two places: + +- **Short and simple** → an inline `mise` task (`run = """ … """` in + `.mise/config.toml` or a `.mise/config..toml`). It stays discoverable + via `mise tasks` and runs as `mise run `. +- **More involved** → its own tool directory under `utilities/` with its own + `README.md` documenting what it does and how to run it. + ## Rust Workspace Single Cargo workspace (`Cargo.toml`). Shared dependency versions are declared in `[workspace.dependencies]`. Add new deps there, not in individual crate `[dependencies]`. +## Clippy lints + +**Never weaken or disable a lint to make code pass — not the workspace lint +config (`[workspace.lints.*]`, `.clippy.toml` thresholds, the ast-grep / taplo +rules) — without explicit operator permission.** Setting a denied lint to +`allow` or raising a threshold is a project-policy change, not a fix. If a lint +is in the way, fix the code (or justify it with a scoped +`#[expect(..., reason = "…")]`); if you believe the lint itself is wrong, stop +and ask. + +Clearing an `#[expect(...)]` that clippy reports as **unfulfilled** is normally +fine and expected — that's the intended cleanup. **The one exception is +`clippy::cognitive_complexity`** (and other macro-expansion-sensitive lints): +do **not** auto-remove those `#[expect]`s, because of the gotcha below. + +**Feature-unification gotcha.** `clippy::cognitive_complexity` can fire in the +full-workspace build but not in an isolated `cargo clippy -p `, because +`-p` enables fewer features (e.g. the `tracing` macros expand further when the +whole workspace's tracing/otel features are on). So its `#[expect]` can look +_unfulfilled_ in an isolated run yet be _required_ by CI. **Validate with the +full `mise run check`, not an isolated `-p` clippy, before touching one of these +`#[expect]`s.** (The clean fix — `[resolver] feature-unification = "workspace"` +so every build sees the same features — is still nightly-only via +`-Z feature-unification`.) + +The workspace denies a broad set of clippy lints (see `[workspace.lints.clippy]` +in `Cargo.toml`), including restriction lints. One you'll hit often: +**`clippy::single_call_fn`** fires on a private function called from exactly one +site. Do **not** inline the function just to silence it — a function that is a +distinct, named step (kept separate for readability, or that will gain more +callers) is legitimate. Keep it and annotate with `#[expect(...)]` and a real +justification: + +```rust +#[expect(clippy::single_call_fn, reason = "distinct step of X; kept separate for readability and future reuse")] +fn helper(...) { ... } +``` + +Use `#[expect(...)]` rather than `#[allow(...)]` (the workspace denies +`unfulfilled_lint_expectations`, so an `expect` that stops applying fails the +build instead of silently lingering). The same applies to other restriction +lints whose pattern is intentional in a given spot — prefer a justified +`#[expect(..., reason = "…")]` over contorting the code to dodge the lint. + ## Naming conventions - **`.map_err` wrappers must be named `map_*`.** Extension methods that diff --git a/Dockerfile.windows b/Dockerfile.windows index 42bb0ad..be02b14 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -3,13 +3,6 @@ # character to a backtick instead of the default `\`, so backslashes in Windows # paths (C:\..., trailing `C:\`) stay literal and lines continue with a backtick. -# SKETCH (UNVERIFIED): build edge-toolkit on a *bare* Windows box where mise -# supplies the entire toolchain -- the point is that nothing is hand-installed -# but mise itself. Windows containers build only on a Windows Docker host, so -# this runs solely in the `windows` job of .github/workflows/docker.yml; it -# can't be built on the Linux dev/CI machines. Treat it as a CI-iterated -# starting point, not a known-good image. -# # Staged to mirror the Linux Dockerfile so the layers cache + target the same # way: vcruntime -> build-minimal -> build -> prefetch -> precompile -> test / # server. Each stage `FROM`s the previous, so installed tools, fetched deps and @@ -61,12 +54,11 @@ USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] # Diagnostic (informational -- never fails the build): does this base image ship -# a bash? The `bash -euo pipefail` mise tasks need one; Nano Server has none, so -# mise pulls conda:m2-bash (see the os-guarded Windows tools in .mise/config.toml). -# Record which it is in the build log so the "does mise find bash" unknown is -# answered up front. +# a bash? The `bash -euo pipefail` mise tasks need one, and Nano Server doesn't +# bundle it. Record which it is in the build log so the "is bash present" unknown +# is answered up front. RUN where bash >nul 2>&1 && (echo [bash-check] pre-installed: & where bash) || ` - echo [bash-check] not pre-installed -- mise provides conda:m2-bash for tasks + echo [bash-check] bash is not pre-installed on this base image # Set HOME to the admin profile so the container has a sane home (Nano Server # leaves it unset). This also renders the config's `{{ env.HOME }}` paths; note @@ -121,30 +113,54 @@ COPY gh_token C:\gh_token # mise setup via the same task a workstation runs (see the README): setup-windows # depends on setup-all (experimental + cargo.binstall, cargo-binstall, node, -# openssl), then installs the gnu toolchain (gpg, llvm-mingw, make, rust) and -# flips rustup to the gnu host, so cargo links via llvm-mingw rather than the -# absent MSVC link.exe. The Docker-only settings stay inline: `mise settings` -# writes the global config (C:\.config\mise on Windows, not auto-trusted -- hence -# the explicit trust); aqua's signature checks (attestations/cosign/slsa) are off -# because their sigstore/TUF step can't find a cache dir here; pipx.uvx=false -# makes the pipx: backend use pip, not uv (uv's PE-resource trampoline isn't -# implemented on Nano Server, error 120). +# openssl), then installs the gnu toolchain (git, gpg, llvm-mingw, make, rust) +# and flips rustup to the gnu host, so cargo links via llvm-mingw rather than the +# absent MSVC link.exe. bash (conda:m2-bash) is installed FIRST, as a plain +# command -- Nano Server ships no bash and the mise tasks run on `bash -euo +# pipefail`, so bash has to exist before any `mise run`. (experimental is enabled +# here too, ahead of setup-all, because the conda backend needs it.) The +# Docker-only settings stay inline: `mise settings` writes the global config +# (C:\.config\mise on Windows, not auto-trusted -- hence the explicit trust); +# aqua's signature checks (attestations/cosign/slsa) are off because their +# sigstore/TUF step can't find a cache dir here; pipx.uvx=false makes the pipx: +# backend use pip, not uv (uv's PE-resource trampoline isn't implemented on Nano +# Server, error 120). RUN set /p GITHUB_TOKEN=nul` keeps exit 0). +# mise's rust tool always sets RUSTUP_TOOLCHAIN to the bare *version* (e.g. +# 1.96.0) and there's no setting to stop it -- but rustup resolves a bare version +# against the default host, which the setup-windows flip set to gnu. So these +# answer the open questions in the CI log: which host `mise exec -- rustc` really +# uses (the `host:` line -- should be ...-pc-windows-gnu), what mise resolves the +# toolchain to (`rustup show`), and whether the gnu linker + dlltool are on PATH +# while MSVC's link.exe is absent. +RUN echo [diag] env RUSTUP_TOOLCHAIN=[%RUSTUP_TOOLCHAIN%] CARGO_BUILD_TARGET=[%CARGO_BUILD_TARGET%] & ` + mise exec -- rustup show & ` + mise exec -- rustc -vV & ` + where x86_64-w64-mingw32-gcc & where x86_64-w64-mingw32-dlltool & ` + where dlltool & where link.exe & ` + ver>nul # The rest of the always-loaded tools (cargo-binstall + node + conda:openssl # already came via setup-all), now that rust links via gnu: `mise install` builds diff --git a/README.md b/README.md index dfbb430..f8fca78 100644 --- a/README.md +++ b/README.md @@ -68,31 +68,40 @@ mise run setup-linux ### Windows only -`mise install` brings the language toolchains, but on Windows a few things must -be installed first — mise can't supply them: - -- **A recent mise** (2026.6.2 or later). An older one (e.g. 2026.3) fails reading - the config with `unknown field run_auto_install`. -- **Visual Studio Build Tools** — the "Desktop development with C++" workload - (the MSVC compiler + Windows SDK) that Rust's default `x86_64-pc-windows-msvc` - target links through; without them `cargo` fails with `link.exe` not found. -- **LLVM** at `C:\Program Files\LLVM` — bindgen (pulled in by the deno web - runner's `libsqlite3-sys`) loads its `libclang.dll` from there. -- **pipx** — mise has no Windows build of it, so install it separately with - `python -m pip install pipx` (see the - [pipx Windows instructions](https://pipx.pypa.io/stable/how-to/install-pipx/)); - mise's `pipx:*` tools (e.g. semgrep) then resolve through it. - -Then run `setup-windows` — it installs the rest mise can supply (gpg so mise -verifies downloads, the llvm-mingw gnu toolchain, make and rust) and flips rust -to the `x86_64-pc-windows-gnu` host, so cargo links via llvm-mingw. (That gnu -path is why the Build Tools / LLVM above are only needed if you instead build -for the default msvc target.) Run it before the "All OS" steps below: +On Windows you install two things yourself; mise supplies everything else, the +compiler toolchain included: + +- **A recent mise** (2026.6.2 or later — an older one fails reading the config + with `unknown field run_auto_install`). mise can't install itself. +- **The Microsoft VC++ runtime** (`vcruntime140.dll`) — mise.exe and the + rust/cargo it installs are built against it. It ships with Windows and most + apps, so it's usually already there; check with `where.exe vcruntime140.dll`, + and if that finds nothing install the + [VC++ Redistributable](https://aka.ms/vs/17/release/vc_redist.x64.exe). mise + can't install it — it's what mise.exe needs to start. + +mise's tasks run on `bash -euo pipefail`, so install bash first — it's the one +step that can't itself use bash (harmless if you already have Git Bash): + +```bash +mise install conda:m2-bash +``` + +Then `mise run setup-windows` installs the rest of the gnu toolchain — git, gpg, +the llvm-mingw clang/lld/mingw-w64 (which also provides the `libclang.dll` +bindgen needs), make and rust — and flips rust to the `x86_64-pc-windows-gnu` +host so cargo links via llvm-mingw. So **neither Visual Studio Build Tools nor a +separate LLVM are needed** — mise supplies the compiler, linker and libclang. +Run it before the "All OS" steps below: ```bash mise run setup-windows ``` +pipx is not a separate prerequisite either: mise has no Windows pipx build, but +it installs with mise's own Python (`python -m pip install pipx`), after which +mise's `pipx:*` tools (e.g. semgrep) resolve through it. + ### MacOS only On MacOS, the Xcode Command Line Tools (`clang`, `git`, `make`, etc.) must be diff --git a/config/ast-grep/rules/gha-default-shell-bash.yml b/config/ast-grep/rules/gha-default-shell-bash.yml new file mode 100644 index 0000000..39111cb --- /dev/null +++ b/config/ast-grep/rules/gha-default-shell-bash.yml @@ -0,0 +1,19 @@ +id: gha-default-shell-bash +language: yaml +severity: error +message: | + Every GitHub Actions workflow must default to bash, so steps run in bash on + every runner (Windows included) without per-step `shell:`. Add: + defaults: + run: + shell: bash + (Pairs with gha-no-step-shell-bash, which forbids per-step `shell: bash` -- so + the only `shell: bash` left is this default.) +files: + - .github/workflows/*.yml +rule: + kind: stream + not: + has: + pattern: "shell: bash" + stopBy: end diff --git a/config/ast-grep/rules/gha-no-step-shell-bash.yml b/config/ast-grep/rules/gha-no-step-shell-bash.yml new file mode 100644 index 0000000..dfd299f --- /dev/null +++ b/config/ast-grep/rules/gha-no-step-shell-bash.yml @@ -0,0 +1,14 @@ +id: gha-no-step-shell-bash +language: yaml +severity: error +message: | + Don't set `shell: bash` on a workflow step -- the workflow defaults to bash + (defaults.run.shell), so it's redundant. Drop it. A step that genuinely needs + another shell (e.g. pwsh on Windows) sets that explicitly. +files: + - .github/workflows/*.yml +rule: + pattern: "shell: bash" + inside: + kind: block_sequence + stopBy: end diff --git a/config/semgrep/mise-config.yml b/config/semgrep/mise-config.yml index 3bfd21f..6276b74 100644 --- a/config/semgrep/mise-config.yml +++ b/config/semgrep/mise-config.yml @@ -10,3 +10,17 @@ rules: multi-line `"""..."""`. Keep the description short enough to fit on one line (under the 120-char limit). severity: ERROR + + - id: multiline-task-run-needs-bash-shell + languages: [generic] + paths: + include: + - "/.mise/config*.toml" + pattern-regex: 'run = ("""|'''''')\n(?:(?!\1\n).*\n)*?\1\n(?!shell = "bash -euo pipefail -c")' + message: >- + A mise task with a multiline `run` must be immediately followed by + `shell = "bash -euo pipefail -c"` (on the line right after the closing + `"""`/`'''`), so a failing command fails the task instead of being masked. + Put `shell` directly after `run`. bash is installed before any task on + Windows (see setup-windows), so even the setup tasks can use it. + severity: ERROR diff --git a/config/taplo.toml b/config/taplo.toml index f986a73..9b6b171 100644 --- a/config/taplo.toml +++ b/config/taplo.toml @@ -2,7 +2,10 @@ # taplo's baseline `taplo lint` / `format` don't honour .gitignore: `target/` # holds scratch + build artifacts (and auto-associates any stray `Cargo.toml` # with the SchemaStore Cargo schema), and `data/` holds external upstream repo -# clones whose TOML files we must never reformat. +# clones whose TOML files we must never reformat. target/ is also CLAUDE.md's +# scratch area, so scratch TOML is skipped here too -- to lint or test a scratch +# TOML, pipe it through `taplo format -` / `taplo lint -` (stdin has no path to +# exclude). exclude = ["target/**", "data/**"] [formatting] diff --git a/config/taplo/mise-run-not-array.schema.json b/config/taplo/mise-run-not-array.schema.json new file mode 100644 index 0000000..2bf550d --- /dev/null +++ b/config/taplo/mise-run-not-array.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise tasks — `run` must be a string, not an array", + "description": "`run` must be a string, not an array: taplo reorder_arrays would re-sort the commands.", + "type": "object", + "properties": { + "tasks": { + "type": "object", + "additionalProperties": { + "properties": { + "run": { + "type": "string", + "description": "use a multiline string (one command per line); an array gets re-sorted by taplo" + } + } + } + } + } +} From 0943908719c94e3a661e4ff21ce340ff72bf59ff Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 13:29:27 +0800 Subject: [PATCH 22/60] fix windows bash in docker --- .github/workflows/docker.yml | 3 +- .mise/config.toml | 12 ++++++ Dockerfile.windows | 37 +++++++++++-------- .../ast-grep/rules/gha-no-step-shell-bash.yml | 14 ------- config/ast-grep/rules/gha-no-step-shell.yml | 15 ++++++++ 5 files changed, 50 insertions(+), 31 deletions(-) delete mode 100644 config/ast-grep/rules/gha-no-step-shell-bash.yml create mode 100644 config/ast-grep/rules/gha-no-step-shell.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8e76cd7..26582ef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,9 +62,8 @@ jobs: # the build can fail connecting to the docker_engine pipe. Start it (no-op # if already running) and confirm connectivity before building. - name: Start the Docker daemon - shell: pwsh run: | - if ((Get-Service docker).Status -ne 'Running') { Start-Service docker } + sc query docker | grep -q RUNNING || net start docker docker version - name: Stage mise and the token in the build context diff --git a/.mise/config.toml b/.mise/config.toml index 5abfe94..96bb1d6 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -127,6 +127,13 @@ libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{ # is "not found in context" off-Windows and hard-fails the whole config render. py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" +# mise's task bash-resolution probes only Git Bash / standalone MSYS2 (see mise's +# task_executor.rs bash_candidates), never its own conda install -- so on a bare +# Windows box it can't find the conda:m2-bash we install and falls back to the +# WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here. The msys2 bash +# lands under the conda prefix's Library\usr\bin. Windows-only; the env entry is +# empty elsewhere, which mise ignores. +winbash = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\conda-m2-bash\latest\Library\usr\bin\bash.exe' # Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env # values below — kept as vars so those values stay on a single line. lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" @@ -192,6 +199,11 @@ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars # empty"); Dockerfile.windows sets those. CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER = "x86_64-w64-mingw32-gcc" CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS = "-Cdlltool=x86_64-w64-mingw32-dlltool" +# Point mise's task shell at the conda:m2-bash it installs (Docker + workstation), +# so `shell = "bash …"` tasks don't fall back to the Windows WSL launcher. mise +# checks MISE_BASH_PATH first and ignores it when empty, so the off-Windows empty +# value here is a safe no-op (winbash is only meaningful on Windows anyway). +MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.winbash }}{% endif %}" # bindgen (pulled in transitively by the deno web runner) needs the C # `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ # API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys diff --git a/Dockerfile.windows b/Dockerfile.windows index be02b14..c24fb91 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -111,20 +111,16 @@ COPY .mise/config.toml .mise/config.toml # throwaway job image. COPY gh_token C:\gh_token -# mise setup via the same task a workstation runs (see the README): setup-windows -# depends on setup-all (experimental + cargo.binstall, cargo-binstall, node, -# openssl), then installs the gnu toolchain (git, gpg, llvm-mingw, make, rust) -# and flips rustup to the gnu host, so cargo links via llvm-mingw rather than the -# absent MSVC link.exe. bash (conda:m2-bash) is installed FIRST, as a plain -# command -- Nano Server ships no bash and the mise tasks run on `bash -euo -# pipefail`, so bash has to exist before any `mise run`. (experimental is enabled -# here too, ahead of setup-all, because the conda backend needs it.) The -# Docker-only settings stay inline: `mise settings` writes the global config -# (C:\.config\mise on Windows, not auto-trusted -- hence the explicit trust); -# aqua's signature checks (attestations/cosign/slsa) are off because their -# sigstore/TUF step can't find a cache dir here; pipx.uvx=false makes the pipx: -# backend use pip, not uv (uv's PE-resource trampoline isn't implemented on Nano -# Server, error 120). +# mise setup. bash (conda:m2-bash) is installed FIRST, as a plain command -- +# Nano Server ships no bash and the mise tasks run on `bash -euo pipefail`, so +# bash must exist before any `mise run`. mise can't auto-find a conda bash, so +# the config's [env] sets MISE_BASH_PATH to it. The Docker-only settings stay +# inline: `mise settings` writes the global config (C:\.config\mise on Windows, +# not auto-trusted -- hence the explicit trust); experimental is enabled ahead of +# setup-all because the conda backend needs it; aqua's signature checks +# (attestations/cosign/slsa) are off (their sigstore/TUF step can't find a cache +# dir here); pipx.uvx=false makes the pipx: backend use pip, not uv (uv's +# PE-resource trampoline isn't implemented on Nano Server, error 120). RUN set /p GITHUB_TOKEN=nul` keeps exit 0 even if findstr matches nothing. +RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash" 2>nul | findstr /i "bash.exe" & ver>nul + +# setup-windows depends on setup-all (experimental + cargo.binstall, +# cargo-binstall, node, openssl), then installs the gnu toolchain (git, gpg, +# llvm-mingw, make, rust) and flips rustup to the gnu host so cargo links via +# llvm-mingw rather than the absent MSVC link.exe. Runs in bash via MISE_BASH_PATH. +RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 13:41:13 +0800 Subject: [PATCH 23/60] fix win debugging --- .github/workflows/check.yml | 7 ++++++- .github/workflows/test.yml | 2 -- Dockerfile.windows | 11 ++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 55bedeb..16269ef 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -53,7 +53,6 @@ jobs: - name: Install mise tools run: | - mise settings add idiomatic_version_file_enable_tools "[]" # Platform preinstall via the shared setup task (see .mise/config.toml): # setup-all does settings + cargo-binstall + node + conda:openssl; # setup-macos adds conda:lld. Running it before the main `mise install` @@ -73,6 +72,12 @@ jobs: # doesn't fail the whole `mise install` step. MISE_HTTP_TIMEOUT: "120" + - name: Prefetch Rust dependencies + run: mise run prefetch:rust + env: + GITHUB_TOKEN: ${{ github.token }} + CARGO_NET_RETRY: "5" + - name: Run checkers run: | mise run check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1dc7a4..fae68bc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,8 +107,6 @@ jobs: - name: Install mise tools run: | - mise settings add idiomatic_version_file_enable_tools "[]" - # Platform preinstall via the shared setup task (see .mise/config.toml). # setup-all does settings + cargo-binstall + node + conda:openssl; # setup-macos adds conda:lld. Running it before the main `mise install` diff --git a/Dockerfile.windows b/Dockerfile.windows index c24fb91..e9f2add 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -97,7 +97,11 @@ ENV MISE_JOBS=1 # is mise\bin\mise.exe) and put that on PATH so the steps below can call `mise`. COPY mise.zip C:\mise.zip RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip -ENV PATH="C:\mise\bin;${PATH}" +# Prepend mise's bin, and re-list System32 + Windows explicitly: the base image +# exposes its search path as `Path`, which Docker's case-sensitive `${PATH}` +# doesn't match, so it expands empty -- dropping System32 (findstr, net, sc, ...) +# off PATH for every step below. +ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${PATH}" WORKDIR C:\workspace # Only the always-loaded config is needed for the default tools; the @@ -133,8 +137,9 @@ RUN set /p GITHUB_TOKEN=nul` keeps exit 0 even if findstr matches nothing. -RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash" 2>nul | findstr /i "bash.exe" & ver>nul +# Library\usr\bin). `dir /s /b \bash.exe` (a cmd builtin, no findstr) lists +# every match; `& ver>nul` keeps exit 0 even when none are found. +RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\bash.exe" 2>nul & ver>nul # setup-windows depends on setup-all (experimental + cargo.binstall, # cargo-binstall, node, openssl), then installs the gnu toolchain (git, gpg, From 0987ee528196b490dba6235fe3d4812b28968716 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 13:50:25 +0800 Subject: [PATCH 24/60] fix win bash --- .mise/config.toml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 96bb1d6..9d6377a 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -55,7 +55,12 @@ cmake = "latest" # plus the `libclang.dll` bindgen loads -- found via PATH once mise activates the # tool. The MSVC CRT + Windows SDK can't come from mise, so the Windows build # targets `-gnu`, not `-msvc`. -"conda:m2-bash" = { version = "latest", os = ["windows"] } +# m2-bash is pinned (not "latest" like its siblings) because the `wbash` var +# below hard-codes its install dir, and mise resolves a "latest" spec to the +# concrete version for that dir (installs\conda-m2-bash\\...). Keep the +# version here and in `wbash` in sync; a mismatch makes MISE_BASH_PATH point at +# a missing dir and mise falls back to the unusable WSL-launcher bash. +"conda:m2-bash" = { version = "5.2.037.2", os = ["windows"] } "conda:m2-git" = { version = "latest", os = ["windows"] } "conda:m2-gnupg" = { version = "latest", os = ["windows"] } "conda:m2-make" = { version = "latest", os = ["windows"] } @@ -131,9 +136,10 @@ py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" # task_executor.rs bash_candidates), never its own conda install -- so on a bare # Windows box it can't find the conda:m2-bash we install and falls back to the # WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here. The msys2 bash -# lands under the conda prefix's Library\usr\bin. Windows-only; the env entry is -# empty elsewhere, which mise ignores. -winbash = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\conda-m2-bash\latest\Library\usr\bin\bash.exe' +# lands under the conda prefix's Library\usr\bin, keyed by the pinned m2-bash +# version (see [tools] above — keep the two in sync). Windows-only; the env entry +# is empty elsewhere, which mise ignores. +wbash = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe' # Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env # values below — kept as vars so those values stay on a single line. lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" @@ -202,8 +208,8 @@ CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS = "-Cdlltool=x86_64-w64-mingw32-dll # Point mise's task shell at the conda:m2-bash it installs (Docker + workstation), # so `shell = "bash …"` tasks don't fall back to the Windows WSL launcher. mise # checks MISE_BASH_PATH first and ignores it when empty, so the off-Windows empty -# value here is a safe no-op (winbash is only meaningful on Windows anyway). -MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.winbash }}{% endif %}" +# value here is a safe no-op (wbash is only meaningful on Windows anyway). +MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.wbash }}{% endif %}" # bindgen (pulled in transitively by the deno web runner) needs the C # `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ # API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys From 6ea6e55b62f0c08608ba62fd95e329f9be679448 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:07:37 +0800 Subject: [PATCH 25/60] debug windows docker --- Dockerfile.windows | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Dockerfile.windows b/Dockerfile.windows index e9f2add..12354e3 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -88,6 +88,13 @@ ENV TEMP=C:\Temp ` # mean no source builds); the gnu Docker path does. One job at a time avoids it. ENV MISE_JOBS=1 +# Verbose mise + full Rust backtraces while the Windows build is being brought +# up: surface tool/task resolution, the exact bash mise launches, and any panic +# in the CI log. Noisy but invaluable while diagnosing the bare-Nano-Server path. +ENV MISE_VERBOSE=1 ` + MISE_LOG_LEVEL=debug ` + RUST_BACKTRACE=full + # mise.zip is staged in the build context by the windows job in # .github/workflows/docker.yml (which pins the version) and copied in here. We # don't curl a versioned URL inside the build: the classic Windows builder @@ -141,6 +148,27 @@ RUN set /p GITHUB_TOKEN=nul` keeps exit 0 even when none are found. RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\bash.exe" 2>nul & ver>nul +# Make the conda bash path explicit for the probe below (a persistent ENV so the +# %WBASH% expansions resolve at cmd parse time). Mirrors the wbash var in +# .mise/config.toml; removed once the Windows path is green. +ENV WBASH=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe + +# Diagnostic (informational -- never fails the build): localize the setup-windows +# crash (exit 0xC0000139 STATUS_ENTRYPOINT_NOT_FOUND -- a DLL-load failure). This +# is the first time the build runs bash; mise.exe under cmd has worked all along. +# (A) a bash builtin -- does bash.exe/msys-2.0.dll load on Nano Server at all? +# (B) a read-only mise call -- does a native child spawned by bash work? +# (C) the exact `mise settings` write setup-all runs first -- is *changing +# settings under bash* the failing step (the open question)? (D) an msys2 child +# (uname) -- does msys2 fork/exec work? `ldd` lists the DLLs bash imports so an +# unresolved one is visible. Each reports OK/FAIL; trailing `ver` keeps exit 0. +RUN ("%WBASH%" -c "echo [diag] A-builtin ran" && echo [diag] A OK || echo [diag] A FAIL) & ` + ("%WBASH%" -c "mise --version" && echo [diag] B OK || echo [diag] B FAIL) & ` + ("%WBASH%" -c "mise settings experimental=true" && echo [diag] C OK || echo [diag] C FAIL) & ` + ("%WBASH%" -lc "uname -a" && echo [diag] D OK || echo [diag] D FAIL) & ` + ("%WBASH%" -lc "ldd /usr/bin/bash" || echo [diag] ldd FAIL) & ` + ver>nul + # setup-windows depends on setup-all (experimental + cargo.binstall, # cargo-binstall, node, openssl), then installs the gnu toolchain (git, gpg, # llvm-mingw, make, rust) and flips rustup to the gnu host so cargo links via From fc753d37f06802bc96e7408bb80082e54d6a5ebc Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:22:07 +0800 Subject: [PATCH 26/60] debug nano bash: dump conda m2-bash tree + retry with runtime dirs on PATH bash.exe crashes with 0xC0000139 (STATUS_ENTRYPOINT_NOT_FOUND) on Nano Server -- a loaded DLL is present but missing an export, not a missing DLL. Replace the now-answered A/B/C/D probe (bare `bash -c echo` already proven to crash) with data gathering: dump the full conda m2-bash tree and its DLLs to see whether the msys-* runtime is complete and where it sits, and retry a bare bash builtin with Library\usr\bin + Library\bin on PATH in case mise's absolute-path launch left the runtime off the loader search path. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 12354e3..6cd4e42 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -148,25 +148,30 @@ RUN set /p GITHUB_TOKEN=nul` keeps exit 0 even when none are found. RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\bash.exe" 2>nul & ver>nul -# Make the conda bash path explicit for the probe below (a persistent ENV so the -# %WBASH% expansions resolve at cmd parse time). Mirrors the wbash var in -# .mise/config.toml; removed once the Windows path is green. +# Persistent ENVs so the %...% expansions below resolve at cmd parse time (a `set` +# in the same RUN would expand too late). WBASH mirrors the wbash var in +# .mise/config.toml; the two bin dirs are where the conda msys2 runtime DLLs live. +# Removed once the Windows path is green. ENV WBASH=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe +ENV CBASHBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin +ENV CBASHBIN2=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\bin -# Diagnostic (informational -- never fails the build): localize the setup-windows -# crash (exit 0xC0000139 STATUS_ENTRYPOINT_NOT_FOUND -- a DLL-load failure). This -# is the first time the build runs bash; mise.exe under cmd has worked all along. -# (A) a bash builtin -- does bash.exe/msys-2.0.dll load on Nano Server at all? -# (B) a read-only mise call -- does a native child spawned by bash work? -# (C) the exact `mise settings` write setup-all runs first -- is *changing -# settings under bash* the failing step (the open question)? (D) an msys2 child -# (uname) -- does msys2 fork/exec work? `ldd` lists the DLLs bash imports so an -# unresolved one is visible. Each reports OK/FAIL; trailing `ver` keeps exit 0. -RUN ("%WBASH%" -c "echo [diag] A-builtin ran" && echo [diag] A OK || echo [diag] A FAIL) & ` - ("%WBASH%" -c "mise --version" && echo [diag] B OK || echo [diag] B FAIL) & ` - ("%WBASH%" -c "mise settings experimental=true" && echo [diag] C OK || echo [diag] C FAIL) & ` - ("%WBASH%" -lc "uname -a" && echo [diag] D OK || echo [diag] D FAIL) & ` - ("%WBASH%" -lc "ldd /usr/bin/bash" || echo [diag] ldd FAIL) & ` +# Diagnostic (informational -- never fails the build): bash.exe crashed with +# 0xC0000139 (entry point not found -- a DLL is present but missing an export, so +# a runtime/version mismatch or an unresolved import-tree dep, NOT a plain missing +# DLL). Gather the facts to fix it on Nano: (1) the full conda m2-bash tree and +# (2) every .dll in it, to see whether msys-2.0.dll + the msys-* runtime +# (readline/ncurses/intl) are present and in which dir. Then (3) re-run a bare +# bash builtin with the conda Library\usr\bin + Library\bin dirs prepended to +# PATH, in case mise launching bash by absolute path just left its runtime DLLs +# off the loader's search path. `ver` keeps the RUN green. +RUN echo [tree] all conda-m2-bash files: & ` + dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash" & ` + echo [dlls] DLLs under conda-m2-bash: & ` + dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\*.dll" & ` + ver>nul +RUN set "PATH=%CBASHBIN%;%CBASHBIN2%;%PATH%" & ` + ("%WBASH%" -c "echo [diag2] bash ran with conda bin dirs on PATH" && echo [diag2] OK || echo [diag2] FAIL) & ` ver>nul # setup-windows depends on setup-all (experimental + cargo.binstall, From cad09f9545574e25836328bcc28ce9aff8962c83 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:27:22 +0800 Subject: [PATCH 27/60] debug nano bash: dump msys-2.0.dll imports with llvm-readobj The tree dump proved msys-2.0.dll (v3.6) sits next to bash.exe and a PATH-augmented retry still crashed with 0xC0000139, so msys-2.0.dll imports a Win32 entry point Nano Server doesn't export. Install llvm-mingw (static native toolchain that runs on Nano) just to dump msys-2.0.dll's and bash.exe's imported DLLs/symbols, naming the exact system DLL to backfill from the servercore donor. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 6cd4e42..0e3499a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -150,28 +150,21 @@ RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\bash.exe" 2>nul & ver> # Persistent ENVs so the %...% expansions below resolve at cmd parse time (a `set` # in the same RUN would expand too late). WBASH mirrors the wbash var in -# .mise/config.toml; the two bin dirs are where the conda msys2 runtime DLLs live. -# Removed once the Windows path is green. +# .mise/config.toml; CBASHBIN is the dir holding bash.exe + msys-2.0.dll. Removed +# once the Windows path is green. ENV WBASH=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe ENV CBASHBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin -ENV CBASHBIN2=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\bin -# Diagnostic (informational -- never fails the build): bash.exe crashed with -# 0xC0000139 (entry point not found -- a DLL is present but missing an export, so -# a runtime/version mismatch or an unresolved import-tree dep, NOT a plain missing -# DLL). Gather the facts to fix it on Nano: (1) the full conda m2-bash tree and -# (2) every .dll in it, to see whether msys-2.0.dll + the msys-* runtime -# (readline/ncurses/intl) are present and in which dir. Then (3) re-run a bare -# bash builtin with the conda Library\usr\bin + Library\bin dirs prepended to -# PATH, in case mise launching bash by absolute path just left its runtime DLLs -# off the loader's search path. `ver` keeps the RUN green. -RUN echo [tree] all conda-m2-bash files: & ` - dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash" & ` - echo [dlls] DLLs under conda-m2-bash: & ` - dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\*.dll" & ` - ver>nul -RUN set "PATH=%CBASHBIN%;%CBASHBIN2%;%PATH%" & ` - ("%WBASH%" -c "echo [diag2] bash ran with conda bin dirs on PATH" && echo [diag2] OK || echo [diag2] FAIL) & ` +# Diagnostic (informational -- never fails the build): the tree dump confirmed +# msys-2.0.dll (v3.6) sits right next to bash.exe and a PATH-augmented retry still +# 0xC0000139'd, so msys-2.0.dll imports a Win32 entry point Nano Server doesn't +# export. Install llvm-mingw (a static, native toolchain that runs on Nano) purely +# to dump msys-2.0.dll's + bash.exe's imported DLLs/symbols with llvm-readobj -- +# that names the exact system DLL to backfill from the servercore donor stage. +RUN mise install github:mstorsjo/llvm-mingw +RUN for /f "delims=" %i in ('dir /s /b "%LOCALAPPDATA%\mise\installs\llvm-mingw\llvm-readobj.exe" 2^>nul') do ` + (echo [readobj] msys-2.0.dll imports: & "%i" --coff-imports "%CBASHBIN%\msys-2.0.dll") & ` + (echo [readobj] bash.exe imports: & "%i" --coff-imports "%WBASH%") & ` ver>nul # setup-windows depends on setup-all (experimental + cargo.binstall, From eb45da41144b07f4a50d70c1e9f2aff07ed8009e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:33:04 +0800 Subject: [PATCH 28/60] debug nano bash: fix llvm-readobj path (github- backend dir) mise's github: backend installs under installs\github-mstorsjo-llvm-mingw\ \bin, not \llvm-mingw\, so the prior dir match was empty and the empty for-loop left errorlevel 1 (skipping ver>/dev/null). Point RODIR at the real versioned bin dir and call llvm-readobj directly, with ver>/dev/null outside any loop so the diagnostic always exits 0. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 0e3499a..ff26feb 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -162,9 +162,13 @@ ENV CBASHBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m # to dump msys-2.0.dll's + bash.exe's imported DLLs/symbols with llvm-readobj -- # that names the exact system DLL to backfill from the servercore donor stage. RUN mise install github:mstorsjo/llvm-mingw -RUN for /f "delims=" %i in ('dir /s /b "%LOCALAPPDATA%\mise\installs\llvm-mingw\llvm-readobj.exe" 2^>nul') do ` - (echo [readobj] msys-2.0.dll imports: & "%i" --coff-imports "%CBASHBIN%\msys-2.0.dll") & ` - (echo [readobj] bash.exe imports: & "%i" --coff-imports "%WBASH%") & ` +# mise's github: backend installs under installs\github--\\bin +# (not \llvm-mingw\). RODIR is that bin dir for the version @latest resolved to. +ENV RODIR=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-mstorsjo-llvm-mingw\20260602\bin +RUN echo [readobj] msys-2.0.dll imports: & ` + "%RODIR%\llvm-readobj.exe" --coff-imports "%CBASHBIN%\msys-2.0.dll" & ` + echo [readobj] bash.exe imports: & ` + "%RODIR%\llvm-readobj.exe" --coff-imports "%WBASH%" & ` ver>nul # setup-windows depends on setup-all (experimental + cargo.binstall, From e1708a6ba82b46dd8b5d913261ebc1948c10a28d Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:52:05 +0800 Subject: [PATCH 29/60] nano bash: backfill kernel32 IdnToAscii/IdnToUnicode from servercore Root cause of the 0xC0000139 (STATUS_ENTRYPOINT_NOT_FOUND) bash crash on Nano Server, found by extracting msys-2.0.dll + bash.exe from the conda packages, pulling Nano's full System32 DLL set + api-set schema from mcr, and transitively resolving every import: msys-2.0.dll imports exactly two symbols Nano's stripped kernel32 forwarder doesn't re-export -- KERNEL32.dll!IdnToAscii and !IdnToUnicode -- everything else (incl. bash.exe's own kernel32/user32 imports) resolves. The functions exist on Nano (real exports in KernelBase.dll; normaliz.dll + the api-ms-win-core-localization contract expose them); only the kernel32-named door is missing. Server Core's full kernel32.dll exports both (forwarding to normaliz/kernelbase, present here), same ltsc2022 / 10.0.20348 baseline, so drop it in over Nano's kernel32 forwarder (+ a System32 root copy). Replace the readobj diagnostics with a bash smoke-test that confirms the load in the CI log. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 47 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index ff26feb..4029d54 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -47,6 +47,21 @@ COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ +# The conda msys2 runtime (msys-2.0.dll, which bash.exe loads) imports +# KERNEL32.dll!IdnToAscii + !IdnToUnicode. Nano Server's stripped kernel32 +# forwarder doesn't re-export those two names, so bash fails to load with +# 0xC0000139 (STATUS_ENTRYPOINT_NOT_FOUND) -- every other msys/bash import +# resolves. The functions themselves DO exist on Nano (real exports in +# KernelBase.dll; normaliz.dll + the api-ms-win-core-localization contract +# expose them), it's only the kernel32-named door that's missing. Server Core's +# full kernel32.dll exports both (forwarding to normaliz/kernelbase, present +# here), so drop it in as the kernel32 the loader resolves -- both the +# forwarders\ copy (what KnownDLLs maps from on Nano) and a System32 root copy. +# Same servicing baseline (ltsc2022 / 10.0.20348) as Nano's kernelbase, and a +# kernel32's forwarder exports resolve lazily, so the extra entries are inert. +COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\forwarders\kernel32.dll +COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\kernel32.dll + # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive # documented at the top of this file). @@ -142,34 +157,12 @@ RUN set /p GITHUB_TOKEN=\bash.exe` (a cmd builtin, no findstr) lists -# every match; `& ver>nul` keeps exit 0 even when none are found. -RUN dir /s /b "%LOCALAPPDATA%\mise\installs\conda-m2-bash\bash.exe" 2>nul & ver>nul - -# Persistent ENVs so the %...% expansions below resolve at cmd parse time (a `set` -# in the same RUN would expand too late). WBASH mirrors the wbash var in -# .mise/config.toml; CBASHBIN is the dir holding bash.exe + msys-2.0.dll. Removed -# once the Windows path is green. +# Smoke test (informational -- never fails the build): prove the conda msys2 bash +# now LOADS on Nano with the kernel32 donor in place (it previously 0xC0000139'd +# on the missing IdnToAscii/IdnToUnicode kernel32 exports). If this prints FAIL, +# the kernel32 override above didn't take effect for the loader (KnownDLLs). ENV WBASH=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe -ENV CBASHBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin - -# Diagnostic (informational -- never fails the build): the tree dump confirmed -# msys-2.0.dll (v3.6) sits right next to bash.exe and a PATH-augmented retry still -# 0xC0000139'd, so msys-2.0.dll imports a Win32 entry point Nano Server doesn't -# export. Install llvm-mingw (a static, native toolchain that runs on Nano) purely -# to dump msys-2.0.dll's + bash.exe's imported DLLs/symbols with llvm-readobj -- -# that names the exact system DLL to backfill from the servercore donor stage. -RUN mise install github:mstorsjo/llvm-mingw -# mise's github: backend installs under installs\github--\\bin -# (not \llvm-mingw\). RODIR is that bin dir for the version @latest resolved to. -ENV RODIR=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-mstorsjo-llvm-mingw\20260602\bin -RUN echo [readobj] msys-2.0.dll imports: & ` - "%RODIR%\llvm-readobj.exe" --coff-imports "%CBASHBIN%\msys-2.0.dll" & ` - echo [readobj] bash.exe imports: & ` - "%RODIR%\llvm-readobj.exe" --coff-imports "%WBASH%" & ` - ver>nul +RUN ("%WBASH%" -c "echo [smoke] bash loads on nano" && echo [smoke] OK || echo [smoke] FAIL) & ver>nul # setup-windows depends on setup-all (experimental + cargo.binstall, # cargo-binstall, node, openssl), then installs the gnu toolchain (git, gpg, From 538befa85b54c60b64d455f5a14c75ccecf586c9 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 14:57:29 +0800 Subject: [PATCH 30/60] nano bash: overwrite kernel32 KnownDLL source via in-container copy docker COPY can't overwrite Nano's ACL-locked System32\forwarders\ kernel32.dll (Access denied on unlinkat), and kernel32 is a KnownDLL (*kernel32 in the registry) so a root/app-dir copy is ignored by the loader. Stage the Server Core donor kernel32.dll to a fresh path, then overwrite the forwarders copy (the KnownDLLs section source) from inside a RUN as ContainerAdministrator after an icacls grant. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 4029d54..b797ba3 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -55,12 +55,19 @@ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ # KernelBase.dll; normaliz.dll + the api-ms-win-core-localization contract # expose them), it's only the kernel32-named door that's missing. Server Core's # full kernel32.dll exports both (forwarding to normaliz/kernelbase, present -# here), so drop it in as the kernel32 the loader resolves -- both the -# forwarders\ copy (what KnownDLLs maps from on Nano) and a System32 root copy. -# Same servicing baseline (ltsc2022 / 10.0.20348) as Nano's kernelbase, and a -# kernel32's forwarder exports resolve lazily, so the extra entries are inert. -COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\forwarders\kernel32.dll -COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\kernel32.dll +# here) and is the same servicing baseline (ltsc2022 / 10.0.20348) as Nano's +# kernelbase, so use it as the kernel32 the loader resolves. kernel32 is a +# KnownDLL on Nano (*kernel32 in the registry), so a System32-root or app-dir +# copy is ignored -- we must replace the file the KnownDLLs section is built +# from: System32\forwarders\kernel32.dll. `docker COPY` can't overwrite it +# (the daemon hits Access denied on that ACL-locked system file), but an +# in-container `copy` as ContainerAdministrator can, after an icacls grant. +# Stage the donor to a fresh path first (COPY creates new files fine). +COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\stage\kernel32.dll +RUN (icacls C:\Windows\System32\forwarders\kernel32.dll /grant *S-1-5-32-544:F & ver>nul) & ` + copy /y C:\stage\kernel32.dll C:\Windows\System32\forwarders\kernel32.dll & ` + (copy /y C:\stage\kernel32.dll C:\Windows\System32\kernel32.dll & ver>nul) & ` + del /q C:\stage\kernel32.dll # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive From 9aed73c5c64c6ca97643745d56d527583efe344b Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 15:04:05 +0800 Subject: [PATCH 31/60] nano bash: drop *kernel32 KnownDLL so loader binds the donor kernel32 ContainerAdministrator can't overwrite or even re-ACL Nano's TrustedInstaller-owned System32\forwarders\kernel32.dll (the file the *kernel32 KnownDLL resolves to), so the previous icacls+copy approach was denied and bash still 0xC0000139'd. Instead drop Server Core's full kernel32.dll into System32 as a new file (docker COPY can create it) and `reg delete` the *kernel32 KnownDLL entry -- registry ACLs let Administrators edit it -- so the loader falls back to the normal search and binds the donor, which exports IdnToAscii/IdnToUnicode. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index b797ba3..885eea7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -56,18 +56,14 @@ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ # expose them), it's only the kernel32-named door that's missing. Server Core's # full kernel32.dll exports both (forwarding to normaliz/kernelbase, present # here) and is the same servicing baseline (ltsc2022 / 10.0.20348) as Nano's -# kernelbase, so use it as the kernel32 the loader resolves. kernel32 is a -# KnownDLL on Nano (*kernel32 in the registry), so a System32-root or app-dir -# copy is ignored -- we must replace the file the KnownDLLs section is built -# from: System32\forwarders\kernel32.dll. `docker COPY` can't overwrite it -# (the daemon hits Access denied on that ACL-locked system file), but an -# in-container `copy` as ContainerAdministrator can, after an icacls grant. -# Stage the donor to a fresh path first (COPY creates new files fine). -COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\stage\kernel32.dll -RUN (icacls C:\Windows\System32\forwarders\kernel32.dll /grant *S-1-5-32-544:F & ver>nul) & ` - copy /y C:\stage\kernel32.dll C:\Windows\System32\forwarders\kernel32.dll & ` - (copy /y C:\stage\kernel32.dll C:\Windows\System32\kernel32.dll & ver>nul) & ` - del /q C:\stage\kernel32.dll +# kernelbase. Drop it into System32 as a NEW file (docker COPY can create files +# there -- that's how the VC++ DLLs above land -- but can't overwrite the +# ACL-locked System32\forwarders\kernel32.dll, and neither can an in-container +# copy/icacls as ContainerAdministrator). kernel32 is a KnownDLL on Nano +# (*kernel32), which resolves to that forwarders\ file, so the root copy alone +# is ignored; the `reg delete` below (after USER switch) drops the KnownDLL so +# the loader falls back to the normal search and binds this donor instead. +COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\kernel32.dll # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive @@ -75,6 +71,14 @@ RUN (icacls C:\Windows\System32\forwarders\kernel32.dll /grant *S-1-5-32-544:F & USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] +# Remove the *kernel32 KnownDLL entry so the loader stops binding kernel32 to the +# (un-overwritable) System32\forwarders\kernel32.dll and instead resolves the +# Server Core donor we dropped into System32 above -- which exports IdnToAscii/ +# IdnToUnicode, the two symbols msys-2.0.dll needs. Registry ACLs (unlike the +# system-file ACLs) let Administrators edit this key; reg.exe ships on Nano. The +# value name is literally `*kernel32` (the asterisk is part of it), hence quoted. +RUN reg delete "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\KnownDLLs" /v "*kernel32" /f + # Diagnostic (informational -- never fails the build): does this base image ship # a bash? The `bash -euo pipefail` mise tasks need one, and Nano Server doesn't # bundle it. Record which it is in the build log so the "is bash present" unknown From 77f6a95d8da8328093aaed2eaee6f4a3bdae8e13 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 15:45:21 +0800 Subject: [PATCH 32/60] windows bash via busybox-w32 sh + POSIX task bodies msys2 bash can't load on Nano Server (msys-2.0.dll imports KERNEL32!IdnToAscii/IdnToUnicode, absent from Nano's stripped kernel32 forwarder -> 0xC0000139; the system file + registry are ACL-locked, so no backfill). Replace it with busybox-w32's `sh`: a native Win32 single exe with no cygwin runtime, so it loads on bare Nano like any plain exe. - .mise/config.toml: add `http:busybox` (windows-only, frippery.org build, renamed to `sh`, checksum-pinned); point MISE_BASH_PATH at it (winsh var); drop conda:m2-bash + the wbash var. - POSIX-ify the task bodies so real bash (Linux/macOS) and busybox ash (Windows) both run them: taplo-check loses its bash array + process-substitution (find -> temp file + xargs, grep -vxF for the rust-rest exemption); gen:dart-ws / gen-specs-check use `[ = ]` not `[[ == ]]`. Verified: all 93 task scripts pass busybox ash -n + dash -n, and taplo-check runs identically on bash (31 members, rust-rest excluded). busybox ash accepts `-euo pipefail`. - Dockerfile.windows: drop the kernel32 donor / reg-delete / brush attempts; just `mise install http:busybox` (download, no build), then setup-windows runs on busybox sh. No toolchain chicken-and-egg. Co-Authored-By: Claude Opus 4.8 --- .mise/config.dart.toml | 2 +- .mise/config.toml | 103 ++++++++++++++++++++--------------- Dockerfile.windows | 121 +++++++++++++++-------------------------- 3 files changed, 102 insertions(+), 124 deletions(-) diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 23304a4..7104de7 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -80,7 +80,7 @@ description = "Emit the Dart sealed-class WS client via dart-typegen (consumes g # regen there; the committed file (regenerated on Linux/macOS) stays # canonical. run = """ -[[ "${OS:-}" == "Windows_NT" ]] && { echo "gen:dart-ws: skipped on Windows (dart-typegen empty output)"; exit 0; } +[ "${OS:-}" = "Windows_NT" ] && { echo "gen:dart-ws: skipped on Windows (dart-typegen empty output)"; exit 0; } mkdir -p generated/dart-ws/lib dart-typegen generate -i generated/specs/ws.kdl -o generated/dart-ws/lib/ws_messages.dart dart format generated/dart-ws/lib/ws_messages.dart diff --git a/.mise/config.toml b/.mise/config.toml index 9d6377a..197727b 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -48,19 +48,14 @@ cmake = "latest" "conda:clangxx" = { version = "latest", os = ["linux"] } # Windows toolchain, all mise-provided so a bare Windows box needs nothing # preinstalled except mise itself (see Dockerfile.windows). The conda msys2 -# mirror packages give the `bash` the `bash -euo pipefail` tasks expect, `git`, -# `gpg` (so mise verifies tool downloads instead of warning), and `make` (some -# `-sys` build scripts shell out to it); llvm-mingw bundles clang + lld + the -# mingw-w64 runtime/headers/libs (the `x86_64-pc-windows-gnu` C/link toolchain) -# plus the `libclang.dll` bindgen loads -- found via PATH once mise activates the -# tool. The MSVC CRT + Windows SDK can't come from mise, so the Windows build -# targets `-gnu`, not `-msvc`. -# m2-bash is pinned (not "latest" like its siblings) because the `wbash` var -# below hard-codes its install dir, and mise resolves a "latest" spec to the -# concrete version for that dir (installs\conda-m2-bash\\...). Keep the -# version here and in `wbash` in sync; a mismatch makes MISE_BASH_PATH point at -# a missing dir and mise falls back to the unusable WSL-launcher bash. -"conda:m2-bash" = { version = "5.2.037.2", os = ["windows"] } +# mirror packages give `git`, `gpg` (so mise verifies tool downloads instead of +# warning), and `make` (some `-sys` build scripts shell out to it); llvm-mingw +# bundles clang + lld + the mingw-w64 runtime/headers/libs (the +# `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen loads +# -- found via PATH once mise activates the tool. The MSVC CRT + Windows SDK +# can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. The +# bash-shell tasks' `sh` comes from busybox, not conda msys2 bash -- see the +# [tools."http:busybox"] entry below for why. "conda:m2-git" = { version = "latest", os = ["windows"] } "conda:m2-gnupg" = { version = "latest", os = ["windows"] } "conda:m2-make" = { version = "latest", os = ["windows"] } @@ -98,6 +93,23 @@ wasm-tools = "latest" yq = "latest" gh = "latest" +# busybox-w32 provides the `sh` (POSIX ash) the bash-shell mise tasks run on, +# on Windows. It's a native Win32 single exe with no cygwin/msys runtime, so it +# loads on a bare Nano Server -- unlike conda's msys2 bash, whose msys-2.0.dll +# imports KERNEL32!IdnToAscii/IdnToUnicode, names Nano's stripped kernel32 +# forwarder doesn't re-export (-> 0xC0000139, and the system file/registry are +# ACL-locked, so they can't be backfilled). Installed via the http backend from +# the maintainer's official build and renamed to `sh`; MISE_BASH_PATH (below) +# points the task shell at it. Pinned by checksum so a silent upstream rebuild +# fails loudly instead of running unverified bytes. The tasks are POSIX sh, so +# real bash (Linux/macOS) and busybox ash (Windows) both run them. +[tools."http:busybox"] +version = "1.37.0" +os = ["windows"] +bin = "sh" +url = "https://frippery.org/files/busybox/busybox64u.exe" +checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" + [vars] # mise's conda tool install dirs, reused below by OPENSSL_DIR and the # CARGO_TARGET_* RUSTFLAGS. @@ -133,13 +145,13 @@ libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{ py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" # mise's task bash-resolution probes only Git Bash / standalone MSYS2 (see mise's -# task_executor.rs bash_candidates), never its own conda install -- so on a bare -# Windows box it can't find the conda:m2-bash we install and falls back to the -# WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here. The msys2 bash -# lands under the conda prefix's Library\usr\bin, keyed by the pinned m2-bash -# version (see [tools] above — keep the two in sync). Windows-only; the env entry -# is empty elsewhere, which mise ignores. -wbash = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\conda-m2-bash\5.2.037.2\Library\usr\bin\bash.exe' +# task_executor.rs bash_candidates), never an http-backend tool -- so on a bare +# Windows box it can't find the busybox `sh` we install and falls back to the +# WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here: the http backend +# installs the renamed `sh` binary at installs\http-busybox\\sh.exe, +# keyed by the version pinned in [tools] above (keep the two in sync). +# Windows-only; the env entry is empty elsewhere, which mise ignores. +winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\sh.exe' # Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env # values below — kept as vars so those values stay on a single line. lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" @@ -205,11 +217,13 @@ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars # empty"); Dockerfile.windows sets those. CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER = "x86_64-w64-mingw32-gcc" CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS = "-Cdlltool=x86_64-w64-mingw32-dlltool" -# Point mise's task shell at the conda:m2-bash it installs (Docker + workstation), -# so `shell = "bash …"` tasks don't fall back to the Windows WSL launcher. mise -# checks MISE_BASH_PATH first and ignores it when empty, so the off-Windows empty -# value here is a safe no-op (wbash is only meaningful on Windows anyway). -MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.wbash }}{% endif %}" +# Point mise's task shell at the busybox `sh` it installs (Docker + workstation), +# so `shell = "bash …"` tasks run on busybox ash rather than the Windows WSL +# launcher. busybox ash accepts the `-euo pipefail -c` the shell string passes +# (verified), and our task bodies are POSIX. mise checks MISE_BASH_PATH first and +# ignores it when empty, so the off-Windows empty value here is a safe no-op +# (winsh is only meaningful on Windows; Linux/macOS use their real bash). +MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.winsh }}{% endif %}" # bindgen (pulled in transitively by the deno web runner) needs the C # `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ # API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys @@ -372,11 +386,12 @@ taplo lint # `-prune` target/ and .git so find never descends into them: `-not -path` # alone still walks the whole tree, and target/debug/deps/ churns constantly # under a concurrent `cargo-check`, racing find into "No such file" errors. -# (Avoid `mapfile`: it's bash 4+ only and macOS ships bash 3.2.) -members=() -while IFS= read -r line; do - members+=("$line") -done < <(find . -path ./target -prune -o -path ./.git -prune -o -name Cargo.toml ! -path ./Cargo.toml -print) +# POSIX sh (no bash arrays / process substitution -- the tasks run on busybox +# ash on Windows): collect the paths into a temp file once and feed each schema +# run via xargs. +members="$(mktemp)" +trap 'rm -f "$members"' EXIT +find . -path ./target -prune -o -path ./.git -prune -o -name Cargo.toml ! -path ./Cargo.toml -print >"$members" # Banned deps only need to be checked at the workspace root: member # crates are required to reference deps via `workspace = true` (the @@ -385,10 +400,10 @@ done < <(find . -path ./target -prune -o -path ./.git -prune -o -name Cargo.toml taplo lint --schema "file://$PWD/config/taplo/no-banned-deps.schema.json" Cargo.toml # Schemas that apply to every member crate. -taplo lint --schema "file://$PWD/config/taplo/no-path-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/no-wildcard-or-git-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/require-workspace-deps.schema.json" "${members[@]}" -taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.json" "${members[@]}" +xargs taplo lint --schema "file://$PWD/config/taplo/no-path-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/no-wildcard-or-git-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/require-workspace-deps.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.json" <"$members" # `require-lints-section` exempts generated/rust-rest/Cargo.toml because # progenitor's emitted `src/lib.rs` contains patterns our workspace lint @@ -397,12 +412,8 @@ taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.j # on the generated source. The exemption is about the lint table, not # about the deps — its `[dependencies]` still inherits from # `[workspace.dependencies]` like every other crate. -lint_inheritable=() -for f in "${members[@]}"; do - [[ "$f" == "./generated/rust-rest/Cargo.toml" ]] && continue - lint_inheritable+=("$f") -done -taplo lint --schema "file://$PWD/config/taplo/require-lints-section.schema.json" "${lint_inheritable[@]}" +grep -vxF './generated/rust-rest/Cargo.toml' "$members" | + xargs taplo lint --schema "file://$PWD/config/taplo/require-lints-section.schema.json" # mise task `run` must be a string, not an array: taplo's reorder_arrays would # re-sort an array and scramble an ordered command sequence (use a multiline @@ -469,10 +480,12 @@ depends = ["setup-all"] run = "mise install conda:lld" [tasks.setup-windows] -# Windows preinstall, shared by Dockerfile.windows and a real workstation. bash -# (conda:m2-bash) is installed by the bootstrap that runs before any mise task on -# Windows (see Dockerfile.windows / the README), so this task's `bash -euo -# pipefail` shell works. The shared base runs first, then this installs git +# Windows preinstall, shared by Dockerfile.windows and a real workstation. The +# busybox `sh` (http:busybox) is installed by the bootstrap that runs before any +# mise task on Windows (see Dockerfile.windows / the README), so this task's +# `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. It only +# needs a download, no build, so there's no toolchain chicken-and-egg. The +# shared base runs first, then this installs git # (conda:m2-git, for cargo git deps), gpg (conda:m2-gnupg, so mise verifies # downloads), the llvm-mingw gnu C/link toolchain (+ libclang), make and rust -- # then flips rustup to the gnu host so cargo links via llvm-mingw, not MSVC @@ -656,7 +669,7 @@ description = "Fail if any checked-in artifact under generated/ is stale" # indentation differently across nightlies; pipx-driven python codegen # emits subtly different output). Linux/macOS runs are canonical. run = """ -[[ "${OS:-}" == "Windows_NT" ]] && { echo "gen-specs-check: skipped on Windows (non-deterministic)"; exit 0; } +[ "${OS:-}" = "Windows_NT" ] && { echo "gen-specs-check: skipped on Windows (non-deterministic)"; exit 0; } git diff --exit-code -- generated """ shell = "bash -euo pipefail -c" diff --git a/Dockerfile.windows b/Dockerfile.windows index 885eea7..122f87a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -47,23 +47,13 @@ COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ -# The conda msys2 runtime (msys-2.0.dll, which bash.exe loads) imports -# KERNEL32.dll!IdnToAscii + !IdnToUnicode. Nano Server's stripped kernel32 -# forwarder doesn't re-export those two names, so bash fails to load with -# 0xC0000139 (STATUS_ENTRYPOINT_NOT_FOUND) -- every other msys/bash import -# resolves. The functions themselves DO exist on Nano (real exports in -# KernelBase.dll; normaliz.dll + the api-ms-win-core-localization contract -# expose them), it's only the kernel32-named door that's missing. Server Core's -# full kernel32.dll exports both (forwarding to normaliz/kernelbase, present -# here) and is the same servicing baseline (ltsc2022 / 10.0.20348) as Nano's -# kernelbase. Drop it into System32 as a NEW file (docker COPY can create files -# there -- that's how the VC++ DLLs above land -- but can't overwrite the -# ACL-locked System32\forwarders\kernel32.dll, and neither can an in-container -# copy/icacls as ContainerAdministrator). kernel32 is a KnownDLL on Nano -# (*kernel32), which resolves to that forwarders\ file, so the root copy alone -# is ignored; the `reg delete` below (after USER switch) drops the KnownDLL so -# the loader falls back to the normal search and binds this donor instead. -COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\kernel32.dll +# No msys2 bash here: the conda msys2 runtime can't load on Nano Server (its +# msys-2.0.dll imports KERNEL32.dll!IdnToAscii/IdnToUnicode, which Nano's stripped +# kernel32 forwarder doesn't re-export -> 0xC0000139, and the system file/registry +# are ACL-locked so we can't backfill them). Instead the bash-shell mise tasks run +# on busybox-w32's `sh` (the http:busybox tool) -- a native Win32 single exe with +# no cygwin runtime, so it loads on Nano like any plain exe. It's a prebuilt +# download (no build), installed below before any bash (= busybox ash) task runs. # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive @@ -71,14 +61,6 @@ COPY --from=vcruntime C:\Windows\System32\kernel32.dll C:\Windows\System32\kerne USER ContainerAdministrator SHELL ["cmd", "/S", "/C"] -# Remove the *kernel32 KnownDLL entry so the loader stops binding kernel32 to the -# (un-overwritable) System32\forwarders\kernel32.dll and instead resolves the -# Server Core donor we dropped into System32 above -- which exports IdnToAscii/ -# IdnToUnicode, the two symbols msys-2.0.dll needs. Registry ACLs (unlike the -# system-file ACLs) let Administrators edit this key; reg.exe ships on Nano. The -# value name is literally `*kernel32` (the asterisk is part of it), hence quoted. -RUN reg delete "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\KnownDLLs" /v "*kernel32" /f - # Diagnostic (informational -- never fails the build): does this base image ship # a bash? The `bash -euo pipefail` mise tasks need one, and Nano Server doesn't # bundle it. Record which it is in the build log so the "is bash present" unknown @@ -148,71 +130,54 @@ COPY .mise/config.toml .mise/config.toml # throwaway job image. COPY gh_token C:\gh_token -# mise setup. bash (conda:m2-bash) is installed FIRST, as a plain command -- -# Nano Server ships no bash and the mise tasks run on `bash -euo pipefail`, so -# bash must exist before any `mise run`. mise can't auto-find a conda bash, so -# the config's [env] sets MISE_BASH_PATH to it. The Docker-only settings stay -# inline: `mise settings` writes the global config (C:\.config\mise on Windows, -# not auto-trusted -- hence the explicit trust); experimental is enabled ahead of -# setup-all because the conda backend needs it; aqua's signature checks -# (attestations/cosign/slsa) are off (their sigstore/TUF step can't find a cache -# dir here); pipx.uvx=false makes the pipx: backend use pip, not uv (uv's -# PE-resource trampoline isn't implemented on Nano Server, error 120). +# Docker-only mise settings, written inline in cmd (no bash needed): `mise +# settings` writes the global config (C:\.config\mise on Windows, not +# auto-trusted -- hence the explicit trust); experimental enables the conda/cargo +# backends; cargo.binstall lets cargo: tools use prebuilt binaries where they +# exist; aqua signature checks (attestations/cosign/slsa) are off (their +# sigstore/TUF step can't find a cache dir here); pipx.uvx=false makes the pipx: +# backend use pip not uv (uv's PE-resource trampoline isn't implemented on Nano). RUN set /p GITHUB_TOKEN=nul + mise trust C:\.config\mise\config.toml -# setup-windows depends on setup-all (experimental + cargo.binstall, -# cargo-binstall, node, openssl), then installs the gnu toolchain (git, gpg, -# llvm-mingw, make, rust) and flips rustup to the gnu host so cargo links via -# llvm-mingw rather than the absent MSVC link.exe. Runs in bash via MISE_BASH_PATH. +# busybox-w32 `sh` -- the shell the bash-tasks run on (see config.toml). A +# prebuilt native exe, just downloaded (no build), so it's available immediately +# and setup-windows below can run on it. MISE_BASH_PATH (config [env]) points the +# task shell here. RUN set /p GITHUB_TOKEN=nul + +# Force the gnu build target for the project cargo builds below. mise's rust tool +# sets RUSTUP_TOOLCHAIN to the bare version, which rustup resolves against the gnu +# default host setup-windows selects. CARGO_BUILD_TARGET can't live in mise [env] +# (not triple-scoped -- an empty off-Windows value breaks Linux cargo); the gnu +# linker + dlltool do, as CARGO_TARGET_X86_64_PC_WINDOWS_GNU_* (triple-scoped). ENV CARGO_BUILD_TARGET=x86_64-pc-windows-gnu -# Diagnostics (informational -- never fail the build; `& ver>nul` keeps exit 0). -# mise's rust tool always sets RUSTUP_TOOLCHAIN to the bare *version* (e.g. -# 1.96.0) and there's no setting to stop it -- but rustup resolves a bare version -# against the default host, which the setup-windows flip set to gnu. So these -# answer the open questions in the CI log: which host `mise exec -- rustc` really -# uses (the `host:` line -- should be ...-pc-windows-gnu), what mise resolves the -# toolchain to (`rustup show`), and whether the gnu linker + dlltool are on PATH -# while MSVC's link.exe is absent. -RUN echo [diag] env RUSTUP_TOOLCHAIN=[%RUSTUP_TOOLCHAIN%] CARGO_BUILD_TARGET=[%CARGO_BUILD_TARGET%] & ` - mise exec -- rustup show & ` - mise exec -- rustc -vV & ` - where x86_64-w64-mingw32-gcc & where x86_64-w64-mingw32-dlltool & ` - where dlltool & where link.exe & ` - ver>nul +# setup-windows runs on busybox sh (via MISE_BASH_PATH): setup-all (settings + +# cargo-binstall + node + openssl) then the gnu toolchain (git/gpg/llvm-mingw/ +# make/rust) + rustup flip to the gnu host, so cargo links via llvm-mingw rather +# than the absent MSVC link.exe. No build was needed for the shell itself, so +# there's no toolchain chicken-and-egg. +RUN set /p GITHUB_TOKEN= Date: Sat, 13 Jun 2026 15:49:40 +0800 Subject: [PATCH 33/60] windows busybox: name the bin sh.exe (mise writes bin verbatim) mise's http backend wrote the bin name verbatim as `sh` (no extension, like the Linux probe), so MISE_BASH_PATH=...\sh.exe didn't exist and setup-windows fell back off busybox. Set bin = "sh.exe" so the installed file matches and Windows can launch it by name. Add a one-time `dir` to the smoke step to record the actual filename in the log. Co-Authored-By: Claude Opus 4.8 --- .mise/config.toml | 4 +++- Dockerfile.windows | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 197727b..b16e183 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -106,7 +106,9 @@ gh = "latest" [tools."http:busybox"] version = "1.37.0" os = ["windows"] -bin = "sh" +# `sh.exe` (not bare `sh`): mise writes the bin name verbatim, and Windows needs +# the .exe extension to launch it by name (cmd) and for MISE_BASH_PATH below. +bin = "sh.exe" url = "https://frippery.org/files/busybox/busybox64u.exe" checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" diff --git a/Dockerfile.windows b/Dockerfile.windows index 122f87a..aed0a5e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -156,9 +156,13 @@ RUN set /p GITHUB_TOKEN=nul +RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` + ("%WINSH%" -c "echo [smoke] busybox sh runs on nano" && echo [smoke] OK || echo [smoke] FAIL) & ` + ver>nul # Force the gnu build target for the project cargo builds below. mise's rust tool # sets RUSTUP_TOOLCHAIN to the bare version, which rustup resolves against the gnu From 63d7cc1e63f36a56734801605337ab8457bb50e0 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 15:53:49 +0800 Subject: [PATCH 34/60] debug windows busybox: why `mise` is not found under busybox sh busybox sh now runs on Nano ([smoke] OK) but setup-all dies with `mise: not found` (127). Diagnose whether it's PATH parsing (busybox mis-splitting the Windows ;-PATH) or .exe-appending: print busybox's $PATH and try `mise` vs `mise.exe` vs `command -v`. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile.windows b/Dockerfile.windows index aed0a5e..cb8ad5f 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -162,6 +162,10 @@ RUN set /p GITHUB_TOKEN=nul) & ` ver>nul # Force the gnu build target for the project cargo builds below. mise's rust tool From c24ee3bf3c51d449e69fcc0fa64c40fb8c87d6e1 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 15:59:40 +0800 Subject: [PATCH 35/60] debug windows busybox: probe mise exec PATH for missing C:\mise\bin Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile.windows b/Dockerfile.windows index cb8ad5f..9961862 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -167,6 +167,11 @@ RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` ("%WINSH%" -c "mise.exe --version" && echo [diag] mise.exe OK || echo [diag] mise.exe FAIL) & ` ("%WINSH%" -c "command -v mise; command -v mise.exe; type mise" & ver>nul) & ` ver>nul +# The cmd-launched busybox above finds mise, but the task-launched one doesn't. +# Probe the env mise itself hands a child (mise exec ~ the task env): does its +# PATH still contain C:\mise\bin, and can busybox find mise there? +RUN (mise exec -- "%WINSH%" -c "echo [diag2] execPATH=[$PATH]; command -v mise || echo [diag2] NOMISE") & ` + ver>nul # Force the gnu build target for the project cargo builds below. mise's rust tool # sets RUSTUP_TOOLCHAIN to the bare version, which rustup resolves against the gnu From f6a429c26afdb1688ca80de877d73f3d2b7e10eb Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 16:04:22 +0800 Subject: [PATCH 36/60] debug windows busybox: isolate -euo pipefail vs task env for mise lookup Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 9961862..07c6ff1 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -167,10 +167,13 @@ RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` ("%WINSH%" -c "mise.exe --version" && echo [diag] mise.exe OK || echo [diag] mise.exe FAIL) & ` ("%WINSH%" -c "command -v mise; command -v mise.exe; type mise" & ver>nul) & ` ver>nul -# The cmd-launched busybox above finds mise, but the task-launched one doesn't. -# Probe the env mise itself hands a child (mise exec ~ the task env): does its -# PATH still contain C:\mise\bin, and can busybox find mise there? -RUN (mise exec -- "%WINSH%" -c "echo [diag2] execPATH=[$PATH]; command -v mise || echo [diag2] NOMISE") & ` +# The cmd-launched busybox above finds mise (with `-c`), but the task-launched +# one (`-euo pipefail -c`) doesn't. Isolate the cause: (A) -euo pipefail in cmd +# context (PATH known-good); (B) -euo pipefail in mise-exec context. If A finds +# mise, the flags are innocent and it's the task env; if A fails, -euo breaks +# busybox's PATH/command search. +RUN ("%WINSH%" -euo pipefail -c "echo [diag2A] $PATH; command -v mise || echo [diag2A] NOMISE") & ` + (mise exec -- "%WINSH%" -euo pipefail -c "command -v mise || echo [diag2B] NOMISE") & ` ver>nul # Force the gnu build target for the project cargo builds below. mise's rust tool From a1771a23a0cb70a08b8877543c9e31bb68bf783f Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 16:12:47 +0800 Subject: [PATCH 37/60] windows busybox: name the shell ash.exe so mise keeps a Windows PATH The bash-tasks died with `mise: not found` (127) under busybox: mise, when it spawns a shell it recognizes (bash/sh/zsh/fish/ksh/dash), rewrites the task PATH to msys/Cygwin form (/c/mise/bin:...). busybox-w32 is native Win32 and can't resolve those, so command lookup failed -- while the same busybox launched from cmd (Windows-form PATH) found mise fine. `ash` is not in mise's POSIX-shell list, so naming the busybox binary `ash.exe` skips that PATH conversion; busybox still selects its `ash` applet from argv[0], and gets the Windows-form PATH it understands. Update the http:busybox bin, the winsh var, and MISE_BASH_PATH/WINSH accordingly; collapse the diagnostics into one smoke that checks busybox can find mise. Co-Authored-By: Claude Opus 4.8 --- .mise/config.toml | 16 +++++++++++----- Dockerfile.windows | 27 ++++++++------------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index b16e183..69e0626 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -106,9 +106,15 @@ gh = "latest" [tools."http:busybox"] version = "1.37.0" os = ["windows"] -# `sh.exe` (not bare `sh`): mise writes the bin name verbatim, and Windows needs -# the .exe extension to launch it by name (cmd) and for MISE_BASH_PATH below. -bin = "sh.exe" +# `ash.exe`, deliberately: busybox picks its applet from argv[0], and `ash` is +# the busybox shell -- but crucially mise's POSIX-shell list is +# [bash, sh, zsh, fish, ksh, dash] (NOT ash), so naming it `ash` stops mise from +# rewriting the task's PATH into msys/Cygwin form (/c/mise/bin:...) when it spawns +# the shell. busybox-w32 is native Win32 and needs the Windows-form PATH +# (C:\mise\bin;...) to find mise; the converted form breaks command lookup. The +# .exe suffix is needed because mise writes the bin name verbatim and Windows +# launches by extension. +bin = "ash.exe" url = "https://frippery.org/files/busybox/busybox64u.exe" checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" @@ -150,10 +156,10 @@ py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" # task_executor.rs bash_candidates), never an http-backend tool -- so on a bare # Windows box it can't find the busybox `sh` we install and falls back to the # WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here: the http backend -# installs the renamed `sh` binary at installs\http-busybox\\sh.exe, +# installs the renamed `ash` binary at installs\http-busybox\\ash.exe, # keyed by the version pinned in [tools] above (keep the two in sync). # Windows-only; the env entry is empty elsewhere, which mise ignores. -winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\sh.exe' +winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\ash.exe' # Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env # values below — kept as vars so those values stay on a single line. lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" diff --git a/Dockerfile.windows b/Dockerfile.windows index 07c6ff1..423ed4c 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -154,26 +154,15 @@ RUN set /p GITHUB_TOKEN=nul) & ` - ver>nul -# The cmd-launched busybox above finds mise (with `-c`), but the task-launched -# one (`-euo pipefail -c`) doesn't. Isolate the cause: (A) -euo pipefail in cmd -# context (PATH known-good); (B) -euo pipefail in mise-exec context. If A finds -# mise, the flags are innocent and it's the task env; if A fails, -euo breaks -# busybox's PATH/command search. -RUN ("%WINSH%" -euo pipefail -c "echo [diag2A] $PATH; command -v mise || echo [diag2A] NOMISE") & ` - (mise exec -- "%WINSH%" -euo pipefail -c "command -v mise || echo [diag2B] NOMISE") & ` + ("%WINSH%" -euo pipefail -c "command -v mise && echo [smoke] OK || echo [smoke] NOMISE") & ` ver>nul # Force the gnu build target for the project cargo builds below. mise's rust tool From 05f207a628118c6447011de4586b22e4e6cd3c7f Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 16:43:27 +0800 Subject: [PATCH 38/60] windows: pin CARGO_HOME/RUSTUP_HOME so cargo: tools find the gnu default With bash now working (busybox ash), the build reached the final `mise install`, where the cargo: tools (taplo-cli, wasm-pack, ...) failed: `rustup could not choose a version of cargo ... no default is configured`. mise's rust tool uses RUSTUP_HOME=C:\.rustup (where setup-windows set the 1.96.0-gnu default), but mise's cargo backend spawns `cargo install` without rust's exec_env, so rustup fell back to the empty %USERPROFILE%\.rustup. Pin CARGO_HOME/RUSTUP_HOME globally so every cargo/rustup invocation sees the .rustup that has the gnu default. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile.windows b/Dockerfile.windows index 423ed4c..4d6f118 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -172,6 +172,15 @@ RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` # linker + dlltool do, as CARGO_TARGET_X86_64_PC_WINDOWS_GNU_* (triple-scoped). ENV CARGO_BUILD_TARGET=x86_64-pc-windows-gnu +# Pin CARGO_HOME/RUSTUP_HOME to the dirs mise's rust tool uses (C:\.cargo / +# C:\.rustup -- see its exec_env). setup-windows sets the gnu default toolchain in +# C:\.rustup, but when mise's cargo backend builds a `cargo:` tool it spawns +# `cargo install` WITHOUT rust's exec_env, so rustup would otherwise look in the +# empty %USERPROFILE%\.rustup and fail with "no default is configured". Setting +# these globally makes every cargo/rustup invocation find the gnu default. +ENV CARGO_HOME=C:\.cargo ` + RUSTUP_HOME=C:\.rustup + # setup-windows runs on busybox sh (via MISE_BASH_PATH): setup-all (settings + # cargo-binstall + node + openssl) then the gnu toolchain (git/gpg/llvm-mingw/ # make/rust) + rustup flip to the gnu host, so cargo links via llvm-mingw rather From c069e2808a53c6c4baf6b93648e8e2b04a154329 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 17:21:35 +0800 Subject: [PATCH 39/60] windows: put llvm-mingw bin on PATH for cargo: builds; skip taplo With RUSTUP_HOME pinned, the cargo: tools compile but fail: `error calling dlltool 'x86_64-w64-mingw32-dlltool': program not found` -- mise's cargo backend doesn't activate llvm-mingw onto the build subprocess's PATH, so rustc can't find dlltool or the gnu linker. Pin llvm-mingw to a fixed version and prepend its \bin to PATH in Dockerfile.windows. Also skip taplo on Windows: cargo:taplo-cli pulls `ring` (needs a C/asm toolchain the gnu Nano build lacks), and TOML linting runs in the Linux check job -- so exclude the tool on windows and short-circuit taplo-check there (POSIX guard, runs on busybox ash). Co-Authored-By: Claude Opus 4.8 --- .mise/config.toml | 14 ++++++++++++-- Dockerfile.windows | 10 ++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.mise/config.toml b/.mise/config.toml index 69e0626..889d64e 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -30,7 +30,10 @@ cargo-binstall = "latest" "cargo:ast-grep" = "latest" "cargo:cargo-deny" = "latest" "cargo:cargo-expand" = "latest" -"cargo:taplo-cli" = "latest" +# Not on Windows: taplo-cli pulls `ring`, whose build needs a C/asm toolchain the +# gnu Nano image doesn't provide, and TOML linting (taplo-check) is skipped on +# Windows anyway (it runs in the Linux check job). +"cargo:taplo-cli" = { version = "latest", os = ["linux", "macos"] } "cargo:wasm-pack" = "latest" "cargo:watchexec-cli" = "latest" # vfox-chromedriver's PostInstall hook fails on Windows ("syntax of the @@ -59,7 +62,10 @@ cmake = "latest" "conda:m2-git" = { version = "latest", os = ["windows"] } "conda:m2-gnupg" = { version = "latest", os = ["windows"] } "conda:m2-make" = { version = "latest", os = ["windows"] } -"github:mstorsjo/llvm-mingw" = { version = "latest", os = ["windows"], matching = "ucrt-x86_64" } +# Pinned (not "latest") so its install dir is deterministic: Dockerfile.windows +# puts \20260602\bin on PATH for the cargo-tool builds (dlltool + the +# gnu linker). Bump here and in Dockerfile.windows together. +"github:mstorsjo/llvm-mingw" = { version = "20260602", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" @@ -387,6 +393,10 @@ run = "taplo format" # When adding a new schema under `config/taplo/`, append a `taplo lint` # line here too. See `config/taplo/RULE-IDEAS.md` for the investigation. run = """ +# taplo-cli isn't installed on Windows (its `ring` dep needs a C/asm toolchain +# the gnu Nano build lacks); TOML linting runs in the Linux check job. Skip here. +[ "${OS:-}" = "Windows_NT" ] && { echo "taplo-check: skipped on Windows"; exit 0; } + # Baseline TOML validation across every .toml file the auto-config picks up. taplo lint diff --git a/Dockerfile.windows b/Dockerfile.windows index 4d6f118..c9e19a6 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -115,8 +115,14 @@ RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip # Prepend mise's bin, and re-list System32 + Windows explicitly: the base image # exposes its search path as `Path`, which Docker's case-sensitive `${PATH}` # doesn't match, so it expands empty -- dropping System32 (findstr, net, sc, ...) -# off PATH for every step below. -ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${PATH}" +# off PATH for every step below. Also put llvm-mingw's bin on PATH: the cargo: +# tool builds (windows-sys etc.) invoke `x86_64-w64-mingw32-dlltool` + the gnu +# linker by name, and mise's cargo backend doesn't activate llvm-mingw onto the +# build subprocess's PATH. The dir is the pinned-version install (config.toml); +# it doesn't exist until setup-windows installs llvm-mingw, which is fine -- the +# cargo: builds that need it run afterwards. +ENV LLVMBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-mstorsjo-llvm-mingw\20260602\bin +ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${LLVMBIN};${PATH}" WORKDIR C:\workspace # Only the always-loaded config is needed for the default tools; the From fc729f3539ace719e337a04a0d2c520c4a9dcc01 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 17:38:43 +0800 Subject: [PATCH 40/60] ci: windows runs setup-windows, not setup-all directly setup-all is the shared internal base the per-OS setup tasks depend on; it shouldn't be invoked outside the mise config. Switch the workflows' Windows arm from `mise run setup-all` to `mise run setup-windows` (which also installs the gnu toolchain + flips rustup to gnu). README already uses setup-windows. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/check.yml | 13 ++++++------- .github/workflows/test.yml | 16 +++++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 16269ef..db6470d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -53,16 +53,15 @@ jobs: - name: Install mise tools run: | - # Platform preinstall via the shared setup task (see .mise/config.toml): - # setup-all does settings + cargo-binstall + node + conda:openssl; - # setup-macos adds conda:lld. Running it before the main `mise install` - # keeps conda:openssl ahead of any parallel cargo:* openssl-sys build. - # Windows CI uses its native msvc toolchain, so it runs setup-all (the - # base), not setup-windows whose gnu-host flip is for the bare-box path. + # Each OS runs its own setup- task (never setup-all directly -- + # that's the shared internal base those tasks depend on), before the + # main `mise install` so conda:openssl lands ahead of any parallel + # cargo:* openssl-sys build. (This job only runs on ubuntu-latest; the + # other arms are kept in lockstep with test.yml.) case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; - Windows) mise run setup-all ;; + Windows) mise run setup-windows ;; esac mise install env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fae68bc..1eb0e6d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,18 +107,16 @@ jobs: - name: Install mise tools run: | - # Platform preinstall via the shared setup task (see .mise/config.toml). - # setup-all does settings + cargo-binstall + node + conda:openssl; - # setup-macos adds conda:lld. Running it before the main `mise install` - # keeps conda:openssl ahead of any parallel `cargo:*` openssl-sys build - # (which reads the project's OPENSSL_DIR and panics if include/ isn't - # populated yet). Windows CI uses its native msvc toolchain, so it runs - # setup-all (the base), not setup-windows whose gnu-host flip is for the - # bare-box/Docker path. + # Each OS runs its own setup- task (never setup-all directly -- + # that's the shared internal base those tasks depend on). They run + # before the main `mise install` so conda:openssl lands ahead of any + # parallel `cargo:*` openssl-sys build (which reads the project's + # OPENSSL_DIR and panics if include/ isn't populated yet). setup-windows + # also installs the gnu toolchain + flips rustup to the gnu host. case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; - Windows) mise run setup-all ;; + Windows) mise run setup-windows ;; esac mise install From 1ea0d31b6266f7816b3e05a8a95a399626ddf1b8 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 19:24:00 +0800 Subject: [PATCH 41/60] split os config --- .github/workflows/check.yml | 9 +- .github/workflows/dependencies.yml | 5 +- .github/workflows/docker.yml | 3 +- .github/workflows/test.yml | 11 +- .mise/config.linux.toml | 45 +++ .mise/config.macos.toml | 37 +++ .mise/config.toml | 288 ++++-------------- .mise/config.windows.toml | 85 ++++++ .miserc.toml | 8 + CLAUDE.md | 19 ++ Dockerfile.windows | 74 +++-- README.md | 38 +-- .../ast-grep/rules/gha-default-shell-bash.yml | 19 +- config/hadolint.yaml | 6 +- 14 files changed, 330 insertions(+), 317 deletions(-) create mode 100644 .mise/config.linux.toml create mode 100644 .mise/config.macos.toml create mode 100644 .mise/config.windows.toml create mode 100644 .miserc.toml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index db6470d..5459329 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -53,11 +53,10 @@ jobs: - name: Install mise tools run: | - # Each OS runs its own setup- task (never setup-all directly -- - # that's the shared internal base those tasks depend on), before the - # main `mise install` so conda:openssl lands ahead of any parallel - # cargo:* openssl-sys build. (This job only runs on ubuntu-latest; the - # other arms are kept in lockstep with test.yml.) + # Each OS runs its own setup- task, before the main `mise install` + # so conda:openssl lands ahead of any parallel cargo:* openssl-sys + # build. (This job only runs on ubuntu-latest; the other arms are kept + # in lockstep with test.yml.) case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 6019aea..bbe7c9b 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -50,11 +50,8 @@ jobs: - name: cargo deny check run: mise run cargo-deny-check - # osv-scanner runs directly, not via `mise run osv-scanner`: that task - # depends on `cargo-check` (a full workspace build) for local use, which - # this job intentionally skips -- it only scans the committed lockfile. - name: osv-scanner - run: osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml + run: mise run osv-scanner # `cargo unmaintained` persists per-repository archival/last-commit # lookups under `$XDG_CACHE_HOME/cargo-unmaintained` (default diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 26582ef..e5b71da 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,7 +53,8 @@ jobs: env: # The classic Windows builder can't substitute build-args into the Dockerfile's RUN, # and mise's prebuilt "latest" zip is stale (2026.3.0, too old for the config). - MISE_VERSION: "2026.6.2" + # 2026.6.5 is the first release with auto_env (loads .mise/config.windows.toml). + MISE_VERSION: "2026.6.5" steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1eb0e6d..a31f57a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,12 +107,11 @@ jobs: - name: Install mise tools run: | - # Each OS runs its own setup- task (never setup-all directly -- - # that's the shared internal base those tasks depend on). They run - # before the main `mise install` so conda:openssl lands ahead of any - # parallel `cargo:*` openssl-sys build (which reads the project's - # OPENSSL_DIR and panics if include/ isn't populated yet). setup-windows - # also installs the gnu toolchain + flips rustup to the gnu host. + # Each OS runs its own setup- task, before the main `mise install` + # so conda:openssl lands ahead of any parallel `cargo:*` openssl-sys + # build (which reads the project's OPENSSL_DIR and panics if include/ + # isn't populated yet). setup-windows also installs the gnu toolchain + + # flips rustup to the gnu host. case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml new file mode 100644 index 0000000..1ea6422 --- /dev/null +++ b/.mise/config.linux.toml @@ -0,0 +1,45 @@ +# Linux-only mise config. Loaded automatically on Linux by the platform auto_env +# feature (.miserc.toml): the `linux` platform env maps to this file. Shared vars +# (rpath_flag, pylib_flag, conda_openssl, py313_unix, …) come from config.toml, +# which loads first; PYO3_PYTHON keeps its config.toml default (py313_unix). + +[tools] +# Ships the C `libclang.so` (+ clang resource headers) that bindgen needs to +# build the deno web runner's libsqlite3-sys. A mise-only Linux box usually only +# has libclang-cpp (the C++ API); macOS uses Xcode's libclang, Windows +# llvm-mingw's. +"conda:clangxx" = "latest" + +[vars] +conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" +clang_resource = "{{ vars.conda_clangxx }}/lib/clang/22" +# conda's target-triple dir uses x86_64/aarch64; mise's arch() is x64/arm64. +conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" +clang_sysroot = "{{ vars.conda_clangxx }}/{{ vars.conda_arch }}-conda-linux-gnu/sysroot" +# bindgen loads libclang directly rather than driving the clang binary, so it +# can't auto-find its resource dir (builtins like stddef.h) or conda's bundled +# sysroot. Pass `-resource-dir` + `--sysroot` so header resolution is identical +# on every machine (a bare `-isystem` leaves builtin type macros unset and fails +# in CI). Using conda's sysroot rather than the host's /usr/include keeps bindgen +# self-contained. +bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_sysroot }}" + +[env] +# rpath/pylib flags (shared, from config.toml): OPENSSL_DIR points openssl-sys at +# conda's OpenSSL, so anything linking it records conda's libssl soname; without +# an rpath the loader only finds it when conda's soname matches the system one +# (it broke when conda bumped to libssl.so.4). pylib_flag does the same for the +# CPython lib dir. No `-fuse-ld=lld`: the system linker accepts `-Wl,-rpath` +# directly, unlike Apple clang's lld dance. +CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" +# Point clang-sys at conda's libclang.so and feed bindgen conda's resource +# headers + sysroot (see bindgen_args). +LIBCLANG_PATH = "{{ vars.conda_clangxx }}/lib" +BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.bindgen_args }}" + +[tasks.setup-linux] +# Linux preinstall: just the shared cross-platform base -- the C toolchain and +# gpg come from the distro (apt/system), not mise. +description = "Linux preinstall (the shared cross-platform base)" +depends = ["_setup_all"] diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml new file mode 100644 index 0000000..33f7280 --- /dev/null +++ b/.mise/config.macos.toml @@ -0,0 +1,37 @@ +# macOS-only mise config. Loaded automatically on macOS by the platform auto_env +# feature (.miserc.toml): the `macos` platform env maps to this file. Shared vars +# (rpath_flag, pylib_flag, py313_unix, …) come from config.toml, which loads +# first; PYO3_PYTHON keeps its config.toml default (py313_unix) here. + +[tools] +# The LLD linker (ships ld64.lld) the darwin RUSTFLAGS below use; Apple's +# /usr/bin/clang can't resolve `-fuse-ld=lld` without it. macOS-only -- Linux +# uses its system linker, Windows the llvm-mingw one. +"conda:lld" = "latest" + +[vars] +conda_lld = "{{ env.HOME }}/.local/share/mise/installs/conda-lld/latest" +# Point `-fuse-ld=` at the absolute ld64.lld so it resolves without a PATH lookup +# (the cargo subprocess mise spawns for a `cargo:` source build doesn't have +# sibling tools like conda:lld on PATH). +lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" +# Xcode's libclang (the lib dir beside `xcrun`'s clang); clang-sys doesn't always +# auto-find it on CI. The system libclang knows the SDK, so no bindgen args. +mac_libclang = "{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib" + +[env] +# Apple `/usr/bin/clang` rejects `-fuse-ld=lld` ("invalid linker name") unless it +# resolves ld64.lld; lld_flag gives the absolute path. rpath/pylib flags (shared +# with Linux, from config.toml) record conda's OpenSSL soname + the CPython lib +# dir so the runtime loader finds them. Applies to every rustc call, including +# the cargo subprocess mise spawns for a `cargo:` source build. +CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" +CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" +LIBCLANG_PATH = "{{ vars.mac_libclang }}" + +[tasks.setup-macos] +# macOS preinstall, run after Xcode's command-line tools (which mise can't +# supply): the shared base, then conda:lld -- the linker the darwin build uses. +description = "macOS preinstall: shared base + conda:lld (the linker the darwin build uses)" +depends = ["_setup_all"] +run = "mise install conda:lld" diff --git a/.mise/config.toml b/.mise/config.toml index 889d64e..23c3e34 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -11,6 +11,12 @@ # Run a check across everything: mise run check-all (or install-all) # Make a selection sticky: export MISE_ENV=dart +# Require the mise release that introduced auto_env (platform config files): +# below this, .mise/config..toml wouldn't auto-load and per-OS env (linker +# flags, LIBCLANG_PATH, the Windows shell) would silently go missing. Fail loudly +# instead. (.miserc.toml enables auto_env; CI installs latest mise.) +min_version = "2026.6.5" + [settings] # Never auto-install tools implicitly: installs are explicit (`mise install` / # `mise run install-all`, and CI/Docker install up front). This keeps cheap tasks @@ -42,32 +48,16 @@ cargo-binstall = "latest" # in the standard CI `test` flow, so skip it on Windows entirely. "chromedriver" = { version = "146", os = ["linux", "macos"] } cmake = "latest" -"conda:lld" = "latest" +# openssl-sys reads OPENSSL_DIR (below); the conda build is no-admin + identical +# across machines. The per-OS toolchains (conda:clangxx on Linux, conda:lld on +# macOS, the msys2/llvm-mingw/busybox set on Windows) live in the matching +# config..toml. "conda:openssl" = "3" -# Ships the C `libclang.so` (+ clang resource headers) that bindgen needs to -# build the deno web runner's libsqlite3-sys. Linux-only: macOS uses its -# system/Xcode libclang; Windows gets libclang from llvm-mingw (below). -# Consumed by LIBCLANG_PATH / BINDGEN_EXTRA_CLANG_ARGS. -"conda:clangxx" = { version = "latest", os = ["linux"] } -# Windows toolchain, all mise-provided so a bare Windows box needs nothing -# preinstalled except mise itself (see Dockerfile.windows). The conda msys2 -# mirror packages give `git`, `gpg` (so mise verifies tool downloads instead of -# warning), and `make` (some `-sys` build scripts shell out to it); llvm-mingw -# bundles clang + lld + the mingw-w64 runtime/headers/libs (the -# `x86_64-pc-windows-gnu` C/link toolchain) plus the `libclang.dll` bindgen loads -# -- found via PATH once mise activates the tool. The MSVC CRT + Windows SDK -# can't come from mise, so the Windows build targets `-gnu`, not `-msvc`. The -# bash-shell tasks' `sh` comes from busybox, not conda msys2 bash -- see the -# [tools."http:busybox"] entry below for why. -"conda:m2-git" = { version = "latest", os = ["windows"] } -"conda:m2-gnupg" = { version = "latest", os = ["windows"] } -"conda:m2-make" = { version = "latest", os = ["windows"] } -# Pinned (not "latest") so its install dir is deterministic: Dockerfile.windows -# puts \20260602\bin on PATH for the cargo-tool builds (dlltool + the -# gnu linker). Bump here and in Dockerfile.windows together. -"github:mstorsjo/llvm-mingw" = { version = "20260602", os = ["windows"], matching = "ucrt-x86_64" } dprint = "latest" editorconfig-checker = "latest" +# Retries a command with exponential backoff + jitter; wraps the model fetches +# below (see the `retry` var) so a transient HTTP 429 doesn't fail the build. +"github:dbohdan/recur" = "latest" "github:grok-rs/waitup" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.114" hadolint = "latest" @@ -99,76 +89,17 @@ wasm-tools = "latest" yq = "latest" gh = "latest" -# busybox-w32 provides the `sh` (POSIX ash) the bash-shell mise tasks run on, -# on Windows. It's a native Win32 single exe with no cygwin/msys runtime, so it -# loads on a bare Nano Server -- unlike conda's msys2 bash, whose msys-2.0.dll -# imports KERNEL32!IdnToAscii/IdnToUnicode, names Nano's stripped kernel32 -# forwarder doesn't re-export (-> 0xC0000139, and the system file/registry are -# ACL-locked, so they can't be backfilled). Installed via the http backend from -# the maintainer's official build and renamed to `sh`; MISE_BASH_PATH (below) -# points the task shell at it. Pinned by checksum so a silent upstream rebuild -# fails loudly instead of running unverified bytes. The tasks are POSIX sh, so -# real bash (Linux/macOS) and busybox ash (Windows) both run them. -[tools."http:busybox"] -version = "1.37.0" -os = ["windows"] -# `ash.exe`, deliberately: busybox picks its applet from argv[0], and `ash` is -# the busybox shell -- but crucially mise's POSIX-shell list is -# [bash, sh, zsh, fish, ksh, dash] (NOT ash), so naming it `ash` stops mise from -# rewriting the task's PATH into msys/Cygwin form (/c/mise/bin:...) when it spawns -# the shell. busybox-w32 is native Win32 and needs the Windows-form PATH -# (C:\mise\bin;...) to find mise; the converted form breaks command lookup. The -# .exe suffix is needed because mise writes the bin name verbatim and Windows -# launches by extension. -bin = "ash.exe" -url = "https://frippery.org/files/busybox/busybox64u.exe" -checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" - [vars] -# mise's conda tool install dirs, reused below by OPENSSL_DIR and the -# CARGO_TARGET_* RUSTFLAGS. -conda_lld = "{{ env.HOME }}/.local/share/mise/installs/conda-lld/latest" +# conda:openssl install dir, reused by OPENSSL_DIR + rpath_flag. The per-OS +# toolchain vars -- conda_lld/lld_flag + mac libclang (macOS), the clangxx + +# bindgen set (Linux), winsh + win libclang + py313_win (Windows) -- live in the +# matching config..toml. conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" -conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" -clang_resource = "{{ vars.conda_clangxx }}/lib/clang/22" -# conda's target-triple dir uses x86_64/aarch64; mise's arch() is x64/arm64. -conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" -clang_sysroot = "{{ vars.conda_clangxx }}/{{ vars.conda_arch }}-conda-linux-gnu/sysroot" -# bindgen loads libclang directly rather than driving the clang binary, so it -# can't auto-find its resource dir (builtins like stddef.h/stdint.h) or conda's -# bundled sysroot. Pass `-resource-dir` (not just `-isystem .../include`, which -# leaves the builtin type macros unset and fails in CI) + `--sysroot` so header -# resolution is identical on every machine. -bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_sysroot }}" -# macOS libclang dir: point clang-sys at Xcode's libclang (the lib dir beside -# `xcrun`'s clang); clang-sys doesn't always auto-find it on CI. Guarded so the -# exec runs only on macOS; the system libclang knows the SDK, no bindgen args. -mac_libclang = "{% if os() == 'macos' %}{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib{% endif %}" -# Windows libclang: the runner's system LLVM. (Docker's gnu build has no system -# LLVM but finds llvm-mingw's libclang on PATH, which clang-sys also searches.) -win_libclang = "C:\\Program Files\\LLVM\\bin" -nonlin_libclang = "{% if os() == 'macos' %}{{ vars.mac_libclang }}{% else %}{{ vars.win_libclang }}{% endif %}" -# LIBCLANG_PATH per OS: Linux -> conda libclang; macOS -> Xcode libclang; Windows -# -> system LLVM (clang-sys also searches PATH, so llvm-mingw works too). -libclang_path = "{% if os() == 'linux' %}{{ vars.conda_clangxx }}/lib{% else %}{{ vars.nonlin_libclang }}{% endif %}" -# mise-managed CPython 3.13 interpreter, composed into PYO3_PYTHON below. Split -# per-OS because the mise data dir differs (LOCALAPPDATA\mise on Windows, -# HOME-based elsewhere). `get_env(..., default='')` guards the Windows-only -# LOCALAPPDATA so the var still renders on Linux/macOS -- a bare `env.LOCALAPPDATA` -# is "not found in context" off-Windows and hard-fails the whole config render. -py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" +# mise-managed CPython 3.13 (Linux/macOS); the PYO3_PYTHON default below. Windows +# overrides PYO3_PYTHON with its own py313_win in config.windows.toml. py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" -# mise's task bash-resolution probes only Git Bash / standalone MSYS2 (see mise's -# task_executor.rs bash_candidates), never an http-backend tool -- so on a bare -# Windows box it can't find the busybox `sh` we install and falls back to the -# WSL-launcher bash.exe. MISE_BASH_PATH (below) points it here: the http backend -# installs the renamed `ash` binary at installs\http-busybox\\ash.exe, -# keyed by the version pinned in [tools] above (keep the two in sync). -# Windows-only; the env entry is empty elsewhere, which mise ignores. -winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\ash.exe' -# Linker RUSTFLAGS fragments, composed into the per-target CARGO_TARGET_* env -# values below — kept as vars so those values stay on a single line. -lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" +# RUSTFLAGS fragments shared by the per-OS CARGO_TARGET_* env in config..toml; +# kept as vars so those values stay single-line. rpath_flag = "-C link-arg=-Wl,-rpath,{{ vars.conda_openssl }}/lib" # rpath to the mise CPython 3.13 lib dir so the et-ws-pyo3-runner binary finds # libpython at runtime (libpython3.13.so on Linux, libpython3.13.dylib on macOS @@ -186,6 +117,12 @@ har_src = "{{ vars.har_repo }}/raw/refs/heads/main/Models/onnx/hybrid_met.onnx" har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" +# `retry` prefixes each fetch with recur for exponential-backoff-with-jitter +# retries (8 tries, 2s base doubling, capped 2m, 0-5s jitter); the *_http vars +# carry the per-host base URL + flags. Kept as vars so each task line fits. +retry = "recur --attempts 8 --delay 2s --backoff 2s --max-delay 2m --jitter 0,5s --" +hf_http = "--http-url https://huggingface.co --progress --ignore-existing" +gh_http = "--http-url https://github.com --progress --ignore-existing" [env] # Comma-separated list of every language env (one per .mise/config..toml). @@ -199,77 +136,27 @@ ALL_LANGS = "dart,dotnet,java,python,rust,zig" TAPLO_CONFIG = "{{ config_root }}/config/taplo.toml" # Use the conda:openssl install for Rust's OPENSSL_DIR so openssl-sys crate builds. OPENSSL_DIR = "{{ vars.conda_openssl }}" -# Use lld on macos as documented at https://github.com/cameron1024/dart-typegen -# Apple `/usr/bin/clang` rejects `-fuse-ld=lld` outright ("invalid linker -# name") unless it can resolve `ld64.lld` on `PATH`. These `RUSTFLAGS` apply -# to every `rustc` call — including the cargo subprocess mise spawns when a -# `cargo:` tool falls back to a source build (e.g. `cargo:taplo-cli`, whose -# QuickInstall prebuilt 404s on macOS), and mise's install-time `PATH` for -# that subprocess does NOT include sibling tools like `conda:lld`. Point -# `-fuse-ld=` at the absolute `ld64.lld` shipped by `conda:lld` so the flag -# resolves without any `PATH` lookup — same conda root the rpath link-arg -# already points into. -CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" -CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" -# Linux needs the same rpath the macOS flags above carry, for the same reason: -# OPENSSL_DIR points openssl-sys at conda's OpenSSL, so anything linking it -# (e.g. the ort-sys build script) records conda's `libssl` soname. Without an -# rpath the runtime loader only finds it when conda's soname happens to match -# the system one Ubuntu ships on the default path — which broke when conda -# bumped to `libssl.so.4` (no system equivalent). Point the loader at conda's -# own lib dir so the soname is irrelevant. No `-fuse-ld=lld`: the system -# linker on Linux accepts `-Wl,-rpath` directly, unlike Apple clang's lld dance. -CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" -CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" -# Windows targets x86_64-pc-windows-gnu (llvm-mingw -- see Dockerfile.windows / -# the README). cargo reads these CARGO_TARGET__* only when building for -# that triple, so they're inert on Linux/macOS and need no os() guard. The linker -# is llvm-mingw's gcc; -Cdlltool names llvm-mingw's dlltool for the gnu raw-dylib -# import libs (rust-lang/rust#103939). The vars that force the gnu host/target -# (RUSTUP_TOOLCHAIN, CARGO_BUILD_TARGET) can't live here -- they're not -# triple-scoped, so an empty off-Windows value breaks Linux cargo ("target was -# empty"); Dockerfile.windows sets those. -CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER = "x86_64-w64-mingw32-gcc" -CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS = "-Cdlltool=x86_64-w64-mingw32-dlltool" -# Point mise's task shell at the busybox `sh` it installs (Docker + workstation), -# so `shell = "bash …"` tasks run on busybox ash rather than the Windows WSL -# launcher. busybox ash accepts the `-euo pipefail -c` the shell string passes -# (verified), and our task bodies are POSIX. mise checks MISE_BASH_PATH first and -# ignores it when empty, so the off-Windows empty value here is a safe no-op -# (winsh is only meaningful on Windows; Linux/macOS use their real bash). -MISE_BASH_PATH = "{% if os() == 'windows' %}{{ vars.winsh }}{% endif %}" -# bindgen (pulled in transitively by the deno web runner) needs the C -# `libclang.so`. A mise-only Linux box usually only has `libclang-cpp` (the C++ -# API), so install `conda:clangxx` (above) for its `libclang.so`, point clang-sys -# at it, and feed bindgen conda's own resource headers + bundled sysroot (see -# `bindgen_args`). Using conda's sysroot rather than the host's `/usr/include` -# makes bindgen self-contained, so CI resolves headers identically to a local box -# -- no environment-specific divergence. Same conda-managed, no-admin approach as -# OPENSSL_DIR. macOS points LIBCLANG_PATH at Xcode's libclang (see mac_libclang); -# Windows uses the runner's system LLVM (Docker's gnu build falls back to -# llvm-mingw on PATH). BINDGEN args stay Linux-only -- each libclang finds its -# own headers. -LIBCLANG_PATH = "{{ vars.libclang_path }}" -BINDGEN_EXTRA_CLANG_ARGS = "{% if os() == 'linux' %}{{ vars.bindgen_args }}{% endif %}" +# Per-OS RUSTFLAGS (CARGO_TARGET__*), LIBCLANG_PATH, BINDGEN args and the +# Windows MISE_BASH_PATH now live in the config..toml files (no more os() +# guards or triple-scoped-but-cross-OS entries). + # pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from # PYO3_PYTHON, else the first python on PATH -- under mise that's the 32-bit -# Pyodide shim, so the build dies with "target architecture (64-bit) does not -# match your python interpreter (32-bit)". Pin it to the mise-managed CPython -# 3.13 (the `python` tool above). Always-loaded so `cargo check --workspace` -# builds the pyo3 runner without `MISE_ENV=python`. Direct path, not `exec()`: -# an exec runs on every command and a failure hard-fails the config render. -PYO3_PYTHON = "{% if os() == 'windows' %}{{ vars.py313_win }}{% else %}{{ vars.py313_unix }}{% endif %}" -# rclone retry tuning for the `fetch-*-rclone` model-download tasks. -# Hugging Face / GitHub raw responded HTTP 429 ("Too Many Requests") at -# least once with all three of rclone's default high-level retries -# failing inside the same second (no Retry-After honoured at the -# default `--retries-sleep 0s`). Force a 30 s pause between retries -# and bump the count. `RCLONE_*` env vars are rclone's own alternative -# to CLI flags — `rclone.conf` only configures named remotes, not -# global retry policy. +# Pyodide shim ("target architecture (64-bit) does not match ... (32-bit)"). Pin +# it to the mise-managed CPython 3.13. Always-loaded so `cargo check --workspace` +# builds the pyo3 runner without MISE_ENV=python. This is the Linux/macOS path; +# config.windows.toml overrides it with py313_win. +PYO3_PYTHON = "{{ vars.py313_unix }}" +# rclone config for the `fetch-*-rclone` model-download tasks. Those remotes +# (Hugging Face / GitHub raw) intermittently answer HTTP 429; the retry loop is +# the `recur` wrapper (the `retry` var) with exponential backoff + jitter, since +# rclone's own high-level retry is a fixed-interval sleep that can't back off. +# So disable rclone's high-level retries (`--retries 1`) to avoid compounding +# with recur — rclone's fast built-in low-level retries still apply. `RCLONE_*` +# env vars are rclone's alternative to CLI flags; `rclone.conf` only configures +# named remotes, not retry policy. RCLONE_CONFIG = "{{ config_root }}/config/rclone.conf" -RCLONE_RETRIES = "10" -RCLONE_RETRIES_SLEEP = "30s" +RCLONE_RETRIES = "1" [tasks] ec = "ec" @@ -353,13 +240,11 @@ run = "ast-grep scan --error --no-ignore hidden -c config/ast-grep/sgconfig.yml" description = "Run Semgrep generic-mode rules (e.g. Cargo.toml style)" run = "semgrep scan --config config/semgrep --error --metrics=off ." -# hadolint lints the Linux Dockerfiles. Dockerfile.windows is excluded -- its -# backtick-escape continuations + cmd/PowerShell RUNs aren't hadolint-friendly -# (hadolint's ShellCheck pass assumes POSIX sh). Ignored rules live in -# config/hadolint.yaml. [tasks.hadolint-check] -description = "Lint the Linux Dockerfiles with hadolint" -run = "hadolint --config config/hadolint.yaml Dockerfile services/ws-server/Dockerfile" +description = "Lint the Dockerfiles with hadolint" +# Discover every tracked Dockerfile (`Dockerfile`, `Dockerfile.*`, at any depth) +# so new ones are linted automatically without editing this task. +run = "git ls-files '*Dockerfile' '*Dockerfile.*' | xargs hadolint --config config/hadolint.yaml" [tasks.cargo-check] run = "cargo check --workspace" @@ -466,14 +351,15 @@ mise install aube@latest """ shell = "bash -euo pipefail -c" -[tasks.setup-all] -# Shared cross-platform preinstall -- not run directly; the per-OS setup tasks -# (setup-linux / setup-macos / setup-windows) depend on it, so each controls its -# own ordering. Enables experimental + cargo.binstall, then installs -# cargo-binstall (a prebuilt binary, safe to preinstall anywhere), node (mise -# uses it for the npm: tools) and conda:openssl (for openssl-sys) -- all needed -# on every platform. -description = "Shared cross-platform preinstall (invoked via setup-linux/macos/windows, not run directly)" +[tasks._setup_all] +# Private (leading `_`, hide = true): shared cross-platform preinstall, never run +# directly. The per-OS setup tasks (setup-linux / setup-macos / setup-windows) +# depend on it, so each controls its own ordering. Enables experimental + +# cargo.binstall, then installs cargo-binstall (a prebuilt binary, safe to +# preinstall anywhere), node (mise uses it for the npm: tools) and conda:openssl +# (for openssl-sys) -- all needed on every platform. +hide = true +description = "Private shared cross-platform preinstall (depended on by setup-linux/macos/windows)" run = """ mise settings experimental=true mise settings set cargo.binstall true @@ -483,50 +369,12 @@ mise install conda:openssl """ shell = "bash -euo pipefail -c" -[tasks.setup-linux] -# Linux preinstall: just the shared cross-platform base -- the C toolchain and -# gpg come from the distro (apt/system), not mise. -description = "Linux preinstall (the shared cross-platform base)" -depends = ["setup-all"] - -[tasks.setup-macos] -# macOS preinstall, run after Xcode's command-line tools (which mise can't -# supply): the shared base, then conda:lld -- the linker the darwin -# CARGO_TARGET_* RUSTFLAGS use. -description = "macOS preinstall: shared base + conda:lld (the linker the darwin build uses)" -depends = ["setup-all"] -run = "mise install conda:lld" - -[tasks.setup-windows] -# Windows preinstall, shared by Dockerfile.windows and a real workstation. The -# busybox `sh` (http:busybox) is installed by the bootstrap that runs before any -# mise task on Windows (see Dockerfile.windows / the README), so this task's -# `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. It only -# needs a download, no build, so there's no toolchain chicken-and-egg. The -# shared base runs first, then this installs git -# (conda:m2-git, for cargo git deps), gpg (conda:m2-gnupg, so mise verifies -# downloads), the llvm-mingw gnu C/link toolchain (+ libclang), make and rust -- -# then flips rustup to the gnu host so cargo links via llvm-mingw, not MSVC -# link.exe (no Visual Studio Build Tools needed). The prereqs mise CAN'T supply -# (a recent mise + the VC++ runtime it needs to start) are in the README's -# "Windows only" section. -description = "Windows preinstall: install the Windows toolchain (git, gpg, llvm-mingw, rust) + flip rustup to gnu" -depends = ["setup-all"] -run = """ -mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust -# Install + default to the gnu build of the EXACT rust version mise pinned (not -# the "stable" channel, which only happens to match mise's "latest"). mise sets -# RUSTUP_TOOLCHAIN to that bare version, so once the default host is gnu it -# resolves to this -gnu toolchain. -ver="$(mise exec -- rustc --version | awk '{print $2}')" -mise exec -- rustup set default-host x86_64-pc-windows-gnu -mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnu" -mise exec -- rustup default "${ver}-x86_64-pc-windows-gnu" -""" -shell = "bash -euo pipefail -c" +# The per-OS preinstall tasks (setup-linux / setup-macos / setup-windows) live in +# the config..toml files; each depends on the private _setup_all above. [tasks.osv-scanner] -depends = ["cargo-check"] +# Scans the committed Cargo.lock only -- no build dependency, so the same task +# serves both ad-hoc local use and the dependencies workflow. run = "osv-scanner --lockfile Cargo.lock --config config/osv-scanner.toml" [tasks."gen:osv-scanner"] @@ -774,15 +622,11 @@ depends = ["openobserve", "ws-server"] run = "open http://localhost:5080/" [tasks.fetch-face1-rclone] -run = """ -rclone copyto {{ vars.face1_src }} {{ vars.face1_dst }} --http-url https://huggingface.co --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.face1_src }} {{ vars.face1_dst }} {{ vars.hf_http }}" shell = "bash -euo pipefail -c" [tasks.fetch-har-motion1-rclone] -run = """ -rclone copyto {{ vars.har_src }} {{ vars.har_dst }} --http-url https://github.com --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.har_src }} {{ vars.har_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" [tasks.fetch-mnist-rclone] @@ -790,9 +634,7 @@ shell = "bash -euo pipefail -c" # wasi-nn end-to-end. Sourced from the canonical ONNX Model Zoo; size 26143 # bytes. Lands under the module's gitignored pkg/ so et-modules-service can # serve it statically alongside the .wasm. -run = """ -rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} --http-url https://github.com --progress --ignore-existing -""" +run = "{{ vars.retry }} rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" [tasks.download-models] diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml new file mode 100644 index 0000000..dd0ff2f --- /dev/null +++ b/.mise/config.windows.toml @@ -0,0 +1,85 @@ +# Windows-only mise config. Loaded automatically on Windows by the platform +# auto_env feature (enabled in .miserc.toml): the `windows` platform env maps to +# this file. Everything here is therefore Windows-scoped, so it needs no per-entry +# `os = ["windows"]` guards or `{% if os() == 'windows' %}` conditionals -- values +# defined here override the cross-platform defaults in config.toml on Windows. +# Shared vars (rpath_flag, py313_*, …) come from config.toml, which loads first. + +[tools] +# msys2 mirror packages: git (cargo git deps), gpg (so mise verifies tool +# downloads), make (some -sys build scripts shell out to it). The MSVC CRT + +# Windows SDK can't come from mise, so the build uses the LLVM mingw toolchain +# (the gnullvm target below); llvm-mingw bundles clang + lld + the mingw-w64 +# runtime/headers/libs + the libclang.dll bindgen loads. Pinned so its install +# dir is deterministic for the PATH entry in Dockerfile.windows -- bump both. +"conda:m2-git" = "latest" +"conda:m2-gnupg" = "latest" +"conda:m2-make" = "latest" +"github:mstorsjo/llvm-mingw" = { version = "20260602", matching = "ucrt-x86_64" } + +# busybox-w32 `sh` (POSIX ash) -- the shell the bash-tasks run on. A native Win32 +# single exe with no cygwin/msys runtime, so it loads on a bare Nano Server, +# unlike conda's msys2 bash whose msys-2.0.dll imports KERNEL32!IdnToAscii / +# IdnToUnicode that Nano's stripped kernel32 forwarder doesn't re-export +# (-> 0xC0000139; unfixable -- the system file + registry are ACL-locked). +# Renamed `ash.exe` so mise -- whose POSIX-shell list is bash/sh/zsh/fish/ksh/ +# dash, NOT ash -- doesn't rewrite the task PATH into msys form, which native +# busybox-w32 can't resolve. Checksum-pinned so a silent upstream rebuild fails +# loudly; `.exe` because mise writes the bin name verbatim and Windows launches +# by extension. +[tools."http:busybox"] +version = "1.37.0" +bin = "ash.exe" +url = "https://frippery.org/files/busybox/busybox64u.exe" +checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" + +[vars] +# busybox `ash` install path (for MISE_BASH_PATH), keyed by the pinned +# http:busybox version above -- keep in sync. +winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\ash.exe' +# mise-managed CPython 3.13; the mise data dir is LOCALAPPDATA\mise on Windows. +py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" +# clang-sys finds libclang on PATH (Docker's gnu build has llvm-mingw there); a +# workstation with system LLVM also works. +win_libclang = "C:\\Program Files\\LLVM\\bin" + +[env] +# Run `shell = "bash …"` tasks on busybox ash rather than the Windows WSL +# launcher (mise can't auto-find an http-backend shell). busybox ash accepts the +# `-euo pipefail -c` the shell string passes; the task bodies are POSIX. +MISE_BASH_PATH = "{{ vars.winsh }}" +# Build for the LLVM mingw target. gnullvm, not gnu: llvm-mingw ships +# compiler-rt + libunwind, not the GCC runtime (libgcc/libgcc_eh) the gnu +# target's link line demands. setup-windows flips the rustup host to gnullvm too, +# so build scripts/proc-macros (built for the host) link the same way. The linker +# is clang and the raw-dylib import tool is llvm-dlltool, both from llvm-mingw on +# PATH. These can live in [env] here (unlike config.toml) because this file only +# loads on Windows, so there's no empty off-Windows value to break Linux cargo. +CARGO_BUILD_TARGET = "x86_64-pc-windows-gnullvm" +CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_LINKER = "clang" +CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTFLAGS = "-Cdlltool=llvm-dlltool" +LIBCLANG_PATH = "{{ vars.win_libclang }}" +PYO3_PYTHON = "{{ vars.py313_win }}" + +[tasks.setup-windows] +# Windows preinstall, shared by Dockerfile.windows and a real workstation. The +# busybox `sh` (http:busybox) is installed by the cmd bootstrap that runs before +# any bash task on Windows (Dockerfile.windows / the README), so this task's +# `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. Installs +# the build utils (git/gpg/make), llvm-mingw and rust, then flips rustup's +# default host to gnullvm so both host and target link via llvm-mingw rather than +# the absent MSVC link.exe (no Visual Studio Build Tools needed). The prereqs +# mise CAN'T supply (a recent mise + the VC++ runtime) are in the README. +description = "Windows preinstall: Windows toolchain + flip rustup to the gnullvm host" +depends = ["_setup_all"] +run = """ +mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust +# Install + default to the gnullvm build of the EXACT rust version mise pinned; +# mise sets RUSTUP_TOOLCHAIN to that bare version, so once the default host is +# gnullvm it resolves to this -gnullvm toolchain. +ver="$(mise exec -- rustc --version | awk '{print $2}')" +mise exec -- rustup set default-host x86_64-pc-windows-gnullvm +mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnullvm" +mise exec -- rustup default "${ver}-x86_64-pc-windows-gnullvm" +""" +shell = "bash -euo pipefail -c" diff --git a/.miserc.toml b/.miserc.toml new file mode 100644 index 0000000..0f0a2d6 --- /dev/null +++ b/.miserc.toml @@ -0,0 +1,8 @@ +# Early-init mise settings (read before the regular config, so they can affect +# which config files load). auto_env turns on platform environments: mise then +# loads .mise/config..toml automatically for the current OS -- so +# config.windows.toml loads on Windows and config.macos.toml on macOS, with no +# manual MISE_ENV. It's harmless elsewhere (there's no config.linux.toml etc.), +# and setting it here also silences mise's 2026.12 "auto_env is becoming default" +# rollout warning. See https://mise.jdx.dev/configuration/environments.html +auto_env = true diff --git a/CLAUDE.md b/CLAUDE.md index 85b0233..e8273c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,25 @@ on a follow-up fix-up pass. The most common offenders are: long `reason = "…"` strings on lint attributes, JSON `description` fields, markdown table rows, and CI-task `description` fields. +## Document each thing exactly once + +Document each thing **once**, in the single place it is most relevant — +the code, config entry, or task it describes — and **nowhere else**. Do +not restate the same explanation in a second comment, in the README, in +this file, or in a commit message that competes with it. + +Do not even add a pointer to where something is documented ("see X", +"as described in Y", "(documented in Z)"). Such cross-references are +themselves duplication: they go stale, and they multiply the places that +must change when the thing changes. Trust the developer to find the +relevant documentation themselves — they know how to read the code, +`grep`, and follow the obvious file. + +When you find yourself about to explain something that is already +explained elsewhere, stop: either the existing spot is the right home (so +say nothing here), or this is the better home (so move it here and remove +the original). One canonical location, never two. + ## Prerequisites Install [`mise`](https://mise.jdx.dev/) with shell integration, then configure: diff --git a/Dockerfile.windows b/Dockerfile.windows index c9e19a6..ae15dfa 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -3,6 +3,12 @@ # character to a backtick instead of the default `\`, so backslashes in Windows # paths (C:\..., trailing `C:\`) stay literal and lines continue with a backtick. +# hadolint runs ShellCheck (POSIX sh) over every RUN, but these are Windows cmd, +# so it false-errors on cmd `if` (SC1072) and on `\x` in paths like C:\gh_token +# (SC1001). hadolint/hadolint#645. Suppress just those two for this file (a file- +# scoped pragma, so the Linux Dockerfiles keep full ShellCheck coverage). +# hadolint global ignore=SC1072,SC1001 + # Staged to mirror the Linux Dockerfile so the layers cache + target the same # way: vcruntime -> build-minimal -> build -> prefetch -> precompile -> test / # server. Each stage `FROM`s the previous, so installed tools, fetched deps and @@ -16,10 +22,7 @@ # GB). It has no MSI/installer stack, no PowerShell, and runs unprivileged -- so # the Visual Studio Build Tools installer can't run here at all. That's the # point: nothing is installed the Windows way. mise (a single .exe the workflow -# stages in the build context) is copied in, and `mise install` pulls the rest -- -# rust, llvm-mingw (clang/lld/mingw-w64 + libclang), bash and git (see the -# os-guarded Windows tools in .mise/config.toml). The MSVC CRT + Windows SDK -# can't come from mise, so the build targets x86_64-pc-windows-gnu. +# stages in the build context) is copied in, and `mise install` pulls the rest. # --- vcruntime: donor stage for the VC++ runtime DLLs. --- # Nano Server omits the VC++ runtime that msvc-built executables (mise.exe, and @@ -47,13 +50,8 @@ COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ -# No msys2 bash here: the conda msys2 runtime can't load on Nano Server (its -# msys-2.0.dll imports KERNEL32.dll!IdnToAscii/IdnToUnicode, which Nano's stripped -# kernel32 forwarder doesn't re-export -> 0xC0000139, and the system file/registry -# are ACL-locked so we can't backfill them). Instead the bash-shell mise tasks run -# on busybox-w32's `sh` (the http:busybox tool) -- a native Win32 single exe with -# no cygwin runtime, so it loads on Nano like any plain exe. It's a prebuilt -# download (no build), installed below before any bash (= busybox ash) task runs. +# The bash-shell mise tasks run on busybox (the http:busybox tool), installed +# below in cmd before any of them -- a prebuilt download, no build step. # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive @@ -125,9 +123,12 @@ ENV LLVMBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-m ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${LLVMBIN};${PATH}" WORKDIR C:\workspace -# Only the always-loaded config is needed for the default tools; the -# guest-language configs come in the build stage below. +# config.toml (always-loaded) + config.windows.toml (the Windows tools/env/task) +# + .miserc.toml (auto_env=true, which makes mise load config.windows.toml on +# Windows). The guest-language configs come in the build stage below. COPY .mise/config.toml .mise/config.toml +COPY .mise/config.windows.toml .mise/config.windows.toml +COPY .miserc.toml .miserc.toml # The windows job stages the GitHub token as gh_token in the build context # (lifting mise's 60-req/hr anonymous limit); we read it per-RUN rather than via @@ -153,45 +154,38 @@ RUN set /p GITHUB_TOKEN=nul -# Force the gnu build target for the project cargo builds below. mise's rust tool -# sets RUSTUP_TOOLCHAIN to the bare version, which rustup resolves against the gnu -# default host setup-windows selects. CARGO_BUILD_TARGET can't live in mise [env] -# (not triple-scoped -- an empty off-Windows value breaks Linux cargo); the gnu -# linker + dlltool do, as CARGO_TARGET_X86_64_PC_WINDOWS_GNU_* (triple-scoped). -ENV CARGO_BUILD_TARGET=x86_64-pc-windows-gnu +# CARGO_BUILD_TARGET (x86_64-pc-windows-gnullvm) + the gnullvm linker/dlltool now +# come from config.windows.toml (auto-loaded by auto_env), so they're not set +# here. # Pin CARGO_HOME/RUSTUP_HOME to the dirs mise's rust tool uses (C:\.cargo / -# C:\.rustup -- see its exec_env). setup-windows sets the gnu default toolchain in -# C:\.rustup, but when mise's cargo backend builds a `cargo:` tool it spawns -# `cargo install` WITHOUT rust's exec_env, so rustup would otherwise look in the -# empty %USERPROFILE%\.rustup and fail with "no default is configured". Setting -# these globally makes every cargo/rustup invocation find the gnu default. +# C:\.rustup -- see its exec_env). setup-windows sets the gnullvm default +# toolchain in C:\.rustup, but when mise's cargo backend builds a `cargo:` tool it +# spawns `cargo install` WITHOUT rust's exec_env, so rustup would otherwise look +# in the empty %USERPROFILE%\.rustup and fail with "no default is configured". +# Setting these globally makes every cargo/rustup invocation find the default. ENV CARGO_HOME=C:\.cargo ` RUSTUP_HOME=C:\.rustup -# setup-windows runs on busybox sh (via MISE_BASH_PATH): setup-all (settings + -# cargo-binstall + node + openssl) then the gnu toolchain (git/gpg/llvm-mingw/ -# make/rust) + rustup flip to the gnu host, so cargo links via llvm-mingw rather -# than the absent MSVC link.exe. No build was needed for the shell itself, so -# there's no toolchain chicken-and-egg. +# setup-windows runs on busybox sh (via MISE_BASH_PATH): the shared _setup_all +# base (cargo-binstall + node + openssl) then the gnu toolchain (git/gpg/ +# llvm-mingw/make/rust) + rustup flip to the gnullvm host, so cargo links via +# llvm-mingw (compiler-rt/libunwind) rather than the absent MSVC link.exe. No +# build was needed for the shell itself, so there's no toolchain chicken-and-egg. RUN set /p GITHUB_TOKEN= run -> shell: bash nesting. The `has`/`pattern` matches that +# specific three-level mapping (a stray `shell: bash` on a step does NOT +# satisfy it -- it isn't under defaults.run), and `not` turns "present" into +# "required". rule: kind: stream not: has: - pattern: "shell: bash" stopBy: end + pattern: | + defaults: + run: + shell: bash diff --git a/config/hadolint.yaml b/config/hadolint.yaml index 7218a6c..6c53554 100644 --- a/config/hadolint.yaml +++ b/config/hadolint.yaml @@ -1,7 +1,5 @@ -# hadolint config, consumed by `mise run hadolint-check`. Each entry is a -# best-practice rule we consciously skip, with the reason; drop one once it no -# longer applies. (Dockerfile.windows is not linted -- see the hadolint-check -# task in .mise/config.toml.) +# hadolint config. Each entry below is a best-practice rule we consciously skip, +# with the reason; drop one once it no longer applies. ignored: # Pinning apt package versions is brittle: they float with the Ubuntu base # image + security updates, so an exact pin breaks on every base bump. We From 93b0b73dedbfdd4ba1bc4ae653e8766c39e494d3 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 19:31:09 +0800 Subject: [PATCH 42/60] ci: pin mise@2026.6.5 in install-action tool lists taiki-e/install-action installs mise 2026.6.1, below the new min_version = "2026.6.5" floor, so every mise invocation aborts and check/test/dependencies fail at the first step. Pin the install to 2026.6.5 (the release with auto_env) so CI satisfies the floor. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/check.yml | 2 +- .github/workflows/dependencies.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5459329..e6d8e6e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -31,7 +31,7 @@ jobs: - name: Install mise uses: taiki-e/install-action@v2 with: - tool: cargo-binstall,mise + tool: cargo-binstall,mise@2026.6.5 - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index bbe7c9b..b820bd5 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -39,7 +39,7 @@ jobs: - name: Install dep-audit tools uses: taiki-e/install-action@v2 with: - tool: cargo-deny,cargo-unmaintained,mise,osv-scanner + tool: cargo-deny,cargo-unmaintained,mise@2026.6.5,osv-scanner - name: Trust mise config run: mise trust diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a31f57a..ca94237 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,7 +85,7 @@ jobs: - name: Install mise uses: taiki-e/install-action@v2 with: - tool: cargo-binstall,mise + tool: cargo-binstall,mise@2026.6.5 - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" From ab98935cfde7210fec49bd729813f81da577f412 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 20:15:53 +0800 Subject: [PATCH 43/60] ci: fix docker builds for the split os mise config Two regressions surfaced once the gnullvm link fix let the Windows build run far enough to reach tooling, and once setup-linux moved out of config.toml: - Dockerfile.windows: pipx:semgrep shells out to `pipx`, but the full `mise install` ran before pipx was pip-installed. Install python + pipx (Scripts dir on PATH) first, then the full install. - Dockerfile (linux): build-minimal copied only config.toml, so the relocated setup-linux task (now in config.linux.toml) and auto_env (.miserc.toml) were missing. Copy both into the minimal stage. Co-Authored-By: Claude Opus 4.8 --- Dockerfile | 9 ++++++--- Dockerfile.windows | 15 ++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ae239d..d80dad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,10 +71,13 @@ RUN curl -fsSL https://mise.run | sh ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" WORKDIR /workspace -# Only the always-loaded config is needed for the default tools; the -# guest-language configs come in the build stage below. setup-linux is a repo -# task, so the config has to be copied + trusted before it can run. +# The default tools need the always-loaded config plus config.linux.toml (where +# setup-linux + the Linux env live) and .miserc.toml (auto_env, which auto-loads +# config.linux.toml); the other guest-language configs come in the build stage +# below. These are repo configs, so they're copied + trusted before mise runs. +COPY .miserc.toml .miserc.toml COPY .mise/config.toml .mise/config.toml +COPY .mise/config.linux.toml .mise/config.linux.toml RUN mise trust diff --git a/Dockerfile.windows b/Dockerfile.windows index ae15dfa..19f3eaf 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -189,13 +189,18 @@ ENV CARGO_HOME=C:\.cargo ` RUN set /p GITHUB_TOKEN=.toml). --- # install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are From 35f02ded57186eb4efa2bbdb7f1a924baf5bb800 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 21:14:20 +0800 Subject: [PATCH 44/60] ci: serialize linux install + widen windows pipx PATH lookup Both docker jobs got further after the prior fixes and surfaced new failures: - Dockerfile (linux): the `rust` tool is a two-version list (latest + nightly); installed in parallel they run two rustup-inits at once and race on the shared rustup binary's self-update, so the second exits 1. MISE_JOBS=1 serializes the install. Validated with a local `docker build --target build-minimal`. - Dockerfile.windows: pip dropped pipx.exe outside the prefix Scripts dir that was on PATH, so the full `mise install` couldn't find pipx for pipx:semgrep. Add the per-user AppData Scripts dir to PATH too and log the resolved location. Co-Authored-By: Claude Opus 4.8 --- Dockerfile | 6 ++++-- Dockerfile.windows | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d80dad7..716d7ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,10 +85,12 @@ RUN mise trust # its setup-all base enables experimental + cargo.binstall and installs # cargo-binstall, node and conda:openssl; then `mise install` adds the rest of # the always-loaded tools. A GitHub token (if provided) lifts the anonymous rate -# limit for the release fetches. +# limit for the release fetches. MISE_JOBS=1 serializes the install: the `rust` +# tool is a two-version list (latest + nightly) that otherwise runs two +# rustup-inits at once, racing on the shared rustup binary's self-update (exit 1). RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ - sh -c 'mise run setup-linux && mise install' + sh -c 'mise run setup-linux && MISE_JOBS=1 mise install' # --- build: add the guest-language toolchains (config..toml). --- # install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are diff --git a/Dockerfile.windows b/Dockerfile.windows index 19f3eaf..2b66f0c 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -190,12 +190,17 @@ RUN set /p GITHUB_TOKEN=nul & ` + dir /s /b "%APPDATA%\Python\pipx.exe" 2>nul & exit /b 0) +ENV PYSCR_PREFIX=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\python\3.13\Scripts +ENV PYSCR_USER=C:\Users\ContainerAdministrator\AppData\Roaming\Python\Python313\Scripts +ENV PATH="${PYSCR_PREFIX};${PYSCR_USER};${PATH}" # The remaining always-loaded tools (the cargo: tools + pipx:semgrep) now that # rust links via gnullvm and pipx is on PATH. From d7b125c38b149f5775d8a07fb8a29b664a56201e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 13 Jun 2026 21:53:32 +0800 Subject: [PATCH 45/60] ci: copy pipx.exe to a stable PATH dir on windows pip installs pipx.exe into python's versioned Scripts dir (...\python\3.13.14\Scripts), whose patch component isn't stable, so the hardcoded ...\python\3.13\Scripts on PATH never matched and pipx:semgrep still failed with "program not found". Locate pipx.exe with `dir /s` and copy it into C:\mise\bin (already on PATH); the pip launcher .exe embeds the python path so it runs from there. Confirmed the real location via the prior build's diagnostic output. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 2b66f0c..859a59e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -190,17 +190,15 @@ RUN set /p GITHUB_TOKEN=\Scripts), and the patch component isn't stable, so locate it +# with `dir /s` rather than hardcoding the path. The pip launcher .exe embeds the +# python path, so it still works from C:\mise\bin. RUN set /p GITHUB_TOKEN=nul & ` - dir /s /b "%APPDATA%\Python\pipx.exe" 2>nul & exit /b 0) -ENV PYSCR_PREFIX=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\python\3.13\Scripts -ENV PYSCR_USER=C:\Users\ContainerAdministrator\AppData\Roaming\Python\Python313\Scripts -ENV PATH="${PYSCR_PREFIX};${PYSCR_USER};${PATH}" + mise exec -- python -m pip install --no-warn-script-location pipx && ` + for /f "delims=" %F in ('dir /s /b "%LOCALAPPDATA%\mise\installs\python\pipx.exe"') do copy /Y "%F" C:\mise\bin\ # The remaining always-loaded tools (the cargo: tools + pipx:semgrep) now that # rust links via gnullvm and pipx is on PATH. From c03950788bb1c329860ae848b86cb5945d62296e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 04:04:16 +0800 Subject: [PATCH 46/60] ci: skip pipx:semgrep in the windows image (Nano can't run pipx) pipx crashes at import on Nano Server: platformdirs.user_data_path() calls SHGetFolderPathW, a shell known-folder API Nano lacks, and it runs at module import so no pipx env var avoids it. semgrep is only ever run by `mise run check` on Linux, so disable just pipx:semgrep via MISE_DISABLE_TOOLS rather than excluding it on all Windows (real Windows dev boxes run pipx fine). Drops the now-pointless python+pipx install/copy dance; python installs with the rest via the main `mise install`. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 859a59e..89da480 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -189,19 +189,15 @@ ENV CARGO_HOME=C:\.cargo ` RUN set /p GITHUB_TOKEN=\Scripts), and the patch component isn't stable, so locate it -# with `dir /s` rather than hardcoding the path. The pip launcher .exe embeds the -# python path, so it still works from C:\mise\bin. -RUN set /p GITHUB_TOKEN= Date: Sun, 14 Jun 2026 05:06:40 +0800 Subject: [PATCH 47/60] ci(windows): install only the Nano-compatible guest langs The guest-language stage failed because Nano Server can't run dotnet's PowerShell installer or pipx (python's tools). Restrict MISE_ENV to the guests that install cleanly on Nano -- dart, java, rust, zig -- and use `mise install` instead of install-all (which forces $ALL_LANGS and would re-add dotnet + python). semgrep stays skipped via MISE_DISABLE_TOOLS. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.windows | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 89da480..575f1c3 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -202,15 +202,17 @@ RUN set /p GITHUB_TOKEN=.toml). --- -# install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are -# already installed by build-minimal, so this adds dart/dotnet/java/zig/etc. -# MISE_ENV is set persistently so the later stages act on every language too. +# MISE_ENV lists only the guests that install on Nano Server: dotnet is excluded +# (its installer is a PowerShell script, and Nano has no PowerShell) and python is +# excluded (its tools install via pipx, which can't run on Nano). dart/java/rust/ +# zig install cleanly. Use `mise install` rather than install-all (which forces +# $ALL_LANGS) so this restricted MISE_ENV is honored; it persists to later stages. FROM build-minimal AS build COPY .mise/ .mise/ RUN mise trust -ENV MISE_ENV=dart,dotnet,java,python,rust,zig +ENV MISE_ENV=dart,java,rust,zig RUN set /p GITHUB_TOKEN= Date: Sun, 14 Jun 2026 06:51:53 +0800 Subject: [PATCH 48/60] tidy & precompile on windows docker --- .dprint.jsonc | 42 +------ .github/workflows/check.yml | 8 -- .github/workflows/dependencies.yml | 7 +- .github/workflows/docker.yml | 13 +-- .github/workflows/test.yml | 9 -- .mise/config.linux.toml | 22 +++- .mise/config.macos.toml | 17 ++- .mise/config.python.toml | 1 - .mise/config.toml | 8 +- .mise/config.windows.toml | 16 ++- Cargo.lock | 18 +-- Cargo.toml | 3 +- Dockerfile.windows | 18 ++- .../ast-grep/rules/no-cargo-manifest-dir.yml | 16 +++ config/ast-grep/rules/no-current-dir.yml | 16 +++ .../rules/no-relative-path-literal.yml | 24 ++++ config/ast-grep/rules/no-std-env-var.yml | 4 +- config/dprint.jsonc | 41 +++++++ config/taplo/no-lib-name.schema.json | 14 +++ config/typos.toml | 4 + libs/edge-toolkit/Cargo.toml | 2 +- libs/edge-toolkit/src/config.rs | 16 +-- libs/path/Cargo.toml | 16 +++ libs/path/src/lib.rs | 104 ++++++++++++++++++ libs/path/tests/find.rs | 32 ++++++ libs/ws-runner-common/Cargo.toml | 1 - services/ws-modules/wasi-comm1/Cargo.toml | 4 + services/ws-modules/wasi-comm1/build.rs | 10 ++ services/ws-modules/wasi-comm1/src/lib.rs | 3 +- services/ws-modules/wasi-data1/Cargo.toml | 4 + services/ws-modules/wasi-data1/build.rs | 10 ++ services/ws-modules/wasi-data1/src/lib.rs | 3 +- services/ws-wasi-runner/Cargo.toml | 1 - services/ws-wasi-runner/src/bindings.rs | 5 + services/ws-web-runner/Cargo.toml | 1 - utilities/cli/Cargo.toml | 1 + utilities/cli/src/cli.rs | 33 ++++++ .../src/deployment_types/docker_compose.rs | 7 +- utilities/cli/src/deployment_types/mise.rs | 5 +- utilities/cli/src/lib.rs | 68 +----------- utilities/cli/src/main.rs | 35 +----- 41 files changed, 440 insertions(+), 222 deletions(-) create mode 100644 config/ast-grep/rules/no-cargo-manifest-dir.yml create mode 100644 config/ast-grep/rules/no-current-dir.yml create mode 100644 config/ast-grep/rules/no-relative-path-literal.yml create mode 100644 config/dprint.jsonc create mode 100644 config/taplo/no-lib-name.schema.json create mode 100644 config/typos.toml create mode 100644 libs/path/Cargo.toml create mode 100644 libs/path/src/lib.rs create mode 100644 libs/path/tests/find.rs create mode 100644 services/ws-modules/wasi-comm1/build.rs create mode 100644 services/ws-modules/wasi-data1/build.rs create mode 100644 utilities/cli/src/cli.rs diff --git a/.dprint.jsonc b/.dprint.jsonc index 5e30b14..41d6eee 100644 --- a/.dprint.jsonc +++ b/.dprint.jsonc @@ -1,40 +1,6 @@ +// dprint anchors its base directory (the tree it formats) to the directory of +// the config file it discovers. This stub keeps that base at the repo root while +// the real config lives in config/ alongside the other linter configs. { - "dockerfile": { - "associations": ["**/Dockerfile", "**/*.dockerfile", "**/Dockerfile.windows"], - }, - "java": { - }, - "json": { - }, - // Match the repo-wide 120 line-length set in .editorconfig and ruff.toml, - // otherwise dprint's bundled ruff would reformat Python files to its - // default and fight with `mise run ruff-fmt`. - "ruff": { - "lineLength": 120, - }, - "malva": { - }, - "markdown": { - }, - "markup": { - }, - "typescript": { - }, - "yaml": { - }, - "excludes": [ - "**/node_modules", - "**/*-lock.json", - ], - "plugins": [ - "https://github.com/speakeasy-api/dprint-plugin-java/releases/latest/download/dprint_plugin_java.wasm", - "https://plugins.dprint.dev/g-plane/malva-v0.15.2.wasm", - "https://plugins.dprint.dev/g-plane/markup_fmt-v0.27.0.wasm", - "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.6.0.wasm", - "https://plugins.dprint.dev/dockerfile-0.4.0.wasm", - "https://plugins.dprint.dev/json-0.21.3.wasm", - "https://plugins.dprint.dev/markdown-0.21.1.wasm", - "https://plugins.dprint.dev/ruff-0.7.10.wasm", - "https://plugins.dprint.dev/typescript-0.95.15.wasm", - ], + "extends": "config/dprint.jsonc", } diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e6d8e6e..3a5b34d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -36,10 +36,6 @@ jobs: - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - name: Install pipx (Windows only — aqua has no Windows build) - if: runner.os == 'Windows' - run: python -m pip install pipx - # Optional npm backend, installed before the main `mise install`. # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - name: Install aube (optional npm backend, allowed to fail) @@ -53,10 +49,6 @@ jobs: - name: Install mise tools run: | - # Each OS runs its own setup- task, before the main `mise install` - # so conda:openssl lands ahead of any parallel cargo:* openssl-sys - # build. (This job only runs on ubuntu-latest; the other arms are kept - # in lockstep with test.yml.) case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index b820bd5..5693f4e 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -31,12 +31,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # The audit binaries come from taiki-e (fast prebuilts); the checks below - # are driven through `mise run` so the commands + config paths have one - # source of truth in .mise/config.toml. mise's `task.run_auto_install = - # false` means `mise run` won't drag in the whole toolchain just to run a - # task, so this stays a lightweight, audit-only job. - - name: Install dep-audit tools + - name: Install tools uses: taiki-e/install-action@v2 with: tool: cargo-deny,cargo-unmaintained,mise@2026.6.5,osv-scanner diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e5b71da..163e1cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,7 +39,7 @@ jobs: sudo systemctl start docker docker info --format 'Docker Root Dir: {{.DockerRootDir}}' - - name: Build the edge-toolkit test image + - name: Build stage test env: GITHUB_TOKEN: ${{ github.token }} run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . @@ -67,16 +67,11 @@ jobs: sc query docker | grep -q RUNNING || net start docker docker version - - name: Stage mise and the token in the build context + - name: Prepare mise and Github token for the build context run: | v="${{ env.MISE_VERSION }}" curl -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" printf '%s' "${{ github.token }}" > gh_token - # Build only through the `build` stage (mise + the full toolchain): that's - # the current frontier -- the install pipeline is what we're proving on a - # bare Nano Server. The later stages (prefetch/precompile/test/server) - # mirror the Linux Dockerfile but aren't reached yet; bump --target as they - # come green. - - name: Build the Windows setup image - run: docker build -f Dockerfile.windows --target build -t edge-toolkit-windows . + - name: Build stage precompile + run: docker build -f Dockerfile.windows --target precompile -t edge-toolkit-windows . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca94237..34f0572 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,10 +90,6 @@ jobs: - name: Select all language envs run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - name: Install pipx (Windows only — aqua has no Windows build) - if: runner.os == 'Windows' - run: python -m pip install pipx - # Optional npm backend, installed before the main `mise install`. # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - name: Install aube (optional npm backend, allowed to fail) @@ -107,11 +103,6 @@ jobs: - name: Install mise tools run: | - # Each OS runs its own setup- task, before the main `mise install` - # so conda:openssl lands ahead of any parallel `cargo:*` openssl-sys - # build (which reads the project's OPENSSL_DIR and panics if include/ - # isn't populated yet). setup-windows also installs the gnu toolchain + - # flips rustup to the gnu host. case "${{ runner.os }}" in Linux) mise run setup-linux ;; macOS) mise run setup-macos ;; diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml index 1ea6422..77af307 100644 --- a/.mise/config.linux.toml +++ b/.mise/config.linux.toml @@ -39,7 +39,23 @@ LIBCLANG_PATH = "{{ vars.conda_clangxx }}/lib" BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.bindgen_args }}" [tasks.setup-linux] -# Linux preinstall: just the shared cross-platform base -- the C toolchain and -# gpg come from the distro (apt/system), not mise. -description = "Linux preinstall (the shared cross-platform base)" +# Linux preinstall: the shared cross-platform base (via _setup_all), then verify +# the system build prerequisites mise can't supply -- the C/C++ toolchain, gpg +# and archive tools the Dockerfile installs via apt. A workstation missing any is +# told to install them with its package manager (CI/Docker already have them). +description = "Linux preinstall: shared base + verify system build prerequisites" depends = ["_setup_all"] +run = """ +pkgs="bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils" +missing="" +for cmd in gcc g++ make git curl gpg unzip bzip2 xz; do + command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd" +done +if [ -n "$missing" ]; then + echo "setup-linux: missing required system tools:$missing" >&2 + echo "Install them with your package manager. On Debian/Ubuntu:" >&2 + echo " sudo apt-get install -y $pkgs" >&2 + exit 1 +fi +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml index 33f7280..df14feb 100644 --- a/.mise/config.macos.toml +++ b/.mise/config.macos.toml @@ -30,8 +30,17 @@ CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_ LIBCLANG_PATH = "{{ vars.mac_libclang }}" [tasks.setup-macos] -# macOS preinstall, run after Xcode's command-line tools (which mise can't -# supply): the shared base, then conda:lld -- the linker the darwin build uses. -description = "macOS preinstall: shared base + conda:lld (the linker the darwin build uses)" +# macOS preinstall: verify Xcode's command-line tools (cc/clang etc., which mise +# can't supply), then the shared base + conda:lld -- the linker the darwin build +# uses. A box missing the CLT is told to install them. +description = "macOS preinstall: verify Xcode CLT + shared base + conda:lld (the darwin linker)" depends = ["_setup_all"] -run = "mise install conda:lld" +run = """ +xcode-select -p >/dev/null 2>&1 || { + echo "setup-macos: Xcode command-line tools not found." >&2 + echo "Install them with: xcode-select --install" >&2 + exit 1 +} +mise install conda:lld +""" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.python.toml b/.mise/config.python.toml index d8fd217..cbaf307 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -3,7 +3,6 @@ # the `pipx:*` entries below resolve through it. [tools] -"pipx:cmake" = "latest" "pipx:componentize-py" = "latest" "pipx:datamodel-code-generator" = { version = "latest", extras = "ruff" } "pipx:openapi-python-client" = "0.29.0" diff --git a/.mise/config.toml b/.mise/config.toml index 23c3e34..5b23fbe 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -307,6 +307,7 @@ xargs taplo lint --schema "file://$PWD/config/taplo/no-path-deps.schema.json" <" xargs taplo lint --schema "file://$PWD/config/taplo/no-wildcard-or-git-deps.schema.json" <"$members" xargs taplo lint --schema "file://$PWD/config/taplo/require-workspace-deps.schema.json" <"$members" xargs taplo lint --schema "file://$PWD/config/taplo/require-lib-doctest-false.schema.json" <"$members" +xargs taplo lint --schema "file://$PWD/config/taplo/no-lib-name.schema.json" <"$members" # `require-lints-section` exempts generated/rust-rest/Cargo.toml because # progenitor's emitted `src/lib.rs` contains patterns our workspace lint @@ -326,7 +327,7 @@ taplo lint --schema "file://$PWD/config/taplo/mise-run-not-array.schema.json" .m shell = "bash -euo pipefail -c" [tasks.typos] -run = "typos" +run = "typos --config config/typos.toml" [tasks.setup-aube] # `aube` is an OPTIONAL faster npm backend, kept out of the mandatory `[tools]` @@ -359,7 +360,7 @@ shell = "bash -euo pipefail -c" # preinstall anywhere), node (mise uses it for the npm: tools) and conda:openssl # (for openssl-sys) -- all needed on every platform. hide = true -description = "Private shared cross-platform preinstall (depended on by setup-linux/macos/windows)" +description = "Private shared cross-platform preinstall" run = """ mise settings experimental=true mise settings set cargo.binstall true @@ -369,9 +370,6 @@ mise install conda:openssl """ shell = "bash -euo pipefail -c" -# The per-OS preinstall tasks (setup-linux / setup-macos / setup-windows) live in -# the config..toml files; each depends on the private _setup_all above. - [tasks.osv-scanner] # Scans the committed Cargo.lock only -- no build dependency, so the same task # serves both ad-hoc local use and the dependencies workflow. diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index dd0ff2f..f66235e 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -66,14 +66,20 @@ PYO3_PYTHON = "{{ vars.py313_win }}" # busybox `sh` (http:busybox) is installed by the cmd bootstrap that runs before # any bash task on Windows (Dockerfile.windows / the README), so this task's # `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. Installs -# the build utils (git/gpg/make), llvm-mingw and rust, then flips rustup's -# default host to gnullvm so both host and target link via llvm-mingw rather than -# the absent MSVC link.exe (no Visual Studio Build Tools needed). The prereqs -# mise CAN'T supply (a recent mise + the VC++ runtime) are in the README. -description = "Windows preinstall: Windows toolchain + flip rustup to the gnullvm host" +# the build utils (git/gpg/make), llvm-mingw and rust, pip-installs pipx (the +# backend for the pipx:* tools, since aqua has no Windows pipx build), then flips +# rustup's default host to gnullvm so both host and target link via llvm-mingw +# rather than the absent MSVC link.exe (no Visual Studio Build Tools needed). The +# prereqs mise CAN'T supply (a recent mise + the VC++ runtime) are in the README. +description = "Windows preinstall: Windows toolchain + pipx + flip rustup to the gnullvm host" depends = ["_setup_all"] run = """ mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust +# pipx backs the pipx:* tools but isn't a mise tool on Windows. Install it with +# whatever python is on PATH: the runner's system python on CI/workstations (its +# Scripts dir, where pipx lands, is already on PATH); Dockerfile.windows puts +# mise's python on PATH before this so the command resolves on Nano too. +python -m pip install pipx # Install + default to the gnullvm build of the EXACT rust version mise pinned; # mise sets RUSTUP_TOOLCHAIN to that bare version, so once the default host is # gnullvm it resolves to this -gnullvm toolchain. diff --git a/Cargo.lock b/Cargo.lock index e5f6a73..eab3e17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3998,8 +3998,8 @@ version = "0.1.0" dependencies = [ "asyncapi-rust", "base64 0.22.1", + "et-path", "fs-err", - "lets_find_up", "log", "rstest", "schemars 1.2.1", @@ -4160,6 +4160,7 @@ version = "0.1.0" dependencies = [ "clap", "edge-toolkit", + "et-path", "fs-err", "serde", "serde_json", @@ -4244,6 +4245,13 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "et-path" +version = "0.1.0" +dependencies = [ + "tempfile", +] + [[package]] name = "et-rest-client" version = "0.1.0" @@ -4611,6 +4619,7 @@ dependencies = [ name = "et-ws-wasi-comm1" version = "0.1.0" dependencies = [ + "et-path", "serde_json", "wit-bindgen 0.57.1", ] @@ -4619,6 +4628,7 @@ dependencies = [ name = "et-ws-wasi-data1" version = "0.1.0" dependencies = [ + "et-path", "serde_json", "wit-bindgen 0.57.1", ] @@ -6636,12 +6646,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "lets_find_up" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8052b3d5cfa8bae8af3b44aae11a43e9fa48ce0ae477c4a39733a8deff34059" - [[package]] name = "libc" version = "0.2.186" diff --git a/Cargo.toml b/Cargo.toml index 19311fe..9ab0c7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "libs/edge-toolkit", "libs/et-otlp", "libs/otlp-mock", + "libs/path", "libs/web", "libs/ws-runner-common", "services/ws-modules/audio1", @@ -65,6 +66,7 @@ deno_runtime = { version = "0.257.0", features = ["transpile", "hmr"] } edge-toolkit = { path = "libs/edge-toolkit", version = "0.1.0" } et-modules-service = { path = "services/modules", version = "0.1.0" } et-otlp = { path = "libs/et-otlp", version = "0.1.0" } +et-path = { path = "libs/path", version = "0.1.0" } et-rest-client = { path = "generated/rust-rest", version = "0.1.0", default-features = false } et-storage-service = { path = "services/storage", version = "0.1.0" } et-web = { path = "libs/web", version = "0.1.0" } @@ -81,7 +83,6 @@ hostname = "0.4" humantime-serde = "1" js-sys = "0.3" kdl = { version = "6", features = ["v1"] } -lets_find_up = "0.0.4" local-ip-address = "0.6" log = "0.4" onnx-extractor = "0.3" diff --git a/Dockerfile.windows b/Dockerfile.windows index 575f1c3..d45b3f7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -181,11 +181,19 @@ RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` ENV CARGO_HOME=C:\.cargo ` RUSTUP_HOME=C:\.rustup +# setup-windows pip-installs pipx (the pipx:* backend). Nano has no system python, +# so install mise's python and put it on PATH first; pipx then lands in that +# python's Scripts dir -- never invoked here (this image disables pipx:semgrep and +# builds no python guest, and pipx can't run on Nano anyway), so that's fine. +RUN set /p GITHUB_TOKEN= PathBuf { - match lets_find_up::find_up(".dprint.jsonc") { - Ok(Some(mut path)) => { - assert!(path.pop(), "Failed to drop the filename"); - path - } - Ok(None) => std::env::current_dir().unwrap(), - Err(err) => { - log::error!("{err}"); - std::env::current_dir().unwrap() - } - } + et_path::find_project_root(&std::env::current_dir().unwrap()) } /// Returns the default module search paths for ws-server. diff --git a/libs/path/Cargo.toml b/libs/path/Cargo.toml new file mode 100644 index 0000000..d6d4abc --- /dev/null +++ b/libs/path/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "et-path" +description = "Path utilities: locate the repository root + build relative/absolute paths for generated output" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/libs/path/src/lib.rs b/libs/path/src/lib.rs new file mode 100644 index 0000000..2ea31e0 --- /dev/null +++ b/libs/path/src/lib.rs @@ -0,0 +1,104 @@ +//! Path utilities shared across the workspace. +//! +//! Root-finding has two entry points: [`find_project_root`] (from an explicit +//! start, used by the runtime `edge_toolkit::config::get_project_root`) and +//! [`find_project_root_from_manifest`] (from `CARGO_MANIFEST_DIR`, used by build +//! scripts). The path builders [`absolute_from`] and [`relative_path_from`] +//! render POSIX-style paths for `mise.toml` / `docker-compose.yaml` generation. + +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; + +/// Marker file that identifies the repository root. `.dprint.jsonc` is present +/// at the root (and nowhere above it), so it is a reliable anchor. +const ROOT_MARKER: &str = ".dprint.jsonc"; + +/// Walk up from `start` to the first ancestor that contains `.dprint.jsonc`. +/// +/// Falls back to `start` itself when no ancestor has the marker, mirroring the +/// runtime helper's "use what we have" behaviour. +#[must_use] +pub fn find_project_root(start: &Path) -> PathBuf { + start + .ancestors() + .find(|dir| dir.join(ROOT_MARKER).is_file()) + .map_or_else(|| start.to_path_buf(), Path::to_path_buf) +} + +/// Locate the repository root from a build script. +/// +/// Reads `CARGO_MANIFEST_DIR` (which cargo sets for build scripts), so this is +/// the single sanctioned use of that variable (see the `no-cargo-manifest-dir` +/// ast-grep rule). Outside a build script the variable is unset and this +/// returns a meaningless path; use [`find_project_root`] with an explicit +/// start, or `edge_toolkit::config::get_project_root`, instead. +#[must_use] +pub fn find_project_root_from_manifest() -> PathBuf { + let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_default(); + find_project_root(Path::new(&manifest)) +} + +/// Resolve `path` against `base` when relative, then lexically normalize it. +#[must_use] +pub fn absolute_from(base: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + normalize_path(path) + } else { + normalize_path(&base.join(path)) + } +} + +/// Build a relative path from `from_dir` to `target`, always joined with `/`. +/// +/// The result is rendered as a POSIX string regardless of host OS, because +/// every caller writes it into generated `mise.toml` / `docker-compose.yaml` +/// output -- both of which expect forward-slash separators even on Windows. +#[must_use] +pub fn relative_path_from(from_dir: &Path, target: &Path) -> String { + let from_components = normal_components(&normalize_path(from_dir)); + let target_components = normal_components(&normalize_path(target)); + let common_len = from_components + .iter() + .zip(target_components.iter()) + .take_while(|(from, target)| from == target) + .count(); + + let mut parts: Vec = Vec::new(); + for _ in common_len..from_components.len() { + parts.push("..".to_string()); + } + for component in target_components.iter().skip(common_len) { + parts.push(component.to_string_lossy().into_owned()); + } + + if parts.is_empty() { + ".".to_string() + } else { + parts.join("/") + } +} + +fn normal_components(path: &Path) -> Vec { + path.components() + .filter_map(|component| match component { + Component::Normal(value) => Some(value.to_os_string()), + Component::Prefix(_) | Component::RootDir | Component::CurDir | Component::ParentDir => None, + }) + .collect() +} + +fn normalize_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(Path::new("/")), + Component::CurDir => {} + Component::ParentDir => { + let _popped = normalized.pop(); + } + Component::Normal(value) => normalized.push(value), + } + } + normalized +} diff --git a/libs/path/tests/find.rs b/libs/path/tests/find.rs new file mode 100644 index 0000000..02d3232 --- /dev/null +++ b/libs/path/tests/find.rs @@ -0,0 +1,32 @@ +#![cfg(test)] +#![expect( + clippy::unwrap_used, + reason = "test code: failed tempdir/fs setup should fail the test" +)] + +use std::fs; + +use et_path::find_project_root; +use tempfile::tempdir; + +#[test] +fn finds_marker_in_an_ancestor() { + let root = tempdir().unwrap(); + fs::write(root.path().join(".dprint.jsonc"), "{}").unwrap(); + let nested = root.path().join("a/b/c"); + fs::create_dir_all(&nested).unwrap(); + + // Canonicalize both sides so the macOS /var -> /private/var symlink on the + // tempdir doesn't fail an otherwise-correct match. + let found = fs::canonicalize(find_project_root(&nested)).unwrap(); + assert_eq!(found, fs::canonicalize(root.path()).unwrap()); +} + +#[test] +fn falls_back_to_start_when_marker_is_absent() { + let dir = tempdir().unwrap(); + let nested = dir.path().join("x"); + fs::create_dir_all(&nested).unwrap(); + + assert_eq!(find_project_root(&nested), nested); +} diff --git a/libs/ws-runner-common/Cargo.toml b/libs/ws-runner-common/Cargo.toml index c2b80e5..adb2e0b 100644 --- a/libs/ws-runner-common/Cargo.toml +++ b/libs/ws-runner-common/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [lib] doctest = false -name = "et_ws_runner_common" path = "src/lib.rs" [dependencies] diff --git a/services/ws-modules/wasi-comm1/Cargo.toml b/services/ws-modules/wasi-comm1/Cargo.toml index 8616d30..e195743 100644 --- a/services/ws-modules/wasi-comm1/Cargo.toml +++ b/services/ws-modules/wasi-comm1/Cargo.toml @@ -17,5 +17,9 @@ test = false serde_json.workspace = true wit-bindgen.workspace = true +# Build script (runs on the host) locates the repo root to emit ET_WIT_DIR. +[build-dependencies] +et-path.workspace = true + [lints] workspace = true diff --git a/services/ws-modules/wasi-comm1/build.rs b/services/ws-modules/wasi-comm1/build.rs new file mode 100644 index 0000000..17f2602 --- /dev/null +++ b/services/ws-modules/wasi-comm1/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_WIT_DIR` (absolute path to the shared WIT directory) so the +//! `wit_bindgen::generate!` invocation in `src/lib.rs` locates it via `env!`, +//! instead of a `..`-relative path that hardcodes this crate's depth below the +//! repository root. + +fn main() { + let wit_dir = et_path::find_project_root_from_manifest().join("generated/specs/wit"); + println!("cargo:rustc-env=ET_WIT_DIR={}", wit_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/services/ws-modules/wasi-comm1/src/lib.rs b/services/ws-modules/wasi-comm1/src/lib.rs index 112ad89..cf7337a 100644 --- a/services/ws-modules/wasi-comm1/src/lib.rs +++ b/services/ws-modules/wasi-comm1/src/lib.rs @@ -28,7 +28,8 @@ #![expect(unsafe_code)] wit_bindgen::generate!({ - path: "../../../generated/specs/wit", + // ET_WIT_DIR is the absolute path to generated/specs/wit, emitted by build.rs. + path: env!("ET_WIT_DIR"), world: "module", generate_all, }); diff --git a/services/ws-modules/wasi-data1/Cargo.toml b/services/ws-modules/wasi-data1/Cargo.toml index 1659c18..4a26e04 100644 --- a/services/ws-modules/wasi-data1/Cargo.toml +++ b/services/ws-modules/wasi-data1/Cargo.toml @@ -19,5 +19,9 @@ test = false serde_json.workspace = true wit-bindgen.workspace = true +# Build script (runs on the host) locates the repo root to emit ET_WIT_DIR. +[build-dependencies] +et-path.workspace = true + [lints] workspace = true diff --git a/services/ws-modules/wasi-data1/build.rs b/services/ws-modules/wasi-data1/build.rs new file mode 100644 index 0000000..17f2602 --- /dev/null +++ b/services/ws-modules/wasi-data1/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_WIT_DIR` (absolute path to the shared WIT directory) so the +//! `wit_bindgen::generate!` invocation in `src/lib.rs` locates it via `env!`, +//! instead of a `..`-relative path that hardcodes this crate's depth below the +//! repository root. + +fn main() { + let wit_dir = et_path::find_project_root_from_manifest().join("generated/specs/wit"); + println!("cargo:rustc-env=ET_WIT_DIR={}", wit_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/services/ws-modules/wasi-data1/src/lib.rs b/services/ws-modules/wasi-data1/src/lib.rs index 8e22e0b..5daf6d8 100644 --- a/services/ws-modules/wasi-data1/src/lib.rs +++ b/services/ws-modules/wasi-data1/src/lib.rs @@ -28,7 +28,8 @@ #![expect(unsafe_code)] wit_bindgen::generate!({ - path: "../../../generated/specs/wit", + // ET_WIT_DIR is the absolute path to generated/specs/wit, emitted by build.rs. + path: env!("ET_WIT_DIR"), world: "module", generate_all, }); diff --git a/services/ws-wasi-runner/Cargo.toml b/services/ws-wasi-runner/Cargo.toml index 73185cf..92e39d3 100644 --- a/services/ws-wasi-runner/Cargo.toml +++ b/services/ws-wasi-runner/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [lib] doctest = false -name = "et_ws_wasi_runner" path = "src/lib.rs" [[bin]] diff --git a/services/ws-wasi-runner/src/bindings.rs b/services/ws-wasi-runner/src/bindings.rs index 7a08e5d..f6cd0a0 100644 --- a/services/ws-wasi-runner/src/bindings.rs +++ b/services/ws-wasi-runner/src/bindings.rs @@ -23,6 +23,11 @@ //! bindgen-generated marker structs would be opaque and the `resource_table` //! couldn't carry real wgpu objects. wasmtime::component::bindgen!({ + // Sanctioned `..` exception: unlike `wit_bindgen::generate!` (the guest + // modules), wasmtime's `bindgen!` has no macro-string support, so `path:` + // can't be an `env!(...)` fed by build.rs -- it must be a literal resolved + // against this crate's dir. Kept relative-to-manifest as the one place a + // repo-relative `..` is unavoidable. path: "../../generated/specs/wit", world: "runner", imports: { default: async }, diff --git a/services/ws-web-runner/Cargo.toml b/services/ws-web-runner/Cargo.toml index 722b019..fd807f0 100644 --- a/services/ws-web-runner/Cargo.toml +++ b/services/ws-web-runner/Cargo.toml @@ -9,7 +9,6 @@ rust-version.workspace = true [lib] doctest = false -name = "et_ws_web_runner" path = "src/lib.rs" [[bin]] diff --git a/utilities/cli/Cargo.toml b/utilities/cli/Cargo.toml index a11e7d5..3fabf1f 100644 --- a/utilities/cli/Cargo.toml +++ b/utilities/cli/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] clap.workspace = true edge-toolkit.workspace = true +et-path.workspace = true fs-err.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/utilities/cli/src/cli.rs b/utilities/cli/src/cli.rs new file mode 100644 index 0000000..69fe6a5 --- /dev/null +++ b/utilities/cli/src/cli.rs @@ -0,0 +1,33 @@ +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; +use et_cli::OutputType; + +#[derive(Parser)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Generate deployment config from a cluster input YAML. + GenerateDeployment { + #[arg(long)] + input_file: PathBuf, + #[arg(long)] + output_dir: PathBuf, + #[arg(long, value_enum, default_value_t)] + output_type: OutputType, + }, + /// Regenerate verification outputs using verification input/output naming conventions. + RegenVerification { + #[arg(long, default_value = "verification")] + verification_root: PathBuf, + }, + /// Generate pkg/package.json from module metadata. + ModulePackageJson { + #[arg(long, default_value = ".")] + module_dir: PathBuf, + }, +} diff --git a/utilities/cli/src/deployment_types/docker_compose.rs b/utilities/cli/src/deployment_types/docker_compose.rs index dcb8865..ccd4664 100644 --- a/utilities/cli/src/deployment_types/docker_compose.rs +++ b/utilities/cli/src/deployment_types/docker_compose.rs @@ -1,16 +1,15 @@ use std::path::Path; use edge_toolkit::input::ClusterInput; +use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use crate::error::CliError; -use crate::{ - OutputType, absolute_from, cluster_module_names, module_registry, relative_path_from, resolve_module_paths, -}; +use crate::{OutputType, cluster_module_names, module_registry, resolve_module_paths}; pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let output_path = output_dir.join(OutputType::DockerCompose.output_file_name()); - let workspace_root = std::env::current_dir()?; + let workspace_root = edge_toolkit::config::get_project_root(); let output_abs = absolute_from(&workspace_root, output_dir); let workspace_rel = relative_path_from(&output_abs, &workspace_root); let openobserve_env_file_rel = relative_path_from(&output_abs, &workspace_root.join("config/o2.env")); diff --git a/utilities/cli/src/deployment_types/mise.rs b/utilities/cli/src/deployment_types/mise.rs index ad942fc..8c44e4d 100644 --- a/utilities/cli/src/deployment_types/mise.rs +++ b/utilities/cli/src/deployment_types/mise.rs @@ -1,15 +1,16 @@ use std::path::Path; use edge_toolkit::input::ClusterInput; +use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use toml::{Table, Value}; use crate::error::CliError; -use crate::{absolute_from, cluster_module_names, module_registry, relative_path_from, resolve_module_paths}; +use crate::{cluster_module_names, module_registry, resolve_module_paths}; pub fn generate_mise_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let output_path = output_dir.join("mise.toml"); - let workspace_root = std::env::current_dir()?; + let workspace_root = edge_toolkit::config::get_project_root(); let output_abs = absolute_from(&workspace_root, output_dir); let ws_server_dir = workspace_root.join("services/ws-server"); let workspace_rel = relative_path_from(&output_abs, &workspace_root); diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 60ee1a6..08a1e65 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -4,11 +4,11 @@ )] use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::ffi::OsString; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use clap::ValueEnum; use edge_toolkit::input::ClusterInput; +use et_path::relative_path_from; use fs_err as fs; use serde::Deserialize; @@ -593,70 +593,6 @@ where Ok(paths) } -#[must_use] -pub fn absolute_from(base: &Path, path: &Path) -> PathBuf { - if path.is_absolute() { - normalize_path(path) - } else { - normalize_path(&base.join(path)) - } -} - -/// Build a relative path from `from_dir` to `target`, always joined with `/`. -/// -/// The result is rendered as a POSIX string regardless of host OS, because -/// every caller writes it into generated `mise.toml` / `docker-compose.yaml` -/// output -- both of which expect forward-slash separators even on Windows. -#[must_use] -pub fn relative_path_from(from_dir: &Path, target: &Path) -> String { - let from_components = normal_components(&normalize_path(from_dir)); - let target_components = normal_components(&normalize_path(target)); - let common_len = from_components - .iter() - .zip(target_components.iter()) - .take_while(|(from, target)| from == target) - .count(); - - let mut parts: Vec = Vec::new(); - for _ in common_len..from_components.len() { - parts.push("..".to_string()); - } - for component in target_components.iter().skip(common_len) { - parts.push(component.to_string_lossy().into_owned()); - } - - if parts.is_empty() { - ".".to_string() - } else { - parts.join("/") - } -} - -fn normal_components(path: &Path) -> Vec { - path.components() - .filter_map(|component| match component { - Component::Normal(value) => Some(value.to_os_string()), - Component::Prefix(_) | Component::RootDir | Component::CurDir | Component::ParentDir => None, - }) - .collect() -} - -fn normalize_path(path: &Path) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(Path::new("/")), - Component::CurDir => {} - Component::ParentDir => { - let _popped = normalized.pop(); - } - Component::Normal(value) => normalized.push(value), - } - } - normalized -} - #[must_use] pub fn cluster_module_names(cluster: &ClusterInput) -> Vec { cluster diff --git a/utilities/cli/src/main.rs b/utilities/cli/src/main.rs index 726e7c5..a54194a 100644 --- a/utilities/cli/src/main.rs +++ b/utilities/cli/src/main.rs @@ -1,38 +1,11 @@ #![expect(clippy::print_stdout, reason = "CLI tool: println! is the intended UX")] -use std::path::PathBuf; +use clap::Parser as _; +use et_cli::{CliError, generate_deployment, generate_module_package_json, regenerate_verification}; -use clap::{Parser, Subcommand}; -use et_cli::{CliError, OutputType, generate_deployment, generate_module_package_json, regenerate_verification}; +mod cli; -#[derive(Parser)] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Generate deployment config from a cluster input YAML. - GenerateDeployment { - #[arg(long)] - input_file: PathBuf, - #[arg(long)] - output_dir: PathBuf, - #[arg(long, value_enum, default_value_t)] - output_type: OutputType, - }, - /// Regenerate verification outputs using verification input/output naming conventions. - RegenVerification { - #[arg(long, default_value = "verification")] - verification_root: PathBuf, - }, - /// Generate pkg/package.json from module metadata. - ModulePackageJson { - #[arg(long, default_value = ".")] - module_dir: PathBuf, - }, -} +use crate::cli::{Cli, Commands}; fn main() -> Result<(), CliError> { let cli = Cli::parse(); From 81562f4c8b1147e02df2550f468d72e1df5a30f8 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 08:28:26 +0800 Subject: [PATCH 49/60] Another Windows fix, and other things --- .github/workflows/check.yml | 6 +- .github/workflows/docker-linux.yml | 47 +++++++++++ .../{docker.yml => docker-windows.yml} | 37 ++------ .github/workflows/test.yml | 7 +- .mise/config.linux.toml | 4 +- .mise/config.macos.toml | 4 +- .mise/config.toml | 6 +- .mise/config.windows.toml | 12 +-- Dockerfile | 8 +- Dockerfile.windows => Dockerfile.nanoserver | 54 +++++++----- README.md | 84 ++++++++----------- config/dprint.jsonc | 2 +- config/semgrep/mise-config.yml | 2 +- libs/edge-toolkit/src/config.rs | 4 +- libs/path/Cargo.toml | 2 +- 15 files changed, 140 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/docker-linux.yml rename .github/workflows/{docker.yml => docker-windows.yml} (54%) rename Dockerfile.windows => Dockerfile.nanoserver (85%) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3a5b34d..f28ec90 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -49,11 +49,7 @@ jobs: - name: Install mise tools run: | - case "${{ runner.os }}" in - Linux) mise run setup-linux ;; - macOS) mise run setup-macos ;; - Windows) mise run setup-windows ;; - esac + mise run preinstall mise install env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/docker-linux.yml b/.github/workflows/docker-linux.yml new file mode 100644 index 0000000..6c56dae --- /dev/null +++ b/.github/workflows/docker-linux.yml @@ -0,0 +1,47 @@ +--- +name: docker-linux + +"on": + pull_request: + paths: + - .github/workflows/docker-linux.yml + - Dockerfile + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + # The image is large (every language toolchain + prefetched models. + # The runner's root disk (~14 GB free) can't hold it. + # Move Docker's storage to the ~70 GB /mnt scratch disk. + - name: Move Docker storage to /mnt + run: | + sudo systemctl stop docker docker.socket + sudo mkdir -p /mnt/docker + echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json + sudo systemctl start docker + docker info --format 'Docker Root Dir: {{.DockerRootDir}}' + + - name: Build stage test + env: + GITHUB_TOKEN: ${{ github.token }} + run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . + + - name: Run the test suite + run: docker run --rm edge-toolkit-test diff --git a/.github/workflows/docker.yml b/.github/workflows/docker-windows.yml similarity index 54% rename from .github/workflows/docker.yml rename to .github/workflows/docker-windows.yml index 163e1cc..268d0a4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker-windows.yml @@ -1,12 +1,11 @@ --- -name: docker +name: docker-windows "on": pull_request: paths: - - .github/workflows/docker.yml - - Dockerfile - - Dockerfile.windows + - .github/workflows/docker-windows.yml + - Dockerfile.nanoserver workflow_dispatch: permissions: @@ -21,33 +20,7 @@ defaults: shell: bash jobs: - linux: - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@v4 - - # The image is large (every language toolchain + prefetched models. - # The runner's root disk (~14 GB free) can't hold it. - # Move Docker's storage to the ~70 GB /mnt scratch disk. - - name: Move Docker storage to /mnt - run: | - sudo systemctl stop docker docker.socket - sudo mkdir -p /mnt/docker - echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json - sudo systemctl start docker - docker info --format 'Docker Root Dir: {{.DockerRootDir}}' - - - name: Build stage test - env: - GITHUB_TOKEN: ${{ github.token }} - run: DOCKER_BUILDKIT=1 docker build --target test --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit-test . - - - name: Run the test suite - run: docker run --rm edge-toolkit-test - - windows: + build: runs-on: windows-2022 timeout-minutes: 120 env: @@ -74,4 +47,4 @@ jobs: printf '%s' "${{ github.token }}" > gh_token - name: Build stage precompile - run: docker build -f Dockerfile.windows --target precompile -t edge-toolkit-windows . + run: docker build -f Dockerfile.nanoserver --target precompile -t edge-toolkit-windows . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34f0572..4aa6f32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,12 +103,7 @@ jobs: - name: Install mise tools run: | - case "${{ runner.os }}" in - Linux) mise run setup-linux ;; - macOS) mise run setup-macos ;; - Windows) mise run setup-windows ;; - esac - + mise run preinstall mise install env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml index 77af307..56c2bfb 100644 --- a/.mise/config.linux.toml +++ b/.mise/config.linux.toml @@ -38,7 +38,7 @@ CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars LIBCLANG_PATH = "{{ vars.conda_clangxx }}/lib" BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.bindgen_args }}" -[tasks.setup-linux] +[tasks.preinstall] # Linux preinstall: the shared cross-platform base (via _setup_all), then verify # the system build prerequisites mise can't supply -- the C/C++ toolchain, gpg # and archive tools the Dockerfile installs via apt. A workstation missing any is @@ -52,7 +52,7 @@ for cmd in gcc g++ make git curl gpg unzip bzip2 xz; do command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd" done if [ -n "$missing" ]; then - echo "setup-linux: missing required system tools:$missing" >&2 + echo "preinstall: missing required system tools:$missing" >&2 echo "Install them with your package manager. On Debian/Ubuntu:" >&2 echo " sudo apt-get install -y $pkgs" >&2 exit 1 diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml index df14feb..10e12bc 100644 --- a/.mise/config.macos.toml +++ b/.mise/config.macos.toml @@ -29,7 +29,7 @@ CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" LIBCLANG_PATH = "{{ vars.mac_libclang }}" -[tasks.setup-macos] +[tasks.preinstall] # macOS preinstall: verify Xcode's command-line tools (cc/clang etc., which mise # can't supply), then the shared base + conda:lld -- the linker the darwin build # uses. A box missing the CLT is told to install them. @@ -37,7 +37,7 @@ description = "macOS preinstall: verify Xcode CLT + shared base + conda:lld (the depends = ["_setup_all"] run = """ xcode-select -p >/dev/null 2>&1 || { - echo "setup-macos: Xcode command-line tools not found." >&2 + echo "preinstall: Xcode command-line tools not found." >&2 echo "Install them with: xcode-select --install" >&2 exit 1 } diff --git a/.mise/config.toml b/.mise/config.toml index 5b23fbe..8c6b8ce 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -23,7 +23,7 @@ min_version = "2026.6.5" # like `print-all-langs`, and the pre-install `setup-aube` step, from eagerly # pulling the whole toolchain — on CI and the CLI alike. task.run_auto_install # covers `mise run`; exec_auto_install covers `mise exec` (its OWN knob) — -# without it, `mise exec -- python …` in Dockerfile.windows installs *every* +# without it, `mise exec -- python …` in Dockerfile.nanoserver installs *every* # configured tool before running python. Turn both off so the policy holds. # (Current mise accepts the nested task.run_auto_install form; very old builds # like 2026.3 used the flat `task_run_auto_install`.) @@ -354,8 +354,8 @@ shell = "bash -euo pipefail -c" [tasks._setup_all] # Private (leading `_`, hide = true): shared cross-platform preinstall, never run -# directly. The per-OS setup tasks (setup-linux / setup-macos / setup-windows) -# depend on it, so each controls its own ordering. Enables experimental + +# directly. Each platform's `preinstall` task (one per config..toml) depends +# on it, so each controls its own ordering. Enables experimental + # cargo.binstall, then installs cargo-binstall (a prebuilt binary, safe to # preinstall anywhere), node (mise uses it for the npm: tools) and conda:openssl # (for openssl-sys) -- all needed on every platform. diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index f66235e..625c6cc 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -11,7 +11,7 @@ # Windows SDK can't come from mise, so the build uses the LLVM mingw toolchain # (the gnullvm target below); llvm-mingw bundles clang + lld + the mingw-w64 # runtime/headers/libs + the libclang.dll bindgen loads. Pinned so its install -# dir is deterministic for the PATH entry in Dockerfile.windows -- bump both. +# dir is deterministic for the PATH entry in Dockerfile.nanoserver -- bump both. "conda:m2-git" = "latest" "conda:m2-gnupg" = "latest" "conda:m2-make" = "latest" @@ -50,7 +50,7 @@ win_libclang = "C:\\Program Files\\LLVM\\bin" MISE_BASH_PATH = "{{ vars.winsh }}" # Build for the LLVM mingw target. gnullvm, not gnu: llvm-mingw ships # compiler-rt + libunwind, not the GCC runtime (libgcc/libgcc_eh) the gnu -# target's link line demands. setup-windows flips the rustup host to gnullvm too, +# target's link line demands. preinstall flips the rustup host to gnullvm too, # so build scripts/proc-macros (built for the host) link the same way. The linker # is clang and the raw-dylib import tool is llvm-dlltool, both from llvm-mingw on # PATH. These can live in [env] here (unlike config.toml) because this file only @@ -61,10 +61,10 @@ CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTFLAGS = "-Cdlltool=llvm-dlltool" LIBCLANG_PATH = "{{ vars.win_libclang }}" PYO3_PYTHON = "{{ vars.py313_win }}" -[tasks.setup-windows] -# Windows preinstall, shared by Dockerfile.windows and a real workstation. The +[tasks.preinstall] +# Windows preinstall, shared by Dockerfile.nanoserver and a real workstation. The # busybox `sh` (http:busybox) is installed by the cmd bootstrap that runs before -# any bash task on Windows (Dockerfile.windows / the README), so this task's +# any bash task on Windows (Dockerfile.nanoserver / the README), so this task's # `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. Installs # the build utils (git/gpg/make), llvm-mingw and rust, pip-installs pipx (the # backend for the pipx:* tools, since aqua has no Windows pipx build), then flips @@ -77,7 +77,7 @@ run = """ mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust # pipx backs the pipx:* tools but isn't a mise tool on Windows. Install it with # whatever python is on PATH: the runner's system python on CI/workstations (its -# Scripts dir, where pipx lands, is already on PATH); Dockerfile.windows puts +# Scripts dir, where pipx lands, is already on PATH); Dockerfile.nanoserver puts # mise's python on PATH before this so the command resolves on Nano too. python -m pip install pipx # Install + default to the gnullvm build of the EXACT rust version mise pinned; diff --git a/Dockerfile b/Dockerfile index 716d7ad..035d94d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ # finding worth documenting. # # A plain build produces the SERVER image (final stage): a release et-ws-server, -# served automatically. A GitHub token avoids mise's 60-req/hr anonymous limit +# served automatically. A GitHub token avoids mise's anonymous GitHub rate limit # during install-all: # DOCKER_BUILDKIT=1 docker build --secret id=gh_token,env=GITHUB_TOKEN -t edge-toolkit . # docker run --rm -p 8080:8080 edge-toolkit # serves; open http://localhost:8080 @@ -72,7 +72,7 @@ ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" WORKDIR /workspace # The default tools need the always-loaded config plus config.linux.toml (where -# setup-linux + the Linux env live) and .miserc.toml (auto_env, which auto-loads +# preinstall + the Linux env live) and .miserc.toml (auto_env, which auto-loads # config.linux.toml); the other guest-language configs come in the build stage # below. These are repo configs, so they're copied + trusted before mise runs. COPY .miserc.toml .miserc.toml @@ -81,7 +81,7 @@ COPY .mise/config.linux.toml .mise/config.linux.toml RUN mise trust -# Preinstall via the shared setup-linux task (the same a Linux workstation runs): +# Preinstall via the shared preinstall task (the same a Linux workstation runs): # its setup-all base enables experimental + cargo.binstall and installs # cargo-binstall, node and conda:openssl; then `mise install` adds the rest of # the always-loaded tools. A GitHub token (if provided) lifts the anonymous rate @@ -90,7 +90,7 @@ RUN mise trust # rustup-inits at once, racing on the shared rustup binary's self-update (exit 1). RUN --mount=type=secret,id=gh_token,required=false \ GITHUB_TOKEN="$(cat /run/secrets/gh_token 2>/dev/null || true)" \ - sh -c 'mise run setup-linux && MISE_JOBS=1 mise install' + sh -c 'mise run preinstall && MISE_JOBS=1 mise install' # --- build: add the guest-language toolchains (config..toml). --- # install-all == MISE_ENV="$ALL_LANGS" mise install; the always-loaded tools are diff --git a/Dockerfile.windows b/Dockerfile.nanoserver similarity index 85% rename from Dockerfile.windows rename to Dockerfile.nanoserver index d45b3f7..362fb31 100644 --- a/Dockerfile.windows +++ b/Dockerfile.nanoserver @@ -50,9 +50,6 @@ COPY --from=vcruntime C:\Windows\System32\vcruntime140.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\vcruntime140_1.dll C:\Windows\System32\ COPY --from=vcruntime C:\Windows\System32\msvcp140.dll C:\Windows\System32\ -# The bash-shell mise tasks run on busybox (the http:busybox tool), installed -# below in cmd before any of them -- a prebuilt download, no build step. - # cmd is the only shell Nano Server has; ContainerAdministrator so we can write # under C:\ (the backtick line continuations below rely on the escape directive # documented at the top of this file). @@ -86,6 +83,15 @@ RUN if not exist C:\Temp mkdir C:\Temp ENV TEMP=C:\Temp ` TMP=C:\Temp +# wasm-pack's binary cache (binary-install's Cache::new) resolves its dir via +# dirs_next, which on Windows calls the SHGetKnownFolderPath shell API that Nano +# lacks (the same gap that stops pipx) -- so it fails with "couldn't find your +# home directory, is $HOME not set?" (build-ws-wasm-agent's `-Z build-std` hits +# it first). WASM_PACK_CACHE makes wasm-pack use Cache::at() instead, which +# stores the path verbatim and never calls dirs (the dir is created on first +# use). It's a transient build cache, so it lives under the temp dir above. +ENV WASM_PACK_CACHE=C:\Temp\wasm-pack + # Serialize mise's installs (MISE_JOBS=1). On Nano Server the parallel cargo: # source builds -- no gnu binstall prebuilts exist, so they fall back to `cargo # install` -- race on rustup's shared .rustup\downloads dir: one build's @@ -117,7 +123,7 @@ RUN tar -xf C:\mise.zip -C C:\ && del C:\mise.zip # tool builds (windows-sys etc.) invoke `x86_64-w64-mingw32-dlltool` + the gnu # linker by name, and mise's cargo backend doesn't activate llvm-mingw onto the # build subprocess's PATH. The dir is the pinned-version install (config.toml); -# it doesn't exist until setup-windows installs llvm-mingw, which is fine -- the +# it doesn't exist until preinstall installs llvm-mingw, which is fine -- the # cargo: builds that need it run afterwards. ENV LLVMBIN=C:\Users\ContainerAdministrator\AppData\Local\mise\installs\github-mstorsjo-llvm-mingw\20260602\bin ENV PATH="C:\mise\bin;C:\Windows\System32;C:\Windows;${LLVMBIN};${PATH}" @@ -130,12 +136,16 @@ COPY .mise/config.toml .mise/config.toml COPY .mise/config.windows.toml .mise/config.windows.toml COPY .miserc.toml .miserc.toml -# The windows job stages the GitHub token as gh_token in the build context -# (lifting mise's 60-req/hr anonymous limit); we read it per-RUN rather than via -# a build-arg, which the classic Windows builder can't substitute into RUN. It -# carries forward into the later stages that also hit GitHub and expires with the -# throwaway job image. -COPY gh_token C:\gh_token +# gh_token (a GitHub token) is OPTIONAL. With it, mise's GitHub fetches use the +# authenticated rate limit; without it they fall back to the lower anonymous one, +# which can rate-limit `install`. The classic Windows builder has no +# BuildKit secrets, so the token is read per-RUN from a file in the build context +# (it can't be a build-arg substituted into RUN). The glob makes it optional: +# paired with an always-present file (.miserc.toml) so COPY still succeeds when +# gh_token is absent; both land in C:\token (a dir off the WORKDIR ancestor chain, +# so the paired file is inert). WARNING: when a token IS supplied it bakes into +# this image layer -- do NOT publish an image built with one (see the README). +COPY .miserc.toml gh_token* C:/token/ # Docker-only mise settings, written inline in cmd (no bash needed): `mise # settings` writes the global config (C:\.config\mise on Windows, not @@ -144,7 +154,7 @@ COPY gh_token C:\gh_token # exist; aqua signature checks (attestations/cosign/slsa) are off (their # sigstore/TUF step can't find a cache dir here); pipx.uvx=false makes the pipx: # backend use pip not uv (uv's PE-resource trampoline isn't implemented on Nano). -RUN set /p GITHUB_TOKEN=.toml). --- @@ -219,14 +229,14 @@ FROM build-minimal AS build COPY .mise/ .mise/ RUN mise trust ENV MISE_ENV=dart,java,rust,zig -RUN set /p GITHUB_TOKEN= PathBuf { diff --git a/libs/path/Cargo.toml b/libs/path/Cargo.toml index d6d4abc..e44633e 100644 --- a/libs/path/Cargo.toml +++ b/libs/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "et-path" -description = "Path utilities: locate the repository root + build relative/absolute paths for generated output" +description = "Path utilities" version = "0.1.0" edition.workspace = true license.workspace = true From ccb2d8fb34283d1ea7c7fd383ece4c1e734cc4e3 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 09:36:55 +0800 Subject: [PATCH 50/60] ci(mise): prebuilt tool binaries, taplo-safe vars, windows toolchain fix - Switch ast-grep/cargo-deny/taplo/wasm-pack/watchexec from `cargo:` source builds to prebuilt aqua binaries. cargo-expand has no prebuilt (crates.io only), so it stays source-built and is disabled in both Docker images via MISE_DISABLE_TOOLS. - Prefix the Linux config vars (a_/b_/c_ tiers) so taplo's reorder_keys keeps every definition before its use; alphabetising had moved a value after the var that reads it and broke `mise install` tera rendering. - Provision the Windows gnullvm stable+nightly toolchains with their components and wasm targets in preinstall (rustup had installed them against the pre-flip host). Fixes the wasm-agent `-Z build-std` "rust-src missing" failure and stops wasm-pack downloading rust-std mid-build. - Build the wasm-agent with `wasm-pack --mode no-install` so it uses the mise-pinned wasm-bindgen rather than fetching its own. - Move clippy config to config/clippy.toml, add config/lychee.toml + link-check, refresh the .miserc.toml auto_env note, and reclaim Docker CI disk via easimon/maximize-build-space. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/docker-linux.yml | 30 +++++++++++------ .mise/config.linux.toml | 51 +++++++++++++++------------- .mise/config.macos.toml | 29 +++++++--------- .mise/config.toml | 54 ++++++++++++++++++++---------- .mise/config.windows.toml | 37 ++++++++++---------- .miserc.toml | 8 ++--- Dockerfile | 4 +++ Dockerfile.nanoserver | 14 ++++---- .clippy.toml => config/clippy.toml | 0 config/lychee.toml | 26 ++++++++++++++ 10 files changed, 159 insertions(+), 94 deletions(-) rename .clippy.toml => config/clippy.toml (100%) create mode 100644 config/lychee.toml diff --git a/.github/workflows/docker-linux.yml b/.github/workflows/docker-linux.yml index 6c56dae..7de7f53 100644 --- a/.github/workflows/docker-linux.yml +++ b/.github/workflows/docker-linux.yml @@ -27,16 +27,26 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # The image is large (every language toolchain + prefetched models. - # The runner's root disk (~14 GB free) can't hold it. - # Move Docker's storage to the ~70 GB /mnt scratch disk. - - name: Move Docker storage to /mnt - run: | - sudo systemctl stop docker docker.socket - sudo mkdir -p /mnt/docker - echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json - sudo systemctl start docker - docker info --format 'Docker Root Dir: {{.DockerRootDir}}' + # The image is huge (every language toolchain + prefetched models + a full + # debug build, incl. aws-lc-sys's large C objects), and its peak `target/` + # overruns a single runner disk. Reclaim the unused preinstalled SDKs and + # concatenate the freed root space with /mnt into one LVM volume mounted at + # Docker's data dir, then restart Docker so the build uses the combined space. + - name: Maximize build space (combine root + /mnt for Docker) + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 4096 + swap-size-mb: 1024 + remove-dotnet: "true" + remove-android: "true" + remove-haskell: "true" + remove-codeql: "true" + remove-docker-images: "true" + build-mount-path: /var/lib/docker + build-mount-path-ownership: "root:root" + + - name: Restart Docker on the maximized volume + run: sudo systemctl restart docker - name: Build stage test env: diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml index 56c2bfb..2f13b10 100644 --- a/.mise/config.linux.toml +++ b/.mise/config.linux.toml @@ -1,50 +1,53 @@ -# Linux-only mise config. Loaded automatically on Linux by the platform auto_env -# feature (.miserc.toml): the `linux` platform env maps to this file. Shared vars -# (rpath_flag, pylib_flag, conda_openssl, py313_unix, …) come from config.toml, -# which loads first; PYO3_PYTHON keeps its config.toml default (py313_unix). +# Linux-only mise config. Shared vars (rpath_flag, pylib_flag, conda_openssl, +# py313_unix, …) come from config.toml, which loads first; PYO3_PYTHON keeps its +# config.toml default (py313_unix). [tools] -# Ships the C `libclang.so` (+ clang resource headers) that bindgen needs to -# build the deno web runner's libsqlite3-sys. A mise-only Linux box usually only -# has libclang-cpp (the C++ API); macOS uses Xcode's libclang, Windows -# llvm-mingw's. +# Ships the C `libclang.so` (+ clang resource headers) bindgen needs to build the +# deno web runner's libsqlite3-sys; a mise-only box usually only has libclang-cpp +# (the C++ API), not the C one. "conda:clangxx" = "latest" [vars] -conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" -clang_resource = "{{ vars.conda_clangxx }}/lib/clang/22" -# conda's target-triple dir uses x86_64/aarch64; mise's arch() is x64/arm64. -conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" -clang_sysroot = "{{ vars.conda_clangxx }}/{{ vars.conda_arch }}-conda-linux-gnu/sysroot" +# An `a_`/`b_`/`c_` prefix encodes each var's dependency tier so the alphabetical +# order taplo's reorder_keys imposes keeps every definition before its use: `a_` +# vars stand alone, `b_` vars read `a_` vars, `c_` reads `b_`. Without it sorting +# moves a definition after the var that reads it and `mise install` can't render +# the value. `a_conda_arch` picks conda's target-triple dir (x86_64/aarch64, +# vs mise's x64/arm64). +a_conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" +a_conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" +b_clang_resource = "{{ vars.a_conda_clangxx }}/lib/clang/22" +b_clang_sysroot = "{{ vars.a_conda_clangxx }}/{{ vars.a_conda_arch }}-conda-linux-gnu/sysroot" # bindgen loads libclang directly rather than driving the clang binary, so it # can't auto-find its resource dir (builtins like stddef.h) or conda's bundled # sysroot. Pass `-resource-dir` + `--sysroot` so header resolution is identical # on every machine (a bare `-isystem` leaves builtin type macros unset and fails # in CI). Using conda's sysroot rather than the host's /usr/include keeps bindgen # self-contained. -bindgen_args = "-resource-dir {{ vars.clang_resource }} --sysroot={{ vars.clang_sysroot }}" +c_bindgen_args = "-resource-dir {{ vars.b_clang_resource }} --sysroot={{ vars.b_clang_sysroot }}" [env] +# clang-sys finds conda's libclang.so via LIBCLANG_PATH; bindgen gets conda's +# resource headers + sysroot via BINDGEN_EXTRA_CLANG_ARGS (see c_bindgen_args). +BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.c_bindgen_args }}" # rpath/pylib flags (shared, from config.toml): OPENSSL_DIR points openssl-sys at # conda's OpenSSL, so anything linking it records conda's libssl soname; without # an rpath the loader only finds it when conda's soname matches the system one # (it broke when conda bumped to libssl.so.4). pylib_flag does the same for the -# CPython lib dir. No `-fuse-ld=lld`: the system linker accepts `-Wl,-rpath` -# directly, unlike Apple clang's lld dance. +# CPython lib dir. No `-fuse-ld=lld` needed: the system linker accepts +# `-Wl,-rpath` directly. CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "{{ vars.rpath_flag }} {{ vars.pylib_flag }}" -# Point clang-sys at conda's libclang.so and feed bindgen conda's resource -# headers + sysroot (see bindgen_args). -LIBCLANG_PATH = "{{ vars.conda_clangxx }}/lib" -BINDGEN_EXTRA_CLANG_ARGS = "{{ vars.bindgen_args }}" +LIBCLANG_PATH = "{{ vars.a_conda_clangxx }}/lib" [tasks.preinstall] -# Linux preinstall: the shared cross-platform base (via _setup_all), then verify -# the system build prerequisites mise can't supply -- the C/C++ toolchain, gpg -# and archive tools the Dockerfile installs via apt. A workstation missing any is +# Preinstall: the shared cross-platform base (via _setup_all), then verify the +# system build prerequisites mise can't supply -- the C/C++ toolchain, gpg and +# archive tools the Dockerfile installs via apt. A workstation missing any is # told to install them with its package manager (CI/Docker already have them). -description = "Linux preinstall: shared base + verify system build prerequisites" depends = ["_setup_all"] +description = "Preinstall: shared base + verify system build prerequisites" run = """ pkgs="bzip2 ca-certificates curl g++ gcc git gnupg libc6-dev libicu74 make unzip xz-utils" missing="" diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml index 10e12bc..dd1b3b4 100644 --- a/.mise/config.macos.toml +++ b/.mise/config.macos.toml @@ -1,12 +1,10 @@ -# macOS-only mise config. Loaded automatically on macOS by the platform auto_env -# feature (.miserc.toml): the `macos` platform env maps to this file. Shared vars -# (rpath_flag, pylib_flag, py313_unix, …) come from config.toml, which loads -# first; PYO3_PYTHON keeps its config.toml default (py313_unix) here. +# macOS-only mise config. Shared vars (rpath_flag, pylib_flag, py313_unix, …) +# come from config.toml, which loads first; PYO3_PYTHON keeps its config.toml +# default (py313_unix). [tools] -# The LLD linker (ships ld64.lld) the darwin RUSTFLAGS below use; Apple's -# /usr/bin/clang can't resolve `-fuse-ld=lld` without it. macOS-only -- Linux -# uses its system linker, Windows the llvm-mingw one. +# The LLD linker (ships ld64.lld) the RUSTFLAGS below use; Apple's /usr/bin/clang +# can't resolve `-fuse-ld=lld` without it. "conda:lld" = "latest" [vars] @@ -20,21 +18,20 @@ lld_flag = "-C link-arg=-fuse-ld={{ vars.conda_lld }}/bin/ld64.lld" mac_libclang = "{{ exec(command='dirname $(dirname $(xcrun --find clang))') }}/lib" [env] -# Apple `/usr/bin/clang` rejects `-fuse-ld=lld` ("invalid linker name") unless it -# resolves ld64.lld; lld_flag gives the absolute path. rpath/pylib flags (shared -# with Linux, from config.toml) record conda's OpenSSL soname + the CPython lib -# dir so the runtime loader finds them. Applies to every rustc call, including -# the cargo subprocess mise spawns for a `cargo:` source build. +# lld_flag points Apple's clang at ld64.lld (see [vars]); the rpath/pylib flags +# (from config.toml) record conda's OpenSSL soname + the CPython lib dir so the +# runtime loader finds them. Applies to every rustc call, including the `cargo:` +# source-build subprocess. CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "{{ vars.lld_flag }} {{ vars.rpath_flag }} {{ vars.pylib_flag }}" LIBCLANG_PATH = "{{ vars.mac_libclang }}" [tasks.preinstall] -# macOS preinstall: verify Xcode's command-line tools (cc/clang etc., which mise -# can't supply), then the shared base + conda:lld -- the linker the darwin build -# uses. A box missing the CLT is told to install them. -description = "macOS preinstall: verify Xcode CLT + shared base + conda:lld (the darwin linker)" +# Preinstall: verify Xcode's command-line tools (cc/clang etc., which mise can't +# supply), then the shared base + conda:lld (the darwin linker). A box missing +# the CLT is told to install them. depends = ["_setup_all"] +description = "Preinstall: verify Xcode CLT, shared base, conda:lld (darwin linker)" run = """ xcode-select -p >/dev/null 2>&1 || { echo "preinstall: Xcode command-line tools not found." >&2 diff --git a/.mise/config.toml b/.mise/config.toml index 8c6b8ce..90dece2 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -27,21 +27,22 @@ min_version = "2026.6.5" # configured tool before running python. Turn both off so the policy holds. # (Current mise accepts the nested task.run_auto_install form; very old builds # like 2026.3 used the flat `task_run_auto_install`.) -task.run_auto_install = false exec_auto_install = false +task.run_auto_install = false [tools] action-validator = "latest" +"aqua:EmbarkStudios/cargo-deny" = "latest" +ast-grep = "latest" cargo-binstall = "latest" -"cargo:ast-grep" = "latest" -"cargo:cargo-deny" = "latest" +# dtolnay publishes no prebuilt binary for cargo-expand (crates.io only), so it's +# the one tool here still built from source. +"aqua:rustwasm/wasm-pack" = "latest" "cargo:cargo-expand" = "latest" -# Not on Windows: taplo-cli pulls `ring`, whose build needs a C/asm toolchain the -# gnu Nano image doesn't provide, and TOML linting (taplo-check) is skipped on -# Windows anyway (it runs in the Linux check job). -"cargo:taplo-cli" = { version = "latest", os = ["linux", "macos"] } -"cargo:wasm-pack" = "latest" -"cargo:watchexec-cli" = "latest" +# aqua ships a prebuilt taplo binary (no `ring` C build). Off Windows regardless: +# taplo-check (TOML linting) runs only in the Linux check job. +taplo = { version = "latest", os = ["linux", "macos"] } +watchexec = "latest" # vfox-chromedriver's PostInstall hook fails on Windows ("syntax of the # command is incorrect"). Chromedriver is only used by the # `test-ws-wasm-agent-chrome` / `ws-e2e-chrome` tasks, neither of which run @@ -61,6 +62,10 @@ editorconfig-checker = "latest" "github:grok-rs/waitup" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.114" hadolint = "latest" +# Link checker for the `link-check` task. Installs on every OS; the Nano Docker +# image opts out via MISE_DISABLE_TOOLS (link-check isn't run in the Windows +# build/test flow), which keeps that image lean. +lychee = "latest" mprocs = "latest" node = "22" "npm:onnxruntime-web" = "latest" @@ -74,6 +79,7 @@ pipx = { version = "latest", os = ["linux", "macos"] } "pipx:semgrep" = "latest" # The default aqua backend has no darwin/amd64 prebuilt; fall back to # `npm:pnpm` on Intel Mac (node is already a mise tool above). +gh = "latest" "npm:pnpm" = { version = "latest", os = ["macos/x64"] } pnpm = { version = "latest", os = ["linux", "macos/arm64", "windows"] } protoc = "latest" @@ -87,7 +93,6 @@ typos = "latest" uv = "0.11.8" wasm-tools = "latest" yq = "latest" -gh = "latest" [vars] # conda:openssl install dir, reused by OPENSSL_DIR + rpath_flag. The per-OS @@ -110,19 +115,19 @@ pylib_flag = "-C link-arg=-Wl,-rpath,{{ env.HOME }}/.local/share/mise/installs/p wasm_rustflags = "-C target-cpu=mvp -C target-feature=+mutable-globals,+sign-ext,+nontrapping-fptoint" # Source/dest paths for the fetch-*-rclone model downloads — kept as vars so # each `rclone copyto` command stays on a single line without continuations. -face1_src = ":http:amd/retinaface/resolve/main/weights/RetinaFace_int.onnx" face1_dst = "data/model-modules/model-face1/pkg/video_cv.onnx" +face1_src = ":http:amd/retinaface/resolve/main/weights/RetinaFace_int.onnx" +har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" har_repo = ":http:acd17sk/MET-Metabolic-Equivalent-of-Task-AI-Android-APP" har_src = "{{ vars.har_repo }}/raw/refs/heads/main/Models/onnx/hybrid_met.onnx" -har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" -mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" +mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" # `retry` prefixes each fetch with recur for exponential-backoff-with-jitter # retries (8 tries, 2s base doubling, capped 2m, 0-5s jitter); the *_http vars # carry the per-host base URL + flags. Kept as vars so each task line fits. -retry = "recur --attempts 8 --delay 2s --backoff 2s --max-delay 2m --jitter 0,5s --" -hf_http = "--http-url https://huggingface.co --progress --ignore-existing" gh_http = "--http-url https://github.com --progress --ignore-existing" +hf_http = "--http-url https://huggingface.co --progress --ignore-existing" +retry = "recur --attempts 8 --delay 2s --backoff 2s --max-delay 2m --jitter 0,5s --" [env] # Comma-separated list of every language env (one per .mise/config..toml). @@ -134,6 +139,9 @@ ALL_LANGS = "dart,dotnet,java,python,rust,zig" # no longer auto-discovers; TAPLO_CONFIG points every `taplo lint`/`format` at # it. config_root keeps the path valid regardless of the invocation's cwd. TAPLO_CONFIG = "{{ config_root }}/config/taplo.toml" +# Likewise clippy.toml lives under config/; clippy reads its config from the dir +# named by CLIPPY_CONF_DIR rather than auto-discovering a root .clippy.toml. +CLIPPY_CONF_DIR = "{{ config_root }}/config" # Use the conda:openssl install for Rust's OPENSSL_DIR so openssl-sys crate builds. OPENSSL_DIR = "{{ vars.conda_openssl }}" # Per-OS RUSTFLAGS (CARGO_TARGET__*), LIBCLANG_PATH, BINDGEN args and the @@ -329,6 +337,14 @@ shell = "bash -euo pipefail -c" [tasks.typos] run = "typos --config config/typos.toml" +# Not in the `check` aggregate: lychee makes live network requests, so it would +# make CI flaky. Run on demand (or from a scheduled workflow). Checks every URL +# in the repo's Markdown + Rust sources; lychee extracts URLs from .rs comments +# and string literals too. config/lychee.toml excludes generated/ + target/. +[tasks.link-check] +description = "Check that URLs in .md and .rs files are reachable (network; not in check)" +run = "lychee --config config/lychee.toml '**/*.md' '**/*.rs'" + [tasks.setup-aube] # `aube` is an OPTIONAL faster npm backend, kept out of the mandatory `[tools]` # list because its install is flaky on some platforms and edge-toolkit falls @@ -359,8 +375,8 @@ shell = "bash -euo pipefail -c" # cargo.binstall, then installs cargo-binstall (a prebuilt binary, safe to # preinstall anywhere), node (mise uses it for the npm: tools) and conda:openssl # (for openssl-sys) -- all needed on every platform. -hide = true description = "Private shared cross-platform preinstall" +hide = true run = """ mise settings experimental=true mise settings set cargo.binstall true @@ -568,7 +584,11 @@ shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent] description = "Build the WebSocket WASM client" dir = "services/ws-wasm-agent" -run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" +# --mode no-install: use the mise-pinned wasm-bindgen (and skip wasm-opt) instead +# of letting wasm-pack download its own. The wasm32 target + rust-src come from +# the `rust` tool's targets/components (Windows re-adds them to its gnullvm +# toolchains in preinstall), so nothing is fetched at build time. +run = "wasm-pack build . --target web --mode no-install -- -Z build-std=std,panic_abort" [tasks.build-ws-wasm-agent.env] # Scope these flags to the wasm target rather than setting plain `RUSTFLAGS`: diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index 625c6cc..f636bb5 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -1,8 +1,6 @@ -# Windows-only mise config. Loaded automatically on Windows by the platform -# auto_env feature (enabled in .miserc.toml): the `windows` platform env maps to -# this file. Everything here is therefore Windows-scoped, so it needs no per-entry -# `os = ["windows"]` guards or `{% if os() == 'windows' %}` conditionals -- values -# defined here override the cross-platform defaults in config.toml on Windows. +# Windows-only mise config. Everything here is Windows-scoped, so it needs no +# per-entry `os = ["windows"]` guards or `{% if os() == 'windows' %}` +# conditionals; values here override the cross-platform defaults in config.toml. # Shared vars (rpath_flag, py313_*, …) come from config.toml, which loads first. [tools] @@ -28,16 +26,16 @@ # loudly; `.exe` because mise writes the bin name verbatim and Windows launches # by extension. [tools."http:busybox"] -version = "1.37.0" bin = "ash.exe" -url = "https://frippery.org/files/busybox/busybox64u.exe" checksum = "sha256:6e263d154d8548d1eb936f65d1d8312c80df31c45974e48d6335e4dcc0f4f34c" +url = "https://frippery.org/files/busybox/busybox64u.exe" +version = "1.37.0" [vars] # busybox `ash` install path (for MISE_BASH_PATH), keyed by the pinned # http:busybox version above -- keep in sync. winsh = '{{ get_env(name="LOCALAPPDATA", default="") }}\mise\installs\http-busybox\1.37.0\ash.exe' -# mise-managed CPython 3.13; the mise data dir is LOCALAPPDATA\mise on Windows. +# mise-managed CPython 3.13; the mise data dir is LOCALAPPDATA\mise. py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\python\\3.13\\python.exe" # clang-sys finds libclang on PATH (Docker's gnu build has llvm-mingw there); a # workstation with system LLVM also works. @@ -62,17 +60,17 @@ LIBCLANG_PATH = "{{ vars.win_libclang }}" PYO3_PYTHON = "{{ vars.py313_win }}" [tasks.preinstall] -# Windows preinstall, shared by Dockerfile.nanoserver and a real workstation. The -# busybox `sh` (http:busybox) is installed by the cmd bootstrap that runs before -# any bash task on Windows (Dockerfile.nanoserver / the README), so this task's -# `bash -euo pipefail` shell (= busybox ash via MISE_BASH_PATH) works. Installs +# Preinstall, shared by Dockerfile.nanoserver and a real workstation. The busybox +# `sh` (http:busybox) is installed by the cmd bootstrap that runs before any bash +# task (Dockerfile.nanoserver / the README), so this task's `bash -euo pipefail` +# shell (= busybox ash via MISE_BASH_PATH) works. Installs # the build utils (git/gpg/make), llvm-mingw and rust, pip-installs pipx (the # backend for the pipx:* tools, since aqua has no Windows pipx build), then flips # rustup's default host to gnullvm so both host and target link via llvm-mingw # rather than the absent MSVC link.exe (no Visual Studio Build Tools needed). The # prereqs mise CAN'T supply (a recent mise + the VC++ runtime) are in the README. -description = "Windows preinstall: Windows toolchain + pipx + flip rustup to the gnullvm host" depends = ["_setup_all"] +description = "Preinstall: toolchain + pipx + flip rustup to the gnullvm host" run = """ mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust # pipx backs the pipx:* tools but isn't a mise tool on Windows. Install it with @@ -80,12 +78,17 @@ mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-mak # Scripts dir, where pipx lands, is already on PATH); Dockerfile.nanoserver puts # mise's python on PATH before this so the command resolves on Nano too. python -m pip install pipx -# Install + default to the gnullvm build of the EXACT rust version mise pinned; -# mise sets RUSTUP_TOOLCHAIN to that bare version, so once the default host is -# gnullvm it resolves to this -gnullvm toolchain. +# rustup installed components/targets against the pre-flip default host, so after +# flipping to gnullvm reinstall the EXACT stable + nightly versions mise pinned as +# gnullvm toolchains, each carrying the components + wasm targets config.toml's +# `rust` tool lists. Pre-adding the wasm target means wasm-pack finds it present +# and never downloads rust-std mid-build. mise sets RUSTUP_TOOLCHAIN to the bare +# stable version, which now resolves to -gnullvm. ver="$(mise exec -- rustc --version | awk '{print $2}')" mise exec -- rustup set default-host x86_64-pc-windows-gnullvm -mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnullvm" +wasm="-t wasm32-unknown-unknown -t wasm32-wasip2" +mise exec -- rustup toolchain install "${ver}-x86_64-pc-windows-gnullvm" -c clippy -c rust-analyzer $wasm +mise exec -- rustup toolchain install nightly-x86_64-pc-windows-gnullvm -c rust-src -c rustfmt $wasm mise exec -- rustup default "${ver}-x86_64-pc-windows-gnullvm" """ shell = "bash -euo pipefail -c" diff --git a/.miserc.toml b/.miserc.toml index 0f0a2d6..36be63a 100644 --- a/.miserc.toml +++ b/.miserc.toml @@ -1,8 +1,8 @@ # Early-init mise settings (read before the regular config, so they can affect # which config files load). auto_env turns on platform environments: mise then # loads .mise/config..toml automatically for the current OS -- so -# config.windows.toml loads on Windows and config.macos.toml on macOS, with no -# manual MISE_ENV. It's harmless elsewhere (there's no config.linux.toml etc.), -# and setting it here also silences mise's 2026.12 "auto_env is becoming default" -# rollout warning. See https://mise.jdx.dev/configuration/environments.html +# config.linux.toml on Linux, config.macos.toml on macOS, and config.windows.toml +# on Windows, with no manual MISE_ENV. Setting it here also silences mise's +# 2026.12 "auto_env is becoming default" rollout warning. +# See https://mise.jdx.dev/configuration/environments.html auto_env = true diff --git a/Dockerfile b/Dockerfile index 035d94d..86304b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,10 @@ RUN apt-get update \ # then resolves the workspace tools. RUN curl -fsSL https://mise.run | sh ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" +# cargo-expand is a dev macro-debugging convenience with no prebuilt binary, so +# in the image it would be a slow from-source cargo build for no build-time use. +# Skip it (inherited by every stage below). Linters/builds/tests don't need it. +ENV MISE_DISABLE_TOOLS=cargo:cargo-expand WORKDIR /workspace # The default tools need the always-loaded config plus config.linux.toml (where diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index 362fb31..8201575 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -207,12 +207,14 @@ ENV PATH="C:\Users\ContainerAdministrator\AppData\Local\mise\installs\python\3.1 RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN= Date: Sun, 14 Jun 2026 09:39:51 +0800 Subject: [PATCH 51/60] docs(mise): trim the linux var-prefix comment Co-Authored-By: Claude Opus 4.8 --- .mise/config.linux.toml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml index 2f13b10..151f4b8 100644 --- a/.mise/config.linux.toml +++ b/.mise/config.linux.toml @@ -9,12 +9,9 @@ "conda:clangxx" = "latest" [vars] -# An `a_`/`b_`/`c_` prefix encodes each var's dependency tier so the alphabetical -# order taplo's reorder_keys imposes keeps every definition before its use: `a_` -# vars stand alone, `b_` vars read `a_` vars, `c_` reads `b_`. Without it sorting -# moves a definition after the var that reads it and `mise install` can't render -# the value. `a_conda_arch` picks conda's target-triple dir (x86_64/aarch64, -# vs mise's x64/arm64). +# `a_`/`b_`/`c_` tiers keep taplo's alphabetical reorder_keys from moving a var +# ahead of one it reads (`b_` reads `a_`, `c_` reads `b_`), which mise can't render. +# a_conda_arch: conda's target-triple dir (x86_64/aarch64 vs mise's x64/arm64). a_conda_arch = "{% if arch() == 'arm64' %}aarch64{% else %}x86_64{% endif %}" a_conda_clangxx = "{{ env.HOME }}/.local/share/mise/installs/conda-clangxx/latest" b_clang_resource = "{{ vars.a_conda_clangxx }}/lib/clang/22" From 5a7ea830441164a643265468ae38e212a205a6bd Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 10:00:18 +0800 Subject: [PATCH 52/60] fix(ci): wasm-pack wasm-bindgen resolution + mise tooling - Revert `wasm-pack --mode no-install` on build-ws-wasm-agent: it stopped wasm-pack downloading wasm-bindgen but also couldn't use the mise-provided one, failing every rust/build job with "Not able to find or install a local wasm-bindgen". - Bump github:wasm-bindgen 0.2.114 -> 0.2.122 to match Cargo.lock, so wasm-pack finds the matching binary on PATH and downloads nothing (what --mode no-install was meant to do). - Add the mise-no-source-backends taplo schema: ban `cargo:`/`ubi:` tool backends in .mise/config*.toml, allowlisting cargo-expand + dart-typegen. - Trim the exec_auto_install / task.run_auto_install comment. Co-Authored-By: Claude Opus 4.8 --- .mise/config.toml | 28 +++++++++---------- .../taplo/mise-no-source-backends.schema.json | 17 +++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 config/taplo/mise-no-source-backends.schema.json diff --git a/.mise/config.toml b/.mise/config.toml index 90dece2..9f01f03 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -18,15 +18,11 @@ min_version = "2026.6.5" [settings] -# Never auto-install tools implicitly: installs are explicit (`mise install` / -# `mise run install-all`, and CI/Docker install up front). This keeps cheap tasks -# like `print-all-langs`, and the pre-install `setup-aube` step, from eagerly -# pulling the whole toolchain — on CI and the CLI alike. task.run_auto_install -# covers `mise run`; exec_auto_install covers `mise exec` (its OWN knob) — -# without it, `mise exec -- python …` in Dockerfile.nanoserver installs *every* -# configured tool before running python. Turn both off so the policy holds. -# (Current mise accepts the nested task.run_auto_install form; very old builds -# like 2026.3 used the flat `task_run_auto_install`.) +# Keep installs explicit (`mise install`; CI/Docker install up front) so a task or +# `mise exec` never pulls the whole toolchain implicitly — e.g. `mise exec -- +# python …` in Dockerfile.nanoserver would otherwise install every tool first. +# The two knobs are separate: exec_auto_install covers `mise exec`, +# task.run_auto_install covers `mise run`. exec_auto_install = false task.run_auto_install = false @@ -60,7 +56,9 @@ editorconfig-checker = "latest" # below (see the `retry` var) so a transient HTTP 429 doesn't fail the build. "github:dbohdan/recur" = "latest" "github:grok-rs/waitup" = "latest" -"github:wasm-bindgen/wasm-bindgen" = "0.2.114" +# Must match the wasm-bindgen crate version in Cargo.lock so wasm-pack finds this +# on PATH instead of downloading its own wasm-bindgen-cli; bump both together. +"github:wasm-bindgen/wasm-bindgen" = "0.2.122" hadolint = "latest" # Link checker for the `link-check` task. Installs on every OS; the Nano Docker # image opts out via MISE_DISABLE_TOOLS (link-check isn't run in the Windows @@ -331,6 +329,10 @@ grep -vxF './generated/rust-rest/Cargo.toml' "$members" | # re-sort an array and scramble an ordered command sequence (use a multiline # string, one command per line). Applies to every .mise/config*.toml. taplo lint --schema "file://$PWD/config/taplo/mise-run-not-array.schema.json" .mise/config*.toml + +# Tools must use a prebuilt backend; `cargo:`/`ubi:` build from source. +# Allowlisted exceptions: cargo:cargo-expand + cargo:dart-typegen. +taplo lint --schema "file://$PWD/config/taplo/mise-no-source-backends.schema.json" .mise/config*.toml """ shell = "bash -euo pipefail -c" @@ -584,11 +586,7 @@ shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent] description = "Build the WebSocket WASM client" dir = "services/ws-wasm-agent" -# --mode no-install: use the mise-pinned wasm-bindgen (and skip wasm-opt) instead -# of letting wasm-pack download its own. The wasm32 target + rust-src come from -# the `rust` tool's targets/components (Windows re-adds them to its gnullvm -# toolchains in preinstall), so nothing is fetched at build time. -run = "wasm-pack build . --target web --mode no-install -- -Z build-std=std,panic_abort" +run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" [tasks.build-ws-wasm-agent.env] # Scope these flags to the wasm target rather than setting plain `RUSTFLAGS`: diff --git a/config/taplo/mise-no-source-backends.schema.json b/config/taplo/mise-no-source-backends.schema.json new file mode 100644 index 0000000..10e96ee --- /dev/null +++ b/config/taplo/mise-no-source-backends.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise [tools] -- no source-built backends", + "description": "`cargo:`/`ubi:` build from source; prefer a prebuilt backend (aqua/registry/github/http).", + "type": "object", + "properties": { + "tools": { + "type": "object", + "propertyNames": { + "anyOf": [ + { "not": { "pattern": "^(cargo|ubi):" } }, + { "enum": ["cargo:cargo-expand", "cargo:dart-typegen"] } + ] + } + } + } +} From 5f4d6ba52fcfa8c7ae7e6913665ddb438148d1f2 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 10:58:46 +0800 Subject: [PATCH 53/60] conftest and a nano fix --- .mise/config.dart.toml | 1 + .mise/config.toml | 46 ++++-- .mise/config.windows.toml | 9 +- CLAUDE.md | 10 ++ Dockerfile | 5 +- Dockerfile.nanoserver | 11 +- README.md | 27 +--- config/conftest/policy/cargo.rego | 148 ++++++++++++++++++ config/conftest/policy/mise.rego | 59 +++++++ config/conftest/policy/wasm-bindgen-sync.rego | 29 ++++ config/lychee.toml | 7 +- 11 files changed, 303 insertions(+), 49 deletions(-) create mode 100644 config/conftest/policy/cargo.rego create mode 100644 config/conftest/policy/mise.rego create mode 100644 config/conftest/policy/wasm-bindgen-sync.rego diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 7104de7..536c5b4 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -7,6 +7,7 @@ dart_release = "https://storage.googleapis.com/dart-archive/channels/stable/rele dart_bucket = "https://storage.googleapis.com/storage/v1/b/dart-archive/o" [tools] +# cargo: backend -- dart-typegen has no prebuilt binary (crates.io only). "cargo:dart-typegen" = "latest" # dart isn't an aqua/registry tool — install it straight from Google's dart diff --git a/.mise/config.toml b/.mise/config.toml index 9f01f03..15c8034 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -35,9 +35,9 @@ cargo-binstall = "latest" # the one tool here still built from source. "aqua:rustwasm/wasm-pack" = "latest" "cargo:cargo-expand" = "latest" -# aqua ships a prebuilt taplo binary (no `ring` C build). Off Windows regardless: -# taplo-check (TOML linting) runs only in the Linux check job. -taplo = { version = "latest", os = ["linux", "macos"] } +# aqua ships a prebuilt taplo binary, so it installs on every OS (no `ring` C +# build, which is what previously kept it off the gnu Windows image). +taplo = "latest" watchexec = "latest" # vfox-chromedriver's PostInstall hook fails on Windows ("syntax of the # command is incorrect"). Chromedriver is only used by the @@ -50,6 +50,9 @@ cmake = "latest" # macOS, the msys2/llvm-mingw/busybox set on Windows) live in the matching # config..toml. "conda:openssl" = "3" +# conftest runs the repo's OPA/Rego policies (config/conftest/policy) -- the +# cross-file config invariants taplo's per-file JSON schemas can't express. +conftest = "latest" dprint = "latest" editorconfig-checker = "latest" # Retries a command with exponential backoff + jitter; wraps the model fetches @@ -217,6 +220,7 @@ depends = [ "gen-specs-check", "hadolint-check", "semgrep-check", + "conftest-check", "taplo-check", "typos", "verification-check", @@ -275,22 +279,36 @@ run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace [tasks.taplo-fmt] run = "taplo format" +[tasks.conftest-check] +description = "Run conftest OPA/Rego policies (config/conftest/policy) over config + lock files" +# --combine feeds every listed file to one policy evaluation as an array of +# {path, contents}, so a policy can both check each file and compare across files +# (the mise wasm-bindgen pin vs Cargo.lock). --parser toml: Cargo.lock has no +# .toml suffix. git ls-files enumerates the tracked Cargo.toml + mise configs. +run = """ +git ls-files '*Cargo.toml' '.mise/config*.toml' Cargo.lock | + xargs conftest test --combine --parser toml --all-namespaces -p config/conftest/policy +""" +shell = "bash -euo pipefail -c" + [tasks.taplo-check] # `taplo lint` reads config/taplo.toml's `[[rule]] schema` entries for editor / # LSP-style validation, but silently ignores their nested constraints in # CLI mode (verified: injecting `path = "../foo"` into a member Cargo.toml # doesn't fire `no-path-deps` via the rule alone). We work around it by # applying each schema explicitly with `taplo lint --schema file://...`. -# When adding a new schema under `config/taplo/`, append a `taplo lint` -# line here too. See `config/taplo/RULE-IDEAS.md` for the investigation. +# conftest-check (config/conftest/policy) intentionally duplicates these lints; +# running both is fine. See `config/taplo/RULE-IDEAS.md` for the investigation. run = """ -# taplo-cli isn't installed on Windows (its `ring` dep needs a C/asm toolchain -# the gnu Nano build lacks); TOML linting runs in the Linux check job. Skip here. -[ "${OS:-}" = "Windows_NT" ] && { echo "taplo-check: skipped on Windows"; exit 0; } - # Baseline TOML validation across every .toml file the auto-config picks up. +# Runs on every OS (taplo is a prebuilt binary now). taplo lint +# The --schema lints below pass file://$PWD URLs, which aren't well-formed on +# Windows (drive-letter paths); run them on Linux/macOS only. conftest-check +# applies the same rules on every OS. +[ "${OS:-}" = "Windows_NT" ] && exit 0 + # All member Cargo.tomls — exclude workspace root and any build output. # `-prune` target/ and .git so find never descends into them: `-not -path` # alone still walks the whole tree, and target/debug/deps/ churns constantly @@ -586,7 +604,15 @@ shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent] description = "Build the WebSocket WASM client" dir = "services/ws-wasm-agent" -run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" +# wasm-opt (binaryen) can't execute on the Nano Server image (it needs more of the +# VC++ runtime than the image carries), so skip it on Windows -- that build is only +# a smoke-test; the served, size-optimised artifact is built on Linux/macOS. +run = """ +opt="" +[ "${OS:-}" = "Windows_NT" ] && opt="--no-opt" +wasm-pack build . --target web $opt -- -Z build-std=std,panic_abort +""" +shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent.env] # Scope these flags to the wasm target rather than setting plain `RUSTFLAGS`: diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index f636bb5..ce54905 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -48,11 +48,10 @@ win_libclang = "C:\\Program Files\\LLVM\\bin" MISE_BASH_PATH = "{{ vars.winsh }}" # Build for the LLVM mingw target. gnullvm, not gnu: llvm-mingw ships # compiler-rt + libunwind, not the GCC runtime (libgcc/libgcc_eh) the gnu -# target's link line demands. preinstall flips the rustup host to gnullvm too, -# so build scripts/proc-macros (built for the host) link the same way. The linker -# is clang and the raw-dylib import tool is llvm-dlltool, both from llvm-mingw on -# PATH. These can live in [env] here (unlike config.toml) because this file only -# loads on Windows, so there's no empty off-Windows value to break Linux cargo. +# target's link line demands. The linker is clang and the raw-dylib import tool +# is llvm-dlltool, both from llvm-mingw on PATH. These can live in [env] here +# (unlike config.toml) because this file only loads on Windows, so there's no +# empty off-Windows value to break Linux cargo. CARGO_BUILD_TARGET = "x86_64-pc-windows-gnullvm" CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_LINKER = "clang" CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTFLAGS = "-Cdlltool=llvm-dlltool" diff --git a/CLAUDE.md b/CLAUDE.md index e8273c2..cddc6d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -255,6 +255,16 @@ If a function is private but needs testing, add a `[lib]` target to the crate an Every file under `tests/` must start with `#![cfg(test)]` (placed after the file's `//!` doc comment, if any). +## Tools must work on every OS + +Every tool in the `.mise/config*.toml` `[tools]` tables must install and run on +every supported OS (Linux, macOS, Windows). Do **not** `os`-scope a tool, or +otherwise skip it on a platform, without explicit operator permission — prefer a +prebuilt-binary backend (aqua/github/http) over a `cargo:` source build, which is +usually what forces a platform exclusion. The one place tool skips need no +permission is the Dockerfiles (`MISE_DISABLE_TOOLS`), where trimming an image to +just what its build needs is expected. + ## Linting Lint checks must be expressed through one of the repo's linters — **never** as a diff --git a/Dockerfile b/Dockerfile index 86304b2..c45132a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,9 +69,8 @@ RUN apt-get update \ # then resolves the workspace tools. RUN curl -fsSL https://mise.run | sh ENV PATH="/root/.local/bin:/root/.local/share/mise/shims:${PATH}" -# cargo-expand is a dev macro-debugging convenience with no prebuilt binary, so -# in the image it would be a slow from-source cargo build for no build-time use. -# Skip it (inherited by every stage below). Linters/builds/tests don't need it. +# cargo-expand is a dev-only macro-debugging tool with no role in image builds, +# so skip it here (this ENV is inherited by every stage below). ENV MISE_DISABLE_TOOLS=cargo:cargo-expand WORKDIR /workspace diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index 8201575..5cffcfa 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -199,11 +199,8 @@ RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN= rejection reason. Members must use workspace = true, so the +# root's [workspace.dependencies] is the only place a ban can bite. +banned := { + "anyhow": "define a thiserror enum instead", + "ring": "use aws-lc-rs (transitive via rcgen only; gated in config/deny.toml)", + "ureq": "use reqwest::blocking or reqwest -- one HTTPS stack only", +} + +deny contains msg if { + some [path, name, _] in dep + path == "Cargo.toml" + reason := banned[name] + msg := sprintf("%s: banned dependency %q -- %s", [path, name, reason]) +} + +# Member crates: no path deps, no wildcard versions, no inline git deps. Pins live +# in the root [workspace.dependencies]; members reference them via workspace = true. +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + is_object(spec) + spec.path + msg := sprintf("%s: dependency %q uses a path dep; use workspace = true instead", [path, name]) +} + +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + wildcard(spec) + msg := sprintf("%s: dependency %q uses a wildcard version; pin via [workspace.dependencies]", [path, name]) +} + +deny contains msg if { + some [path, name, spec] in dep + path != "Cargo.toml" + is_object(spec) + spec.git + msg := sprintf("%s: dependency %q is an inline git dep; pin via [workspace.dependencies]", [path, name]) +} + +wildcard(spec) if { + is_string(spec) + contains(spec, "*") +} + +wildcard(spec) if { + is_object(spec) + contains(spec.version, "*") +} + +# Member [dependencies]/[dev-dependencies] must inherit via workspace = true so +# pins stay in [workspace.dependencies]. (build-dependencies not covered yet.) +deny contains msg if { + some file in input + is_member(file) + some table in {"dependencies", "dev-dependencies"} + some name, spec in file.contents[table] + not spec.workspace == true + msg := sprintf("%s: dependency %q must reference [workspace.dependencies] via workspace = true", [file.path, name]) +} + +# A crate with a [lib] must disable the doctest harness (runnable examples belong +# in tests/ files) and must not rename the lib (keep it the package name). +deny contains msg if { + some file in input + is_member(file) + file.contents.lib + not file.contents.lib.doctest == false + msg := sprintf("%s: [lib] must set doctest = false", [file.path]) +} + +deny contains msg if { + some file in input + is_member(file) + file.contents.lib.name + msg := sprintf("%s: [lib] must not set name (keep it the package name)", [file.path]) +} + +# Every member must inherit the workspace lint tables. generated/rust-rest is +# exempt: progenitor's emitted source trips lints the workspace table denies. +deny contains msg if { + some file in input + is_member(file) + file.path != "generated/rust-rest/Cargo.toml" + not file.contents.lints.workspace == true + msg := sprintf("%s: add [lints] workspace = true", [file.path]) +} + +# Every crate must be a registered workspace member: its directory must appear in +# the root manifest's explicit [workspace].members list (no orphan crates). +workspace_member contains m if { + some file in input + file.path == "Cargo.toml" + some m in file.contents.workspace.members +} + +deny contains msg if { + some file in input + is_member(file) + dir := trim_suffix(file.path, "/Cargo.toml") + not workspace_member[dir] + msg := sprintf("%s: crate is not registered in the root [workspace].members", [file.path]) +} + +# Shared [package] metadata must be inherited from [workspace.package] via +# `.workspace = true`, so the values stay defined in exactly one place. +inherited_package_field := {"edition", "license", "repository"} + +deny contains msg if { + some file in input + is_member(file) + some field in inherited_package_field + not file.contents.package[field].workspace == true + msg := sprintf("%s: [package] %s must inherit via %s.workspace = true", [file.path, field, field]) +} diff --git a/config/conftest/policy/mise.rego b/config/conftest/policy/mise.rego new file mode 100644 index 0000000..2609684 --- /dev/null +++ b/config/conftest/policy/mise.rego @@ -0,0 +1,59 @@ +# .mise/config*.toml policy, evaluated over conftest's `--combine` input (an array +# of {path, contents}). Run with `--namespace mise` (or `--all-namespaces`). +package mise + +is_mise(file) if startswith(file.path, ".mise/config") + +# A task `run` must be a string, not an array: taplo's reorder_arrays would +# re-sort the commands of an array form and scramble the sequence. +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_array(task.run) + msg := sprintf("%s: task %q run must be a string, not an array", [file.path, name]) +} + +# A multiline `run` must use `shell = "bash -euo pipefail -c"` so a failing +# command fails the task instead of being masked. +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_string(task.run) + contains(task.run, "\n") + not task.shell == "bash -euo pipefail -c" + msg := sprintf("%s: task %q has a multiline run; set shell = \"bash -euo pipefail -c\"", [file.path, name]) +} + +# Task descriptions must be single-line (keep them under the 120-char limit). +deny contains msg if { + some file in input + is_mise(file) + some name, task in file.contents.tasks + is_string(task.description) + contains(task.description, "\n") + msg := sprintf("%s: task %q description must be a single line", [file.path, name]) +} + +# `cargo:` tools build from source; prefer a prebuilt backend. Allowlist the two +# that have no prebuilt binary. +allowed_cargo_tool := {"cargo:cargo-expand", "cargo:dart-typegen"} + +deny contains msg if { + some file in input + is_mise(file) + some name, _ in file.contents.tools + startswith(name, "cargo:") + not allowed_cargo_tool[name] + msg := sprintf("%s: tool %q builds from source; use a prebuilt backend", [file.path, name]) +} + +# `ubi:` is deprecated; the `http:` backend replaces it. +deny contains msg if { + some file in input + is_mise(file) + some name, _ in file.contents.tools + startswith(name, "ubi:") + msg := sprintf("%s: tool %q uses the deprecated ubi backend; use http: instead", [file.path, name]) +} diff --git a/config/conftest/policy/wasm-bindgen-sync.rego b/config/conftest/policy/wasm-bindgen-sync.rego new file mode 100644 index 0000000..9962ae8 --- /dev/null +++ b/config/conftest/policy/wasm-bindgen-sync.rego @@ -0,0 +1,29 @@ +# Cross-file invariants, evaluated over conftest's `--combine` input (an array of +# {path, contents}). Run with `--namespace cross`. +package cross + +# The mise `github:wasm-bindgen` pin must equal the wasm-bindgen package version +# in Cargo.lock. wasm-pack requires the wasm-bindgen CLI to match the crate +# version exactly; when they match it uses the on-PATH (mise) binary, otherwise it +# downloads its own. Keeping them equal avoids that download. +mise_pin := pin if { + some file in input + endswith(file.path, ".mise/config.toml") + pin := file.contents.tools["github:wasm-bindgen/wasm-bindgen"] +} + +lock_version := ver if { + some file in input + endswith(file.path, "Cargo.lock") + some pkg in file.contents.package + pkg.name == "wasm-bindgen" + ver := pkg.version +} + +deny contains msg if { + mise_pin != lock_version + msg := sprintf( + "wasm-bindgen: mise pin %q != Cargo.lock %q; bump the pin in .mise/config.toml", + [mise_pin, lock_version], + ) +} diff --git a/config/lychee.toml b/config/lychee.toml index 9282f12..4c81999 100644 --- a/config/lychee.toml +++ b/config/lychee.toml @@ -16,11 +16,10 @@ max_cache_age = "2d" # we're just rate-limited. accept = ["200..=299", "429"] -# Skip loopback/reserved hosts (localhost, 127.0.0.1) -- not real external links -# -- and the generated/ + target/ trees (generated clients + build output, not -# our prose; matches the other linters' exclusions). +# Skip loopback/reserved hosts (localhost, 127.0.0.1) exclude_loopback = true -exclude_path = ["generated", "target"] + +exclude_path = ["data", "generated", "target"] # URL patterns (regex) to skip -- add flaky or auth-gated hosts here. exclude = [] From 4d925cd4e7ebdc75bbdf764a69780a493a74ba05 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 11:32:10 +0800 Subject: [PATCH 54/60] style checks --- .editorconfig | 6 +++++ .mise/config.toml | 27 ++++++++++++++----- Cargo.lock | 20 +++++++------- Cargo.toml | 2 +- config/conftest/policy/cargo.rego | 27 +++++++++++++++++++ config/conftest/policy/gha.rego | 21 +++++++++++++++ config/semgrep/prefer-yaml-toml.yml | 21 +++++++++++++++ libs/otlp-mock/Cargo.toml | 3 ++- services/ws-wasi-runner/Cargo.toml | 2 +- .../ws-wasi-runner/tests/otel_propagation.rs | 2 +- 10 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 config/conftest/policy/gha.rego create mode 100644 config/semgrep/prefer-yaml-toml.yml diff --git a/.editorconfig b/.editorconfig index 2f0e8b4..5276445 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,6 +12,12 @@ trim_trailing_whitespace = true [*.md] indent_size = unset +# OPA/Rego: `conftest fmt` (opa fmt) indents with tabs and isn't configurable, so +# its canonical formatting needs tabs, not the repo's space default. +[*.rego] +indent_style = tab +indent_size = unset + # License files use the canonical upstream formatting (centred headers, odd # indent widths, etc.) — leave them alone. [LICENSE-*] diff --git a/.mise/config.toml b/.mise/config.toml index 15c8034..b0fa952 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -280,32 +280,47 @@ run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace run = "taplo format" [tasks.conftest-check] -description = "Run conftest OPA/Rego policies (config/conftest/policy) over config + lock files" +# Native `depends` glob: every conftest-check- task (toml, yaml, ...) +# joins automatically, so adding a new parser needs no edit here. +depends = ["conftest-check-*"] +description = "Run all conftest policy checks (one per parsed file format)" + +[tasks.conftest-check-toml] +description = "Run conftest OPA/Rego policies over the TOML config + lock files" # --combine feeds every listed file to one policy evaluation as an array of # {path, contents}, so a policy can both check each file and compare across files # (the mise wasm-bindgen pin vs Cargo.lock). --parser toml: Cargo.lock has no -# .toml suffix. git ls-files enumerates the tracked Cargo.toml + mise configs. +# .toml suffix. The TOML rule namespaces are listed explicitly, so the per-file +# `gha` (YAML) namespace is never evaluated against this combined TOML input. run = """ +ns="--namespace cross --namespace cargo --namespace mise" git ls-files '*Cargo.toml' '.mise/config*.toml' Cargo.lock | - xargs conftest test --combine --parser toml --all-namespaces -p config/conftest/policy + xargs conftest test --combine --parser toml $ns -p config/conftest/policy """ shell = "bash -euo pipefail -c" +[tasks.conftest-check-yaml] +description = "Run conftest OPA/Rego policies over the GitHub Actions workflow YAML" +# Each workflow is checked on its own (no cross-file YAML rules) -- no --combine; +# conftest reads the directory directly. --namespace gha selects only the YAML +# policy. Replicates the gha-* ast-grep rules; running both is fine. +run = "conftest test --parser yaml --namespace gha -p config/conftest/policy .github/workflows" + [tasks.taplo-check] # `taplo lint` reads config/taplo.toml's `[[rule]] schema` entries for editor / # LSP-style validation, but silently ignores their nested constraints in # CLI mode (verified: injecting `path = "../foo"` into a member Cargo.toml # doesn't fire `no-path-deps` via the rule alone). We work around it by # applying each schema explicitly with `taplo lint --schema file://...`. -# conftest-check (config/conftest/policy) intentionally duplicates these lints; -# running both is fine. See `config/taplo/RULE-IDEAS.md` for the investigation. +# conftest-check-toml (config/conftest/policy) intentionally duplicates these +# lints; running both is fine. run = """ # Baseline TOML validation across every .toml file the auto-config picks up. # Runs on every OS (taplo is a prebuilt binary now). taplo lint # The --schema lints below pass file://$PWD URLs, which aren't well-formed on -# Windows (drive-letter paths); run them on Linux/macOS only. conftest-check +# Windows (drive-letter paths); run them on Linux/macOS only. conftest-check-toml # applies the same rules on every OS. [ "${OS:-}" = "Windows_NT" ] && exit 0 diff --git a/Cargo.lock b/Cargo.lock index eab3e17..8e1a00e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4645,10 +4645,10 @@ dependencies = [ "et-ws-runner-common", "et-ws-test-server", "futures-util", + "int-otlp-mock", "opentelemetry 0.31.0", "opentelemetry-http 0.31.0", "ort", - "otlp-mock", "pollster", "reqwest 0.13.4", "rstest", @@ -6259,6 +6259,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "int-otlp-mock" +version = "0.1.0" +dependencies = [ + "actix-rt", + "actix-web", + "serde_json", +] + [[package]] name = "inventory" version = "0.3.24" @@ -7948,15 +7957,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "otlp-mock" -version = "0.1.0" -dependencies = [ - "actix-rt", - "actix-web", - "serde_json", -] - [[package]] name = "outref" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 9ab0c7b..7c56685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,7 @@ opentelemetry-otlp = { version = "0.31", default-features = false, features = [ ] } opentelemetry_sdk = "0.31" ort = { version = "=2.0.0-rc.10", default-features = false, features = ["copy-dylibs", "download-binaries"] } -otlp-mock = { path = "libs/otlp-mock", version = "0.1.0" } +int-otlp-mock = { path = "libs/otlp-mock", version = "0.1.0" } pollster = "0.4" pretty_yaml = "0.6" prettyplease = "0.2" diff --git a/config/conftest/policy/cargo.rego b/config/conftest/policy/cargo.rego index 2a19ac3..1132af2 100644 --- a/config/conftest/policy/cargo.rego +++ b/config/conftest/policy/cargo.rego @@ -146,3 +146,30 @@ deny contains msg if { not file.contents.package[field].workspace == true msg := sprintf("%s: [package] %s must inherit via %s.workspace = true", [file.path, field, field]) } + +# Crate names are namespaced: "edge-toolkit" or "et-" for normal crates, "int-" +# for internal (publish = false) ones. +allowed_crate_name(name, _) if startswith(name, "edge-toolkit") + +allowed_crate_name(name, _) if startswith(name, "et-") + +allowed_crate_name(name, pkg) if { + startswith(name, "int-") + pkg.publish == false +} + +deny contains msg if { + some file in input + is_member(file) + name := file.contents.package.name + not allowed_crate_name(name, file.contents.package) + msg := sprintf("%s: crate name %q must start with edge-toolkit/et- (int- if publish=false)", [file.path, name]) +} + +# An empty `features = []` on a dependency is pointless noise -- drop it. +deny contains msg if { + some [path, name, spec] in dep + is_object(spec) + spec.features == [] + msg := sprintf("%s: dependency %q has an empty features = []; remove it", [path, name]) +} diff --git a/config/conftest/policy/gha.rego b/config/conftest/policy/gha.rego new file mode 100644 index 0000000..db9ddc8 --- /dev/null +++ b/config/conftest/policy/gha.rego @@ -0,0 +1,21 @@ +# GitHub Actions workflow policy, evaluated per file: conftest reads each .yml +# independently (no --combine, since there are no cross-file YAML rules). +# Replicates the gha-* ast-grep rules; running both is fine. Selected with +# `--namespace gha`, so it only runs against workflow YAML, never the TOML inputs. +package gha + +# Every workflow must set the default run shell to bash, so steps run in bash on +# every runner (Windows included) without a per-step `shell:`. +deny contains msg if { + not input.defaults.run.shell == "bash" + msg := "workflow must set defaults.run.shell: bash" +} + +# Steps must not override the shell -- rely on the workflow default (write the +# step in bash rather than switching to PowerShell on Windows runners). +deny contains msg if { + some name, job in input.jobs + some step in job.steps + step.shell + msg := sprintf("job %q sets shell: on a step; use the workflow default (bash)", [name]) +} diff --git a/config/semgrep/prefer-yaml-toml.yml b/config/semgrep/prefer-yaml-toml.yml new file mode 100644 index 0000000..a1b8407 --- /dev/null +++ b/config/semgrep/prefer-yaml-toml.yml @@ -0,0 +1,21 @@ +rules: + - id: prefer-yaml-toml-over-json + languages: [generic] + paths: + include: + - "*.json" + - "*.jsonc" + - "*.jsonl" + exclude: + # Formats whose tooling mandates JSON/JSONC -- everything else should be + # YAML or TOML. Add a file here only if its tool can't read YAML/TOML. + - "*.schema.json" # JSON Schema documents + - "package.json" # npm manifests + - "/.vscode/*.json" # editor config + - "dprint.jsonc" # dprint config + - ".dprint.jsonc" + pattern-regex: \A + message: >- + Prefer YAML or TOML over JSON/JSONC/JSONL for config. If a tool requires + this format, add the file to this rule's paths.exclude allowlist. + severity: ERROR diff --git a/libs/otlp-mock/Cargo.toml b/libs/otlp-mock/Cargo.toml index 3a617f8..8007e21 100644 --- a/libs/otlp-mock/Cargo.toml +++ b/libs/otlp-mock/Cargo.toml @@ -1,5 +1,6 @@ [package] -name = "otlp-mock" +name = "int-otlp-mock" +publish = false description = "In-process mock OTLP/HTTP-JSON collector for trace-propagation tests" version = "0.1.0" edition.workspace = true diff --git a/services/ws-wasi-runner/Cargo.toml b/services/ws-wasi-runner/Cargo.toml index 92e39d3..a793ae0 100644 --- a/services/ws-wasi-runner/Cargo.toml +++ b/services/ws-wasi-runner/Cargo.toml @@ -64,7 +64,7 @@ wgpu.workspace = true [dev-dependencies] et-ws-test-server.workspace = true -otlp-mock.workspace = true +int-otlp-mock.workspace = true rstest.workspace = true [lints] diff --git a/services/ws-wasi-runner/tests/otel_propagation.rs b/services/ws-wasi-runner/tests/otel_propagation.rs index 04fdd53..e07754b 100644 --- a/services/ws-wasi-runner/tests/otel_propagation.rs +++ b/services/ws-wasi-runner/tests/otel_propagation.rs @@ -41,7 +41,7 @@ use edge_toolkit::config::{OtlpConfig, OtlpProtocol}; #[cfg_attr(windows, ignore = "pkg/package.json 404 on Windows -- see comment above")] fn trace_ids_propagate_between_runner_and_server() { // 1. Start the mock collector. Both processes will export to it. - let mock = otlp_mock::start(); + let mock = int_otlp_mock::start(); // 2. Init OTLP in the test process *before* spawning the test server, // so the global tracing subscriber + propagator are in place when From 084e3b5dfa64f294423ad4835889131a1f022c15 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 12:48:22 +0800 Subject: [PATCH 55/60] ci(windows): skip wasm-opt on Nano for all wasm-pack builds build-modules failed on the Nano image at build-ws-audio1-module: wasm-opt (binaryen) can't execute there (os error 3). The earlier --no-opt fix only covered build-ws-wasm-agent; the standard module builds still ran wasm-opt. Gate it via a shared {{ vars.no_opt }} (empty default, --no-opt on Windows) used by all 12 wasm-pack module builds and the wasm-agent. wasm-opt still runs on Linux/macOS, where the shipped browser artifacts are built. Co-Authored-By: Claude Opus 4.8 --- .mise/config.rust.toml | 24 ++++++++++++------------ .mise/config.toml | 15 ++++++--------- .mise/config.windows.toml | 3 +++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 044a9ec..07ba43e 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -14,7 +14,7 @@ [tasks.build-ws-data1-module] description = "Build the data1 workflow WASM module" dir = "services/ws-modules/data1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-har1-module] description = "Build the har1 workflow WASM module" @@ -23,59 +23,59 @@ dir = "services/ws-modules/har1" # shells out to `cargo metadata`, and that subprocess can't find `cargo` under # Git Bash on Windows. The default shell (cmd there) resolves cargo like the # single-line wasm-pack modules do. -run = "wasm-pack build . --target web && cargo run -p et-cli -- module-package-json" +run = "wasm-pack build . --target web {{ vars.no_opt }} && cargo run -p et-cli -- module-package-json" [tasks.build-ws-face-detection-module] description = "Build the face detection workflow WASM module" dir = "services/ws-modules/face-detection" # See build-ws-har1-module: `&&` on one line under the default shell so # wasm-pack's `cargo metadata` subprocess can find cargo on Windows. -run = "wasm-pack build . --target web && cargo run -p et-cli -- module-package-json" +run = "wasm-pack build . --target web {{ vars.no_opt }} && cargo run -p et-cli -- module-package-json" [tasks.build-ws-comm1-module] description = "Build the comm1 workflow WASM module" dir = "services/ws-modules/comm1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-sensor1-module] description = "Build the sensor1 workflow WASM module" dir = "services/ws-modules/sensor1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-audio1-module] description = "Build the audio1 workflow WASM module" dir = "services/ws-modules/audio1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-video1-module] description = "Build the video1 workflow WASM module" dir = "services/ws-modules/video1" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-bluetooth-module] description = "Build the bluetooth workflow WASM module" dir = "services/ws-modules/bluetooth" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-geolocation-module] description = "Build the geolocation workflow WASM module" dir = "services/ws-modules/geolocation" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-graphics-info-module] description = "Build the graphics info workflow WASM module" dir = "services/ws-modules/graphics-info" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-speech-recognition-module] description = "Build the speech recognition workflow WASM module" dir = "services/ws-modules/speech-recognition" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-nfc-module] description = "Build the nfc workflow WASM module" dir = "services/ws-modules/nfc" -run = "wasm-pack build . --target web" +run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-wasi-data1-module] description = "Build the Rust WASI data1 module as a WASI Preview 2 component" diff --git a/.mise/config.toml b/.mise/config.toml index b0fa952..46607fc 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -114,6 +114,9 @@ rpath_flag = "-C link-arg=-Wl,-rpath,{{ vars.conda_openssl }}/lib" # the stable install path instead. pylib_flag = "-C link-arg=-Wl,-rpath,{{ env.HOME }}/.local/share/mise/installs/python/3.13/lib" wasm_rustflags = "-C target-cpu=mvp -C target-feature=+mutable-globals,+sign-ext,+nontrapping-fptoint" +# Extra flag for the wasm-pack module builds; empty so wasm-opt runs. +# config.windows.toml overrides it to --no-opt where wasm-opt can't execute. +no_opt = "" # Source/dest paths for the fetch-*-rclone model downloads — kept as vars so # each `rclone copyto` command stays on a single line without continuations. face1_dst = "data/model-modules/model-face1/pkg/video_cv.onnx" @@ -619,15 +622,9 @@ shell = "bash -euo pipefail -c" [tasks.build-ws-wasm-agent] description = "Build the WebSocket WASM client" dir = "services/ws-wasm-agent" -# wasm-opt (binaryen) can't execute on the Nano Server image (it needs more of the -# VC++ runtime than the image carries), so skip it on Windows -- that build is only -# a smoke-test; the served, size-optimised artifact is built on Linux/macOS. -run = """ -opt="" -[ "${OS:-}" = "Windows_NT" ] && opt="--no-opt" -wasm-pack build . --target web $opt -- -Z build-std=std,panic_abort -""" -shell = "bash -euo pipefail -c" +# {{ vars.no_opt }} is --no-opt on Windows (wasm-opt can't run on Nano), empty +# elsewhere -- the served, size-optimised artifact is built on Linux/macOS. +run = "wasm-pack build . --target web {{ vars.no_opt }} -- -Z build-std=std,panic_abort" [tasks.build-ws-wasm-agent.env] # Scope these flags to the wasm target rather than setting plain `RUSTFLAGS`: diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index ce54905..607a1fa 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -40,6 +40,9 @@ py313_win = "{{ get_env(name='LOCALAPPDATA', default='') }}\\mise\\installs\\pyt # clang-sys finds libclang on PATH (Docker's gnu build has llvm-mingw there); a # workstation with system LLVM also works. win_libclang = "C:\\Program Files\\LLVM\\bin" +# wasm-pack's wasm-opt (binaryen) can't execute on Nano Server (os error 3), so +# the module + wasm-agent builds pass --no-opt; they reference {{ vars.no_opt }}. +no_opt = "--no-opt" [env] # Run `shell = "bash …"` tasks on busybox ash rather than the Windows WSL From 47335d42816e6967833542e1a38733a4ef990808 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 13:49:53 +0800 Subject: [PATCH 56/60] more rules --- .dockerignore | 3 +- .github/workflows/{check.yml => check.yaml} | 0 .../{dependencies.yml => dependencies.yaml} | 2 +- .../{docker-linux.yml => docker-linux.yaml} | 2 +- ...docker-windows.yml => docker-windows.yaml} | 2 +- .github/workflows/{test.yml => test.yaml} | 2 +- .gitignore | 3 +- .mise/config.python.toml | 4 + .mise/config.toml | 106 ++++-------------- .mise/config.windows.toml | 2 + CLAUDE.md | 2 +- Cargo.toml | 2 +- Dockerfile.nanoserver | 16 +-- README.md | 4 +- ...l-bash.yml => gha-default-shell-bash.yaml} | 2 +- ...-step-shell.yml => gha-no-step-shell.yaml} | 2 +- ...ttributes.yml => no-allow-attributes.yaml} | 0 ...est-dir.yml => no-cargo-manifest-dir.yaml} | 0 ...-expect.yml => no-consecutive-expect.yaml} | 0 ...no-current-dir.yml => no-current-dir.yaml} | 0 .../rules/{no-doctest.yml => no-doctest.yaml} | 0 ...ate-self.yml => no-extern-crate-self.yaml} | 0 .../{no-inline-mod.yml => no-inline-mod.yaml} | 0 .../rules/{no-map-err.yml => no-map-err.yaml} | 0 ...ts.yml => no-mixed-doc-line-comments.yaml} | 0 .../{no-non-ascii.yml => no-non-ascii.yaml} | 0 ...eral.yml => no-relative-path-literal.yaml} | 0 ...-result-alias.yml => no-result-alias.yaml} | 0 ...ring-err.yml => no-result-string-err.yaml} | 0 ...hadow-result.yml => no-shadow-result.yaml} | 0 ...no-std-env-var.yml => no-std-env-var.yaml} | 0 ...f-mod-use.yml => prefer-self-mod-use.yaml} | 0 .../{use-mod-order.yml => use-mod-order.yaml} | 0 .../ast-grep/{sgconfig.yml => sgconfig.yaml} | 0 config/conftest/policy/cargo.rego | 17 +++ config/conftest/policy/gha.rego | 2 +- config/conftest/policy/pyproject.rego | 14 +++ config/lychee.toml | 20 ++-- config/ryl.yaml | 14 +++ .../{cargo-toml.yml => cargo-toml.yaml} | 0 .../{mise-config.yml => mise-config.yaml} | 0 ...ckslash.yml => no-trailing-backslash.yaml} | 0 config/semgrep/prefer-yaml-extension.yaml | 15 +++ ...er-yaml-toml.yml => prefer-yaml-toml.yaml} | 0 config/taplo.toml | 6 +- 45 files changed, 117 insertions(+), 125 deletions(-) rename .github/workflows/{check.yml => check.yaml} (100%) rename .github/workflows/{dependencies.yml => dependencies.yaml} (98%) rename .github/workflows/{docker-linux.yml => docker-linux.yaml} (97%) rename .github/workflows/{docker-windows.yml => docker-windows.yaml} (97%) rename .github/workflows/{test.yml => test.yaml} (98%) rename config/ast-grep/rules/{gha-default-shell-bash.yml => gha-default-shell-bash.yaml} (96%) rename config/ast-grep/rules/{gha-no-step-shell.yml => gha-no-step-shell.yaml} (93%) rename config/ast-grep/rules/{no-allow-attributes.yml => no-allow-attributes.yaml} (100%) rename config/ast-grep/rules/{no-cargo-manifest-dir.yml => no-cargo-manifest-dir.yaml} (100%) rename config/ast-grep/rules/{no-consecutive-expect.yml => no-consecutive-expect.yaml} (100%) rename config/ast-grep/rules/{no-current-dir.yml => no-current-dir.yaml} (100%) rename config/ast-grep/rules/{no-doctest.yml => no-doctest.yaml} (100%) rename config/ast-grep/rules/{no-extern-crate-self.yml => no-extern-crate-self.yaml} (100%) rename config/ast-grep/rules/{no-inline-mod.yml => no-inline-mod.yaml} (100%) rename config/ast-grep/rules/{no-map-err.yml => no-map-err.yaml} (100%) rename config/ast-grep/rules/{no-mixed-doc-line-comments.yml => no-mixed-doc-line-comments.yaml} (100%) rename config/ast-grep/rules/{no-non-ascii.yml => no-non-ascii.yaml} (100%) rename config/ast-grep/rules/{no-relative-path-literal.yml => no-relative-path-literal.yaml} (100%) rename config/ast-grep/rules/{no-result-alias.yml => no-result-alias.yaml} (100%) rename config/ast-grep/rules/{no-result-string-err.yml => no-result-string-err.yaml} (100%) rename config/ast-grep/rules/{no-shadow-result.yml => no-shadow-result.yaml} (100%) rename config/ast-grep/rules/{no-std-env-var.yml => no-std-env-var.yaml} (100%) rename config/ast-grep/rules/{prefer-self-mod-use.yml => prefer-self-mod-use.yaml} (100%) rename config/ast-grep/rules/{use-mod-order.yml => use-mod-order.yaml} (100%) rename config/ast-grep/{sgconfig.yml => sgconfig.yaml} (100%) create mode 100644 config/conftest/policy/pyproject.rego create mode 100644 config/ryl.yaml rename config/semgrep/{cargo-toml.yml => cargo-toml.yaml} (100%) rename config/semgrep/{mise-config.yml => mise-config.yaml} (100%) rename config/semgrep/{no-trailing-backslash.yml => no-trailing-backslash.yaml} (100%) create mode 100644 config/semgrep/prefer-yaml-extension.yaml rename config/semgrep/{prefer-yaml-toml.yml => prefer-yaml-toml.yaml} (100%) diff --git a/.dockerignore b/.dockerignore index 5d973ab..dd97eb3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,11 +26,12 @@ services/ws-server/static/models/ # `src/bin/` (e.g. utilities/int-gen/src/bin/). **/obj/ services/ws-modules/dotnet-data1/bin/ -# Editor dir (but keep the shared recommended-extensions list), ruff cache, and +# Editor dir (but keep the shared recommended-extensions list), tool caches, and # the ws-server's runtime file storage. .vscode/* !.vscode/extensions.json **/.ruff_cache/ +**/.lycheecache services/ws-server/storage/ **/.git/ **/Dockerfile* diff --git a/.github/workflows/check.yml b/.github/workflows/check.yaml similarity index 100% rename from .github/workflows/check.yml rename to .github/workflows/check.yaml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yaml similarity index 98% rename from .github/workflows/dependencies.yml rename to .github/workflows/dependencies.yaml index 5693f4e..7ddd82d 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yaml @@ -9,7 +9,7 @@ name: dependencies - "**/Cargo.toml" - config/deny.toml - config/osv-scanner.toml - - .github/workflows/dependencies.yml + - .github/workflows/dependencies.yaml workflow_dispatch: permissions: diff --git a/.github/workflows/docker-linux.yml b/.github/workflows/docker-linux.yaml similarity index 97% rename from .github/workflows/docker-linux.yml rename to .github/workflows/docker-linux.yaml index 7de7f53..690f7ae 100644 --- a/.github/workflows/docker-linux.yml +++ b/.github/workflows/docker-linux.yaml @@ -4,7 +4,7 @@ name: docker-linux "on": pull_request: paths: - - .github/workflows/docker-linux.yml + - .github/workflows/docker-linux.yaml - Dockerfile workflow_dispatch: diff --git a/.github/workflows/docker-windows.yml b/.github/workflows/docker-windows.yaml similarity index 97% rename from .github/workflows/docker-windows.yml rename to .github/workflows/docker-windows.yaml index 268d0a4..fbdb884 100644 --- a/.github/workflows/docker-windows.yml +++ b/.github/workflows/docker-windows.yaml @@ -4,7 +4,7 @@ name: docker-windows "on": pull_request: paths: - - .github/workflows/docker-windows.yml + - .github/workflows/docker-windows.yaml - Dockerfile.nanoserver workflow_dispatch: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yaml similarity index 98% rename from .github/workflows/test.yml rename to .github/workflows/test.yaml index 4aa6f32..0e6283c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yaml @@ -107,7 +107,7 @@ jobs: mise install env: GITHUB_TOKEN: ${{ github.token }} - # Match check.yml: bump mise's 30s HTTP timeout so GitHub-release + # Match check.yaml: bump mise's 30s HTTP timeout so GitHub-release # downloads don't fail the install on transient slowness. MISE_HTTP_TIMEOUT: "120" diff --git a/.gitignore b/.gitignore index cf8fc36..9284041 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,10 @@ pnpm-lock.yaml # `src/bin/` (e.g. utilities/int-gen/src/bin/). obj/ services/ws-modules/dotnet-data1/bin/ -# Editor dir (but keep the shared recommended-extensions list), ruff cache, and +# Editor dir (but keep the shared recommended-extensions list), tool caches, and # the ws-server's runtime file storage. .vscode/* !.vscode/extensions.json .ruff_cache/ +.lycheecache services/ws-server/storage/ diff --git a/.mise/config.python.toml b/.mise/config.python.toml index cbaf307..475e0c6 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -29,6 +29,10 @@ py_dirs = "services/ws-modules/ generated/python-ws/ generated/python-rest/" # that module's dir); used by both the bindings and componentize steps. wit_dir = "../../../generated/specs/wit" +[env] +# Keep ruff's cache under target/ (gitignored) instead of a repo-root .ruff_cache. +RUFF_CACHE_DIR = "{{ config_root }}/target/ruff-cache" + [tasks] ruff-check = "ruff check {{ vars.py_dirs }}" ruff-fmt = "ruff format {{ vars.py_dirs }}" diff --git a/.mise/config.toml b/.mise/config.toml index 46607fc..3d01d83 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -11,17 +11,11 @@ # Run a check across everything: mise run check-all (or install-all) # Make a selection sticky: export MISE_ENV=dart -# Require the mise release that introduced auto_env (platform config files): -# below this, .mise/config..toml wouldn't auto-load and per-OS env (linker -# flags, LIBCLANG_PATH, the Windows shell) would silently go missing. Fail loudly -# instead. (.miserc.toml enables auto_env; CI installs latest mise.) +# Require release that introduced auto_env enabled in .miserc.toml min_version = "2026.6.5" [settings] -# Keep installs explicit (`mise install`; CI/Docker install up front) so a task or -# `mise exec` never pulls the whole toolchain implicitly — e.g. `mise exec -- -# python …` in Dockerfile.nanoserver would otherwise install every tool first. -# The two knobs are separate: exec_auto_install covers `mise exec`, +# exec_auto_install covers `mise exec`, # task.run_auto_install covers `mise run`. exec_auto_install = false task.run_auto_install = false @@ -31,12 +25,8 @@ action-validator = "latest" "aqua:EmbarkStudios/cargo-deny" = "latest" ast-grep = "latest" cargo-binstall = "latest" -# dtolnay publishes no prebuilt binary for cargo-expand (crates.io only), so it's -# the one tool here still built from source. "aqua:rustwasm/wasm-pack" = "latest" "cargo:cargo-expand" = "latest" -# aqua ships a prebuilt taplo binary, so it installs on every OS (no `ring` C -# build, which is what previously kept it off the gnu Windows image). taplo = "latest" watchexec = "latest" # vfox-chromedriver's PostInstall hook fails on Windows ("syntax of the @@ -45,13 +35,7 @@ watchexec = "latest" # in the standard CI `test` flow, so skip it on Windows entirely. "chromedriver" = { version = "146", os = ["linux", "macos"] } cmake = "latest" -# openssl-sys reads OPENSSL_DIR (below); the conda build is no-admin + identical -# across machines. The per-OS toolchains (conda:clangxx on Linux, conda:lld on -# macOS, the msys2/llvm-mingw/busybox set on Windows) live in the matching -# config..toml. "conda:openssl" = "3" -# conftest runs the repo's OPA/Rego policies (config/conftest/policy) -- the -# cross-file config invariants taplo's per-file JSON schemas can't express. conftest = "latest" dprint = "latest" editorconfig-checker = "latest" @@ -59,13 +43,9 @@ editorconfig-checker = "latest" # below (see the `retry` var) so a transient HTTP 429 doesn't fail the build. "github:dbohdan/recur" = "latest" "github:grok-rs/waitup" = "latest" -# Must match the wasm-bindgen crate version in Cargo.lock so wasm-pack finds this -# on PATH instead of downloading its own wasm-bindgen-cli; bump both together. +"github:owenlamont/ryl" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.122" hadolint = "latest" -# Link checker for the `link-check` task. Installs on every OS; the Nano Docker -# image opts out via MISE_DISABLE_TOOLS (link-check isn't run in the Windows -# build/test flow), which keeps that image lean. lychee = "latest" mprocs = "latest" node = "22" @@ -96,16 +76,11 @@ wasm-tools = "latest" yq = "latest" [vars] -# conda:openssl install dir, reused by OPENSSL_DIR + rpath_flag. The per-OS -# toolchain vars -- conda_lld/lld_flag + mac libclang (macOS), the clangxx + -# bindgen set (Linux), winsh + win libclang + py313_win (Windows) -- live in the -# matching config..toml. conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" # mise-managed CPython 3.13 (Linux/macOS); the PYO3_PYTHON default below. Windows # overrides PYO3_PYTHON with its own py313_win in config.windows.toml. py313_unix = "{{ env.HOME }}/.local/share/mise/installs/python/3.13/bin/python3" -# RUSTFLAGS fragments shared by the per-OS CARGO_TARGET_* env in config..toml; -# kept as vars so those values stay single-line. +# RUSTFLAGS fragments shared by the CARGO_TARGET_* env in config..toml rpath_flag = "-C link-arg=-Wl,-rpath,{{ vars.conda_openssl }}/lib" # rpath to the mise CPython 3.13 lib dir so the et-ws-pyo3-runner binary finds # libpython at runtime (libpython3.13.so on Linux, libpython3.13.dylib on macOS @@ -139,18 +114,10 @@ retry = "recur --attempts 8 --delay 2s --backoff 2s --max-delay 2m --jitter 0,5s # no MISE_ENV=all). Hardcoded rather than shell-discovered so it works on Windows # too — keep in sync when adding/removing a config..toml. ALL_LANGS = "dart,dotnet,java,python,rust,zig" -# taplo's config now lives under config/ instead of a root .taplo.toml, which it -# no longer auto-discovers; TAPLO_CONFIG points every `taplo lint`/`format` at -# it. config_root keeps the path valid regardless of the invocation's cwd. TAPLO_CONFIG = "{{ config_root }}/config/taplo.toml" -# Likewise clippy.toml lives under config/; clippy reads its config from the dir -# named by CLIPPY_CONF_DIR rather than auto-discovering a root .clippy.toml. CLIPPY_CONF_DIR = "{{ config_root }}/config" # Use the conda:openssl install for Rust's OPENSSL_DIR so openssl-sys crate builds. OPENSSL_DIR = "{{ vars.conda_openssl }}" -# Per-OS RUSTFLAGS (CARGO_TARGET__*), LIBCLANG_PATH, BINDGEN args and the -# Windows MISE_BASH_PATH now live in the config..toml files (no more os() -# guards or triple-scoped-but-cross-OS entries). # pyo3-ffi (et-ws-pyo3-runner) links its embedded interpreter at build time from # PYO3_PYTHON, else the first python on PATH -- under mise that's the 32-bit @@ -159,14 +126,6 @@ OPENSSL_DIR = "{{ vars.conda_openssl }}" # builds the pyo3 runner without MISE_ENV=python. This is the Linux/macOS path; # config.windows.toml overrides it with py313_win. PYO3_PYTHON = "{{ vars.py313_unix }}" -# rclone config for the `fetch-*-rclone` model-download tasks. Those remotes -# (Hugging Face / GitHub raw) intermittently answer HTTP 429; the retry loop is -# the `recur` wrapper (the `retry` var) with exponential backoff + jitter, since -# rclone's own high-level retry is a fixed-interval sleep that can't back off. -# So disable rclone's high-level retries (`--retries 1`) to avoid compounding -# with recur — rclone's fast built-in low-level retries still apply. `RCLONE_*` -# env vars are rclone's alternative to CLI flags; `rclone.conf` only configures -# named remotes, not retry policy. RCLONE_CONFIG = "{{ config_root }}/config/rclone.conf" RCLONE_RETRIES = "1" @@ -196,16 +155,6 @@ depends = ["fmt:*"] description = "Run all formatters: fmt:rust + any loaded guest fmt:" [tasks."check:rust"] -# The dep-audit tools (osv-scanner, cargo-deny, cargo-unmaintained) run -# in the standalone `.github/workflows/dependencies.yml` workflow so they -# only fire when Cargo.lock / Cargo.toml / config/deny.toml change, not on every -# `mise run check`. `osv-scanner` and `cargo-deny` are kept in `[tools]` -# for ad-hoc local invocation (`mise run osv-scanner`, or -# `cargo deny check --config config/deny.toml` -- deny no longer auto-finds the -# config now that it lives under config/); `cargo-unmaintained` is installed only -# by the workflow via `taiki-e/install-action` since it isn't useful enough -# locally to justify the cargo-build cost on every `mise install`. -# # Rust + universal repo-wide checks. Lives in the always-loaded default config, # so `check`'s `check:*` glob always matches at least this one (Rust is in the # default config this pass; the guest `check:` tasks come from their @@ -222,6 +171,8 @@ depends = [ "editorconfig-check", "gen-specs-check", "hadolint-check", + "link-check", + "ryl-check", "semgrep-check", "conftest-check", "taplo-check", @@ -241,13 +192,13 @@ description = "Run all checks: check:rust + any loaded guest check:" [tasks.action-validator] description = "Validate GitHub Actions workflow YAML" -run = "action-validator .github/workflows/*.yml" +run = "action-validator .github/workflows/*.yaml" [tasks.ast-grep-check] # --no-ignore hidden so the gha-* YAML rules reach .github/workflows (ast-grep # skips dot-dirs by default); gitignored paths like target/ stay skipped. description = "Run ast-grep structural-search rules" -run = "ast-grep scan --error --no-ignore hidden -c config/ast-grep/sgconfig.yml" +run = "ast-grep scan --error --no-ignore hidden -c config/ast-grep/sgconfig.yaml" [tasks.semgrep-check] description = "Run Semgrep generic-mode rules (e.g. Cargo.toml style)" @@ -255,8 +206,6 @@ run = "semgrep scan --config config/semgrep --error --metrics=off ." [tasks.hadolint-check] description = "Lint the Dockerfiles with hadolint" -# Discover every tracked Dockerfile (`Dockerfile`, `Dockerfile.*`, at any depth) -# so new ones are linted automatically without editing this task. run = "git ls-files '*Dockerfile' '*Dockerfile.*' | xargs hadolint --config config/hadolint.yaml" [tasks.cargo-check] @@ -283,8 +232,6 @@ run = "cargo clippy --fix --allow-dirty --allow-staged --keep-going --workspace run = "taplo format" [tasks.conftest-check] -# Native `depends` glob: every conftest-check- task (toml, yaml, ...) -# joins automatically, so adding a new parser needs no edit here. depends = ["conftest-check-*"] description = "Run all conftest policy checks (one per parsed file format)" @@ -296,17 +243,14 @@ description = "Run conftest OPA/Rego policies over the TOML config + lock files" # .toml suffix. The TOML rule namespaces are listed explicitly, so the per-file # `gha` (YAML) namespace is never evaluated against this combined TOML input. run = """ -ns="--namespace cross --namespace cargo --namespace mise" -git ls-files '*Cargo.toml' '.mise/config*.toml' Cargo.lock | +ns="--namespace cross --namespace cargo --namespace mise --namespace pyproject" +git ls-files '*Cargo.toml' '.mise/config*.toml' '*pyproject.toml' Cargo.lock | xargs conftest test --combine --parser toml $ns -p config/conftest/policy """ shell = "bash -euo pipefail -c" [tasks.conftest-check-yaml] description = "Run conftest OPA/Rego policies over the GitHub Actions workflow YAML" -# Each workflow is checked on its own (no cross-file YAML rules) -- no --combine; -# conftest reads the directory directly. --namespace gha selects only the YAML -# policy. Replicates the gha-* ast-grep rules; running both is fine. run = "conftest test --parser yaml --namespace gha -p config/conftest/policy .github/workflows" [tasks.taplo-check] @@ -318,8 +262,6 @@ run = "conftest test --parser yaml --namespace gha -p config/conftest/policy .gi # conftest-check-toml (config/conftest/policy) intentionally duplicates these # lints; running both is fine. run = """ -# Baseline TOML validation across every .toml file the auto-config picks up. -# Runs on every OS (taplo is a prebuilt binary now). taplo lint # The --schema lints below pass file://$PWD URLs, which aren't well-formed on @@ -375,21 +317,25 @@ shell = "bash -euo pipefail -c" [tasks.typos] run = "typos --config config/typos.toml" -# Not in the `check` aggregate: lychee makes live network requests, so it would -# make CI flaky. Run on demand (or from a scheduled workflow). Checks every URL -# in the repo's Markdown + Rust sources; lychee extracts URLs from .rs comments -# and string literals too. config/lychee.toml excludes generated/ + target/. +# Part of the `check` aggregate. lychee makes live network requests, so +# config/lychee.toml is tuned (retries, 429-tolerance, disk cache) to keep it +# from being flaky. Checks every URL in the repo's Markdown + Rust sources; +# lychee extracts URLs from .rs comments and string literals too. [tasks.link-check] -description = "Check that URLs in .md and .rs files are reachable (network; not in check)" +description = "Check that URLs in .md and .rs files are reachable (network)" run = "lychee --config config/lychee.toml '**/*.md' '**/*.rs'" +[tasks.ryl-check] +description = "Lint YAML with ryl (a yamllint-compatible Rust linter)" +run = "ryl -c config/ryl.yaml .github config" + [tasks.setup-aube] # `aube` is an OPTIONAL faster npm backend, kept out of the mandatory `[tools]` # list because its install is flaky on some platforms and edge-toolkit falls # back to the classical `lib/node_modules` layout without it (see # `find_npm_modules_path_in`). # -# CI (check.yml / test.yml) runs this task in its own allowed-to-fail step, +# CI (check.yaml / test.yaml) runs this task in its own allowed-to-fail step, # placed BEFORE the main `mise install` so the `npm:onnxruntime-web` install # can pick up aube as its backend. # @@ -407,12 +353,6 @@ mise install aube@latest shell = "bash -euo pipefail -c" [tasks._setup_all] -# Private (leading `_`, hide = true): shared cross-platform preinstall, never run -# directly. Each platform's `preinstall` task (one per config..toml) depends -# on it, so each controls its own ordering. Enables experimental + -# cargo.binstall, then installs cargo-binstall (a prebuilt binary, safe to -# preinstall anywhere), node (mise uses it for the npm: tools) and conda:openssl -# (for openssl-sys) -- all needed on every platform. description = "Private shared cross-platform preinstall" hide = true run = """ @@ -639,11 +579,7 @@ RUSTUP_TOOLCHAIN = "nightly" [tasks.build-modules] # Native `depends` glob over `build-ws-*`: that matches `build-ws-wasm-agent` # (always loaded here, no `-module` suffix — so the glob never matches zero) -# plus every loaded `build-ws-*-module`. The module builds now live in MISE_ENV -# configs: the Rust ones in config.rust.toml, the guest ones (dart-comm1, -# dotnet-data1, java-data1, zig-data1, pydata1, pyface1, wasi-graphics-info) in -# theirs — each joins when its env is loaded. CI sets MISE_ENV to every language -# (or use `build-modules-all`). A nested `mise run` here broke tools on Windows. +# plus every loaded `build-ws-*-module`. depends = ["build-ws-*"] description = "Build all loaded WebAssembly modules" diff --git a/.mise/config.windows.toml b/.mise/config.windows.toml index 607a1fa..5dd411e 100644 --- a/.mise/config.windows.toml +++ b/.mise/config.windows.toml @@ -75,11 +75,13 @@ depends = ["_setup_all"] description = "Preinstall: toolchain + pipx + flip rustup to the gnullvm host" run = """ mise install conda:m2-git conda:m2-gnupg github:mstorsjo/llvm-mingw conda:m2-make rust + # pipx backs the pipx:* tools but isn't a mise tool on Windows. Install it with # whatever python is on PATH: the runner's system python on CI/workstations (its # Scripts dir, where pipx lands, is already on PATH); Dockerfile.nanoserver puts # mise's python on PATH before this so the command resolves on Nano too. python -m pip install pipx + # rustup installed components/targets against the pre-flip default host, so after # flipping to gnullvm reinstall the EXACT stable + nightly versions mise pinned as # gnullvm toolchains, each carrying the components + wasm targets config.toml's diff --git a/CLAUDE.md b/CLAUDE.md index cddc6d9..39c1aa1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -274,7 +274,7 @@ available linters: - **ast-grep** (`config/ast-grep/rules/`) — structural rules for code **and YAML** (e.g. GitHub Actions workflows). - **semgrep** (`config/semgrep/`) — incl. `languages: [generic]`, which works on - TOML/text (e.g. `mise-config.yml` lints `.mise/config*.toml`). + TOML/text (e.g. `mise-config.yaml` lints `.mise/config*.toml`). - **taplo** JSON-schemas (`config/taplo/`) — TOML structure, applied via `taplo lint --schema` in `taplo-check`. - plus hadolint, editorconfig-checker, typos, and action-validator for their diff --git a/Cargo.toml b/Cargo.toml index 7c56685..9fe0539 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -343,7 +343,7 @@ ignore = [ # via wgpu. Compile-time only. "paste", # Archived upstream at GnomedDev/proc-macro-error-2 (the maintained - # fork of the original proc-macro-error). Both now unmaintained. + # fork of the original proc-macro-error). Both unmaintained. # Pulled by getset via neli via local-ip-address. Compile-time only. "proc-macro-error-attr2", "proc-macro-error2", diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index 5cffcfa..dd4dfb0 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -108,7 +108,7 @@ ENV MISE_VERBOSE=1 ` RUST_BACKTRACE=full # mise.zip is staged in the build context by the windows job in -# .github/workflows/docker.yml (which pins the version) and copied in here. We +# .github/workflows/docker.yaml (which pins the version) and copied in here. We # don't curl a versioned URL inside the build: the classic Windows builder # doesn't substitute build-args into RUN, and mise's `mise-latest-windows-x64.zip` # prebuilt is stale (2026.3.0, which predates the nested `task.run_auto_install` @@ -178,10 +178,6 @@ RUN dir "%LOCALAPPDATA%\mise\installs\http-busybox\1.37.0" & ` ("%WINSH%" -euo pipefail -c "command -v mise && echo [smoke] OK || echo [smoke] NOMISE") & ` ver>nul -# CARGO_BUILD_TARGET (x86_64-pc-windows-gnullvm) + the gnullvm linker/dlltool now -# come from config.windows.toml (auto-loaded by auto_env), so they're not set -# here. - # Pin CARGO_HOME/RUSTUP_HOME to the dirs mise's rust tool uses (C:\.cargo / # C:\.rustup -- see its exec_env). preinstall sets the gnullvm default # toolchain in C:\.rustup, but when mise's cargo backend builds a `cargo:` tool it @@ -262,13 +258,3 @@ RUN mise exec -- cargo build --release -p et-ws-server && ` if exist target rmdir /s /q target EXPOSE 8080 8443 CMD ["et-ws-server"] - -# Known unknowns to settle via CI, roughly in the order the stages reach them. -# build-minimal / build: whether the rustup host flip sticks (mise may re-pin its -# own toolchain) and a gnu-host proc-macro/build-script build is clean; whether -# the conda backend (micromamba) and the github llvm-mingw asset match resolve -# cleanly on Nano Server; whether mise finds bash (m2-bash) for the `bash -euo -# pipefail` tasks. prefetch onward (not yet reached -- the frontier): whether -# `ort` (ONNX Runtime, msvc-only prebuilts) breaks the gnu link once a real cargo -# build/test links it; whether build-modules (wasm-pack + the guest toolchains) -# and a release et-ws-server build work at all under the gnu target on Nano Server. diff --git a/README.md b/README.md index 8f2fbc1..2f4d862 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ DRI device. The image skips the `o2`/`ws-server` README steps (runtime services) ### CI -The [`docker-linux`](.github/workflows/docker-linux.yml) and -[`docker-windows`](.github/workflows/docker-windows.yml) workflows rebuild these +The [`docker-linux`](.github/workflows/docker-linux.yaml) and +[`docker-windows`](.github/workflows/docker-windows.yaml) workflows rebuild these images when their respective `Dockerfile` is modified. [`Dockerfile.nanoserver`](Dockerfile.nanoserver) starts from **Nano Server** diff --git a/config/ast-grep/rules/gha-default-shell-bash.yml b/config/ast-grep/rules/gha-default-shell-bash.yaml similarity index 96% rename from config/ast-grep/rules/gha-default-shell-bash.yml rename to config/ast-grep/rules/gha-default-shell-bash.yaml index a92e8b6..10933b7 100644 --- a/config/ast-grep/rules/gha-default-shell-bash.yml +++ b/config/ast-grep/rules/gha-default-shell-bash.yaml @@ -11,7 +11,7 @@ message: | `shell:`. (Pairs with gha-no-step-shell, which forbids per-step `shell:` -- so this top-level default is the only place the shell is set.) files: - - .github/workflows/*.yml + - .github/workflows/*.yaml # Fire on any workflow whose document does NOT contain the exact # defaults -> run -> shell: bash nesting. The `has`/`pattern` matches that # specific three-level mapping (a stray `shell: bash` on a step does NOT diff --git a/config/ast-grep/rules/gha-no-step-shell.yml b/config/ast-grep/rules/gha-no-step-shell.yaml similarity index 93% rename from config/ast-grep/rules/gha-no-step-shell.yml rename to config/ast-grep/rules/gha-no-step-shell.yaml index 5350bb7..8bf8a2d 100644 --- a/config/ast-grep/rules/gha-no-step-shell.yml +++ b/config/ast-grep/rules/gha-no-step-shell.yaml @@ -7,7 +7,7 @@ message: | `sc`/`net`/`printf` instead of PowerShell cmdlets, even on Windows runners, which have Git Bash) rather than overriding the shell per step. files: - - .github/workflows/*.yml + - .github/workflows/*.yaml rule: pattern: "shell: $VALUE" inside: diff --git a/config/ast-grep/rules/no-allow-attributes.yml b/config/ast-grep/rules/no-allow-attributes.yaml similarity index 100% rename from config/ast-grep/rules/no-allow-attributes.yml rename to config/ast-grep/rules/no-allow-attributes.yaml diff --git a/config/ast-grep/rules/no-cargo-manifest-dir.yml b/config/ast-grep/rules/no-cargo-manifest-dir.yaml similarity index 100% rename from config/ast-grep/rules/no-cargo-manifest-dir.yml rename to config/ast-grep/rules/no-cargo-manifest-dir.yaml diff --git a/config/ast-grep/rules/no-consecutive-expect.yml b/config/ast-grep/rules/no-consecutive-expect.yaml similarity index 100% rename from config/ast-grep/rules/no-consecutive-expect.yml rename to config/ast-grep/rules/no-consecutive-expect.yaml diff --git a/config/ast-grep/rules/no-current-dir.yml b/config/ast-grep/rules/no-current-dir.yaml similarity index 100% rename from config/ast-grep/rules/no-current-dir.yml rename to config/ast-grep/rules/no-current-dir.yaml diff --git a/config/ast-grep/rules/no-doctest.yml b/config/ast-grep/rules/no-doctest.yaml similarity index 100% rename from config/ast-grep/rules/no-doctest.yml rename to config/ast-grep/rules/no-doctest.yaml diff --git a/config/ast-grep/rules/no-extern-crate-self.yml b/config/ast-grep/rules/no-extern-crate-self.yaml similarity index 100% rename from config/ast-grep/rules/no-extern-crate-self.yml rename to config/ast-grep/rules/no-extern-crate-self.yaml diff --git a/config/ast-grep/rules/no-inline-mod.yml b/config/ast-grep/rules/no-inline-mod.yaml similarity index 100% rename from config/ast-grep/rules/no-inline-mod.yml rename to config/ast-grep/rules/no-inline-mod.yaml diff --git a/config/ast-grep/rules/no-map-err.yml b/config/ast-grep/rules/no-map-err.yaml similarity index 100% rename from config/ast-grep/rules/no-map-err.yml rename to config/ast-grep/rules/no-map-err.yaml diff --git a/config/ast-grep/rules/no-mixed-doc-line-comments.yml b/config/ast-grep/rules/no-mixed-doc-line-comments.yaml similarity index 100% rename from config/ast-grep/rules/no-mixed-doc-line-comments.yml rename to config/ast-grep/rules/no-mixed-doc-line-comments.yaml diff --git a/config/ast-grep/rules/no-non-ascii.yml b/config/ast-grep/rules/no-non-ascii.yaml similarity index 100% rename from config/ast-grep/rules/no-non-ascii.yml rename to config/ast-grep/rules/no-non-ascii.yaml diff --git a/config/ast-grep/rules/no-relative-path-literal.yml b/config/ast-grep/rules/no-relative-path-literal.yaml similarity index 100% rename from config/ast-grep/rules/no-relative-path-literal.yml rename to config/ast-grep/rules/no-relative-path-literal.yaml diff --git a/config/ast-grep/rules/no-result-alias.yml b/config/ast-grep/rules/no-result-alias.yaml similarity index 100% rename from config/ast-grep/rules/no-result-alias.yml rename to config/ast-grep/rules/no-result-alias.yaml diff --git a/config/ast-grep/rules/no-result-string-err.yml b/config/ast-grep/rules/no-result-string-err.yaml similarity index 100% rename from config/ast-grep/rules/no-result-string-err.yml rename to config/ast-grep/rules/no-result-string-err.yaml diff --git a/config/ast-grep/rules/no-shadow-result.yml b/config/ast-grep/rules/no-shadow-result.yaml similarity index 100% rename from config/ast-grep/rules/no-shadow-result.yml rename to config/ast-grep/rules/no-shadow-result.yaml diff --git a/config/ast-grep/rules/no-std-env-var.yml b/config/ast-grep/rules/no-std-env-var.yaml similarity index 100% rename from config/ast-grep/rules/no-std-env-var.yml rename to config/ast-grep/rules/no-std-env-var.yaml diff --git a/config/ast-grep/rules/prefer-self-mod-use.yml b/config/ast-grep/rules/prefer-self-mod-use.yaml similarity index 100% rename from config/ast-grep/rules/prefer-self-mod-use.yml rename to config/ast-grep/rules/prefer-self-mod-use.yaml diff --git a/config/ast-grep/rules/use-mod-order.yml b/config/ast-grep/rules/use-mod-order.yaml similarity index 100% rename from config/ast-grep/rules/use-mod-order.yml rename to config/ast-grep/rules/use-mod-order.yaml diff --git a/config/ast-grep/sgconfig.yml b/config/ast-grep/sgconfig.yaml similarity index 100% rename from config/ast-grep/sgconfig.yml rename to config/ast-grep/sgconfig.yaml diff --git a/config/conftest/policy/cargo.rego b/config/conftest/policy/cargo.rego index 1132af2..3073b91 100644 --- a/config/conftest/policy/cargo.rego +++ b/config/conftest/policy/cargo.rego @@ -173,3 +173,20 @@ deny contains msg if { spec.features == [] msg := sprintf("%s: dependency %q has an empty features = []; remove it", [path, name]) } + +# A feature must not share its name with a dependency: it shadows the implicit +# feature an optional dep creates and is confusing. generated/rust-rest is exempt +# -- its generator emits a `tracing` feature beside a (non-optional) `tracing` dep. +is_dep_name(file, name) if { + some table in {"dependencies", "dev-dependencies", "build-dependencies"} + file.contents[table][name] +} + +deny contains msg if { + some file in input + is_member(file) + file.path != "generated/rust-rest/Cargo.toml" + some feat, _ in file.contents.features + is_dep_name(file, feat) + msg := sprintf("%s: feature %q shares its name with a dependency; rename it", [file.path, feat]) +} diff --git a/config/conftest/policy/gha.rego b/config/conftest/policy/gha.rego index db9ddc8..df06cb4 100644 --- a/config/conftest/policy/gha.rego +++ b/config/conftest/policy/gha.rego @@ -1,4 +1,4 @@ -# GitHub Actions workflow policy, evaluated per file: conftest reads each .yml +# GitHub Actions workflow policy, evaluated per file: conftest reads each .yaml # independently (no --combine, since there are no cross-file YAML rules). # Replicates the gha-* ast-grep rules; running both is fine. Selected with # `--namespace gha`, so it only runs against workflow YAML, never the TOML inputs. diff --git a/config/conftest/policy/pyproject.rego b/config/conftest/policy/pyproject.rego new file mode 100644 index 0000000..38e7507 --- /dev/null +++ b/config/conftest/policy/pyproject.rego @@ -0,0 +1,14 @@ +# pyproject.toml policy, evaluated over conftest's --combine input ({path, +# contents}); selected with --namespace pyproject. +package pyproject + +# A `path = ".."` uv source points at the parent directory, which isn't a package +# -- almost always a mistake. Sibling packages are referenced by their specific +# relative path (e.g. ../../../generated/python-rest), not the bare parent. +deny contains msg if { + some file in input + endswith(file.path, "pyproject.toml") + some name, src in file.contents.tool.uv.sources + src.path == ".." + msg := sprintf("%s: [tool.uv.sources] %q uses path = \"..\"; point at the package path", [file.path, name]) +} diff --git a/config/lychee.toml b/config/lychee.toml index 4c81999..e3bd895 100644 --- a/config/lychee.toml +++ b/config/lychee.toml @@ -1,7 +1,7 @@ -# Config for the `link-check` task (mise run link-check). lychee makes live -# network requests, so it is deliberately kept out of the blocking `check` -# aggregate; run it on demand. Tuned to tolerate transient failures and to skip -# reserved hosts and non-prose trees. +# Config for the `link-check` task, part of the `check` aggregate. lychee makes +# live network requests, so this is tuned to tolerate transient failures +# (retries, 429-tolerance, disk cache) and to skip reserved hosts + non-prose +# trees. # Retry transient failures; cap each request so a hung host can't stall the run. max_retries = 3 @@ -16,10 +16,16 @@ max_cache_age = "2d" # we're just rate-limited. accept = ["200..=299", "429"] -# Skip loopback/reserved hosts (localhost, 127.0.0.1) +# Skip loopback (localhost, 127.0.0.1) and private/reserved IPs (e.g. the +# 10.0.0.1:9000 test fixture) -- not real external links. exclude_loopback = true +exclude_private = true exclude_path = ["data", "generated", "target"] -# URL patterns (regex) to skip -- add flaky or auth-gated hosts here. -exclude = [] +# URL patterns (regex) to skip: doc/test placeholder hosts (e.g. http://host:8080/) +# and `{...}` format-string templates (.../{repo}/{git_ref}/...), not real links. +exclude = [ + '^https?://host[:/]', + '\{|%7B', +] diff --git a/config/ryl.yaml b/config/ryl.yaml new file mode 100644 index 0000000..90048fa --- /dev/null +++ b/config/ryl.yaml @@ -0,0 +1,14 @@ +# Starts from yamllint's standard ruleset, then aligns the cosmetic rules with the +# repo's other tools so they never disagree. +extends: default + +rules: + # dprint (the YAML formatter) writes one space before an inline `#`; yamllint + # defaults to two. Defer to dprint so the two can't fight. + comments: + min-spaces-from-content: 1 + # Match the repo's 120-char editorconfig limit (yamllint defaults to 80). + line-length: + max: 120 + # The repo's workflows and configs don't use a `---` document-start marker. + document-start: disable diff --git a/config/semgrep/cargo-toml.yml b/config/semgrep/cargo-toml.yaml similarity index 100% rename from config/semgrep/cargo-toml.yml rename to config/semgrep/cargo-toml.yaml diff --git a/config/semgrep/mise-config.yml b/config/semgrep/mise-config.yaml similarity index 100% rename from config/semgrep/mise-config.yml rename to config/semgrep/mise-config.yaml diff --git a/config/semgrep/no-trailing-backslash.yml b/config/semgrep/no-trailing-backslash.yaml similarity index 100% rename from config/semgrep/no-trailing-backslash.yml rename to config/semgrep/no-trailing-backslash.yaml diff --git a/config/semgrep/prefer-yaml-extension.yaml b/config/semgrep/prefer-yaml-extension.yaml new file mode 100644 index 0000000..733aa24 --- /dev/null +++ b/config/semgrep/prefer-yaml-extension.yaml @@ -0,0 +1,15 @@ +rules: + - id: prefer-yaml-extension + languages: [generic] + paths: + include: + - "*.yml" + exclude: + # External upstream clones + generated trees are not our prose. + - "/data" + - "/generated" + pattern-regex: \A + message: >- + Prefer the .yaml extension over .yml. GitHub Actions, ast-grep, semgrep and + uv all read .yaml -- rename the file. + severity: ERROR diff --git a/config/semgrep/prefer-yaml-toml.yml b/config/semgrep/prefer-yaml-toml.yaml similarity index 100% rename from config/semgrep/prefer-yaml-toml.yml rename to config/semgrep/prefer-yaml-toml.yaml diff --git a/config/taplo.toml b/config/taplo.toml index 9b6b171..4ba02e3 100644 --- a/config/taplo.toml +++ b/config/taplo.toml @@ -53,11 +53,7 @@ schema = { path = "config/taplo/require-workspace-deps.schema.json" } # Banned dep names across every dep table — `[dependencies]`, # `[dev-dependencies]`, `[build-dependencies]`, `[workspace.dependencies]`, -# and the `[target.*]` variants. Currently `anyhow` (use a `thiserror` -# enum) and `ureq` (use `reqwest::blocking` -- keeps us on one HTTPS -# stack). Replaces the previous per-crate `no-anyhow` ast-grep rule -# which caught uses but not declarations. Add new bans by extending the -# `depTable.properties` list in the schema. +# and the `[target.*]` variants. [[rule]] include = ["**/Cargo.toml"] schema = { path = "config/taplo/no-banned-deps.schema.json" } From b79d5c71ae35038dc8a540e40eebfca652ae7340 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 14:33:14 +0800 Subject: [PATCH 57/60] fix et-cli race --- .mise/config.python.toml | 5 +++-- .mise/config.rust.toml | 4 ++++ .mise/config.toml | 14 +++++++++++--- config/conftest/policy/pyproject.rego | 17 +++++++++++++++++ config/deny.toml | 5 ++--- config/semgrep/prefer-yaml-extension.yaml | 4 +--- ...=> mise-cargo-backend-allowlist.schema.json} | 6 +++--- config/taplo/mise-no-ubi-backend.schema.json | 12 ++++++++++++ generated/python-rest/pyproject.toml | 2 +- generated/python-ws/pyproject.toml | 2 +- services/ws-modules/pydata1/pyproject.toml | 2 +- services/ws-modules/pyface1/pyproject.toml | 2 +- .../wasi-graphics-info/pyproject.toml | 2 +- utilities/cli/src/error.rs | 2 +- utilities/cli/src/lib.rs | 2 +- 15 files changed, 60 insertions(+), 21 deletions(-) rename config/taplo/{mise-no-source-backends.schema.json => mise-cargo-backend-allowlist.schema.json} (55%) create mode 100644 config/taplo/mise-no-ubi-backend.schema.json diff --git a/.mise/config.python.toml b/.mise/config.python.toml index 475e0c6..a52eda3 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -68,7 +68,7 @@ UV_PYTHON = """\ {% else %}{{ exec(command='mise which python3.13') }}{% endif %}""" [tasks.build-et-ws-wheel] -depends = ["gen:python-ws"] +depends = ["build-et-cli", "gen:python-ws"] description = "Build the generated et-ws Python package as a wheel + ws-module pkg/" dir = "generated/python-ws" # Wheel goes straight into pkg/ alongside the committed et_ws.js shim so the @@ -82,7 +82,7 @@ cargo run -p et-cli -- module-package-json shell = "bash -euo pipefail -c" [tasks.build-et-rest-client-wheel] -depends = ["gen:python-rest"] +depends = ["build-et-cli", "gen:python-rest"] description = "Build the generated et-rest-client Python package as a wheel + ws-module pkg/" dir = "generated/python-rest" run = """ @@ -123,6 +123,7 @@ cargo run -p et-cli -- module-package-json shell = "bash -euo pipefail -c" [tasks.build-ws-wasi-graphics-info-module] +depends = ["build-et-cli"] description = "Build the WASI graphics-info Python module as a WASI Preview 2 component" dir = "services/ws-modules/wasi-graphics-info" # WIT lives once at generated/specs/wit/ — the runner's `runner` world is diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 07ba43e..b966aff 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -17,6 +17,7 @@ dir = "services/ws-modules/data1" run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-har1-module] +depends = ["build-et-cli"] description = "Build the har1 workflow WASM module" dir = "services/ws-modules/har1" # Chain with `&&` on one line rather than a multi-line `bash` block: wasm-pack @@ -26,6 +27,7 @@ dir = "services/ws-modules/har1" run = "wasm-pack build . --target web {{ vars.no_opt }} && cargo run -p et-cli -- module-package-json" [tasks.build-ws-face-detection-module] +depends = ["build-et-cli"] description = "Build the face detection workflow WASM module" dir = "services/ws-modules/face-detection" # See build-ws-har1-module: `&&` on one line under the default shell so @@ -78,6 +80,7 @@ dir = "services/ws-modules/nfc" run = "wasm-pack build . --target web {{ vars.no_opt }}" [tasks.build-ws-wasi-data1-module] +depends = ["build-et-cli"] description = "Build the Rust WASI data1 module as a WASI Preview 2 component" # The crate is a regular workspace member but its lib body is gated on # `#![cfg(target_os = "wasi")]`, so cargo check from the host target gets @@ -93,6 +96,7 @@ cargo run -p et-cli -- module-package-json --module-dir services/ws-modules/wasi shell = "bash -euo pipefail -c" [tasks.build-ws-wasi-comm1-module] +depends = ["build-et-cli"] description = "Build the Rust WASI comm1 module as a WASI Preview 2 component" run = """ cargo build --release -p et-ws-wasi-comm1 --target wasm32-wasip2 diff --git a/.mise/config.toml b/.mise/config.toml index 3d01d83..6589d1a 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -308,9 +308,10 @@ grep -vxF './generated/rust-rest/Cargo.toml' "$members" | # string, one command per line). Applies to every .mise/config*.toml. taplo lint --schema "file://$PWD/config/taplo/mise-run-not-array.schema.json" .mise/config*.toml -# Tools must use a prebuilt backend; `cargo:`/`ubi:` build from source. -# Allowlisted exceptions: cargo:cargo-expand + cargo:dart-typegen. -taplo lint --schema "file://$PWD/config/taplo/mise-no-source-backends.schema.json" .mise/config*.toml +# ubi: is deprecated (use http:); cargo: builds from source (slow), allowed only +# for cargo-expand + dart-typegen. Two separate rules, two reasons. +taplo lint --schema "file://$PWD/config/taplo/mise-no-ubi-backend.schema.json" .mise/config*.toml +taplo lint --schema "file://$PWD/config/taplo/mise-cargo-backend-allowlist.schema.json" .mise/config*.toml """ shell = "bash -euo pipefail -c" @@ -576,6 +577,13 @@ run = "wasm-pack build . --target web {{ vars.no_opt }} -- -Z build-std=std,pani CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS = "{{ vars.wasm_rustflags }}" RUSTUP_TOOLCHAIN = "nightly" +[tasks.build-et-cli] +# Pre-build et-cli once. The parallel build-ws-* module tasks each call +# `cargo run -p et-cli -- module-package-json`; without this they race rebuilding +# et-cli, and one execs the binary mid-replace (ENOENT -- surfaced on macOS). +description = "Build et-cli (prereq for the module-package-json steps)" +run = "cargo build -p et-cli" + [tasks.build-modules] # Native `depends` glob over `build-ws-*`: that matches `build-ws-wasm-agent` # (always loaded here, no `-module` suffix — so the glob never matches zero) diff --git a/config/conftest/policy/pyproject.rego b/config/conftest/policy/pyproject.rego index 38e7507..17f6f58 100644 --- a/config/conftest/policy/pyproject.rego +++ b/config/conftest/policy/pyproject.rego @@ -12,3 +12,20 @@ deny contains msg if { src.path == ".." msg := sprintf("%s: [tool.uv.sources] %q uses path = \"..\"; point at the package path", [file.path, name]) } + +# uv_build must be pinned to exactly the mise uv version, so `uv build` uses the +# matching backend (no out-of-range warning). Bump both together when upgrading uv. +uv_version := v if { + some file in input + endswith(file.path, ".mise/config.toml") + v := file.contents.tools.uv +} + +deny contains msg if { + some file in input + endswith(file.path, "pyproject.toml") + some req in file.contents["build-system"].requires + startswith(req, "uv_build") + req != sprintf("uv_build==%s", [uv_version]) + msg := sprintf("%s: pin uv_build==%s to match mise's uv (found %q)", [file.path, uv_version, req]) +} diff --git a/config/deny.toml b/config/deny.toml index 3f3628c..1f31006 100644 --- a/config/deny.toml +++ b/config/deny.toml @@ -1,6 +1,5 @@ -# `cargo deny check` config. Run locally via `cargo deny check`, in CI via -# `.github/workflows/cargo-deny.yml`. Every option below is intentional; -# cargo-deny's `init` template (with all defaults explicit) is not used. +# cargo-deny config, applied via `mise run cargo-deny-check` -- locally and in +# .github/workflows/dependencies.yaml. [advisories] version = 2 diff --git a/config/semgrep/prefer-yaml-extension.yaml b/config/semgrep/prefer-yaml-extension.yaml index 733aa24..752d6af 100644 --- a/config/semgrep/prefer-yaml-extension.yaml +++ b/config/semgrep/prefer-yaml-extension.yaml @@ -9,7 +9,5 @@ rules: - "/data" - "/generated" pattern-regex: \A - message: >- - Prefer the .yaml extension over .yml. GitHub Actions, ast-grep, semgrep and - uv all read .yaml -- rename the file. + message: "Use the .yaml extension, not .yml -- rename the file." severity: ERROR diff --git a/config/taplo/mise-no-source-backends.schema.json b/config/taplo/mise-cargo-backend-allowlist.schema.json similarity index 55% rename from config/taplo/mise-no-source-backends.schema.json rename to config/taplo/mise-cargo-backend-allowlist.schema.json index 10e96ee..18340f8 100644 --- a/config/taplo/mise-no-source-backends.schema.json +++ b/config/taplo/mise-cargo-backend-allowlist.schema.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "mise [tools] -- no source-built backends", - "description": "`cargo:`/`ubi:` build from source; prefer a prebuilt backend (aqua/registry/github/http).", + "title": "mise [tools] -- cargo backend allowlist", + "description": "`cargo:` builds from source (slow installs); allow only the two with no prebuilt binary.", "type": "object", "properties": { "tools": { "type": "object", "propertyNames": { "anyOf": [ - { "not": { "pattern": "^(cargo|ubi):" } }, + { "not": { "pattern": "^cargo:" } }, { "enum": ["cargo:cargo-expand", "cargo:dart-typegen"] } ] } diff --git a/config/taplo/mise-no-ubi-backend.schema.json b/config/taplo/mise-no-ubi-backend.schema.json new file mode 100644 index 0000000..6b0fe8f --- /dev/null +++ b/config/taplo/mise-no-ubi-backend.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "mise [tools] -- no ubi backend", + "description": "The `ubi:` backend is deprecated; use `http:` instead.", + "type": "object", + "properties": { + "tools": { + "type": "object", + "propertyNames": { "not": { "pattern": "^ubi:" } } + } + } +} diff --git a/generated/python-rest/pyproject.toml b/generated/python-rest/pyproject.toml index 9699f50..07d7199 100644 --- a/generated/python-rest/pyproject.toml +++ b/generated/python-rest/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "et_rest_client" diff --git a/generated/python-ws/pyproject.toml b/generated/python-ws/pyproject.toml index fe8b7d9..1837cb2 100644 --- a/generated/python-ws/pyproject.toml +++ b/generated/python-ws/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "et_ws" diff --git a/services/ws-modules/pydata1/pyproject.toml b/services/ws-modules/pydata1/pyproject.toml index 90063bd..b1ee14b 100644 --- a/services/ws-modules/pydata1/pyproject.toml +++ b/services/ws-modules/pydata1/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "pydata1" diff --git a/services/ws-modules/pyface1/pyproject.toml b/services/ws-modules/pyface1/pyproject.toml index 91a47e0..e858cda 100644 --- a/services/ws-modules/pyface1/pyproject.toml +++ b/services/ws-modules/pyface1/pyproject.toml @@ -11,7 +11,7 @@ dev = ["pytest"] [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "pyface1" diff --git a/services/ws-modules/wasi-graphics-info/pyproject.toml b/services/ws-modules/wasi-graphics-info/pyproject.toml index 18e7fe6..49bbfa2 100644 --- a/services/ws-modules/wasi-graphics-info/pyproject.toml +++ b/services/ws-modules/wasi-graphics-info/pyproject.toml @@ -11,7 +11,7 @@ repository = "https://github.com/edge-toolkit/core" [build-system] build-backend = "uv_build" -requires = ["uv_build>=0.10.2,<0.11.0"] +requires = ["uv_build==0.11.8"] [tool.uv.build-backend] module-name = "wasi_graphics_info" diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index 979150d..683a3dc 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -75,7 +75,7 @@ pub enum CliError { #[error("Verification input file {0:?} has no file stem")] MissingFileStem(PathBuf), - #[error("Verification root {0:?} does not contain any scenario files under */input/*.yaml or */input/*.yml")] + #[error("Verification root {0:?} does not contain any scenario files under */input/*.yaml")] NoScenarios(PathBuf), #[error("Unsupported deployment_type {0:?}. Supported values are currently: mise, docker-compose")] diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 08a1e65..68b3220 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -270,7 +270,7 @@ fn discover_verification_scenarios(verification_root: &Path) -> Result Date: Sun, 14 Jun 2026 15:26:44 +0800 Subject: [PATCH 58/60] tidy --- .github/workflows/check.yaml | 1 + .github/workflows/dependencies.yaml | 2 ++ .github/workflows/docker-linux.yaml | 2 ++ .github/workflows/docker-windows.yaml | 4 ++- .github/workflows/test.yaml | 5 ++++ .mise/config.toml | 14 +++++++++ CLAUDE.md | 39 ++++++++++++++++++++++-- Dockerfile.nanoserver | 14 +++++---- README.md | 13 ++++++++ config/conftest/policy/cargo.rego | 10 +++++++ config/conftest/policy/mise.rego | 20 +++++++++++++ config/ls-lint.yaml | 43 +++++++++++++++++++++++++++ config/zizmor.yaml | 16 ++++++++++ 13 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 config/ls-lint.yaml create mode 100644 config/zizmor.yaml diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index f28ec90..c471796 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -27,6 +27,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false - name: Install mise uses: taiki-e/install-action@v2 diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml index 7ddd82d..ed2baa8 100644 --- a/.github/workflows/dependencies.yaml +++ b/.github/workflows/dependencies.yaml @@ -30,6 +30,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false - name: Install tools uses: taiki-e/install-action@v2 diff --git a/.github/workflows/docker-linux.yaml b/.github/workflows/docker-linux.yaml index 690f7ae..416e4fe 100644 --- a/.github/workflows/docker-linux.yaml +++ b/.github/workflows/docker-linux.yaml @@ -26,6 +26,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false # The image is huge (every language toolchain + prefetched models + a full # debug build, incl. aws-lc-sys's large C objects), and its peak `target/` diff --git a/.github/workflows/docker-windows.yaml b/.github/workflows/docker-windows.yaml index fbdb884..c4bea98 100644 --- a/.github/workflows/docker-windows.yaml +++ b/.github/workflows/docker-windows.yaml @@ -22,7 +22,7 @@ defaults: jobs: build: runs-on: windows-2022 - timeout-minutes: 120 + timeout-minutes: 80 env: # The classic Windows builder can't substitute build-args into the Dockerfile's RUN, # and mise's prebuilt "latest" zip is stale (2026.3.0, too old for the config). @@ -31,6 +31,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false # Hosted Windows runners don't reliably leave the Docker daemon running, so # the build can fail connecting to the docker_engine pipe. Start it (no-op diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0e6283c..5d03a72 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,6 +17,9 @@ name: test - macos-26-intel - windows-latest +permissions: + contents: read + concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} @@ -66,6 +69,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: false # Reclaim the large preinstalled toolchains we never use (~30 GB). # `|| true` so a missing dir (e.g. on the arm image) is not fatal. diff --git a/.mise/config.toml b/.mise/config.toml index 6589d1a..16ccc27 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -46,6 +46,7 @@ editorconfig-checker = "latest" "github:owenlamont/ryl" = "latest" "github:wasm-bindgen/wasm-bindgen" = "0.2.122" hadolint = "latest" +ls-lint = "latest" lychee = "latest" mprocs = "latest" node = "22" @@ -74,6 +75,7 @@ typos = "latest" uv = "0.11.8" wasm-tools = "latest" yq = "latest" +zizmor = "latest" [vars] conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" @@ -172,12 +174,14 @@ depends = [ "gen-specs-check", "hadolint-check", "link-check", + "ls-lint-check", "ryl-check", "semgrep-check", "conftest-check", "taplo-check", "typos", "verification-check", + "zizmor-check", ] description = "Run the Rust + universal repo-wide checks" @@ -194,6 +198,16 @@ description = "Run all checks: check:rust + any loaded guest check:" description = "Validate GitHub Actions workflow YAML" run = "action-validator .github/workflows/*.yaml" +[tasks.zizmor-check] +description = "Audit GitHub Actions workflows for security issues (zizmor)" +# --offline so it needs no GH token and stays deterministic; config/zizmor.yaml +# turns off the hash-pin policy (we pin actions by tag, not commit). +run = "zizmor --offline --no-progress -c config/zizmor.yaml .github/workflows" + +[tasks.ls-lint-check] +description = "Lint file and directory naming conventions (ls-lint)" +run = "ls-lint --config config/ls-lint.yaml" + [tasks.ast-grep-check] # --no-ignore hidden so the gha-* YAML rules reach .github/workflows (ast-grep # skips dot-dirs by default); gitignored paths like target/ stay skipped. diff --git a/CLAUDE.md b/CLAUDE.md index 39c1aa1..fbf0b3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,38 @@ use the `*-all` variants (`check-all`, `test-all`, `build-modules-all`, (e.g., `mise run build-ws-face-detection-module` for the Rust modules, or `MISE_ENV=zig mise run build-ws-zig-data1-module`). +## Formatters & checks by file type + +The `mise run ` formatters and checks per file type (`fmt` / `check` run +them all; guest rows need their `MISE_ENV` loaded). + +| File type | Formatter task(s) | +| --------- | ------------------------------- | +| `*.rs` | `cargo-fmt`, `cargo-clippy-fix` | +| `*.toml` | `taplo-fmt` | +| `*.py` | `ruff-fmt` | +| `*.dart` | `fmt:dart` | +| `*.zig` | `fmt:zig` | +| `*.cs` | `fmt:dotnet` | + +| File type | Check task(s) | +| --------- | ---------------------------------------------------------------------------------------- | +| `*.rs` | `cargo-check`, `cargo-clippy`, `cargo-fmt-check`, `cargo-doc-check`, `ast-grep-check` | +| `*.toml` | `taplo-check`, `conftest-check-toml`, `semgrep-check` | +| `*.yaml` | `ast-grep-check`, `conftest-check-yaml`, `ryl-check`, `action-validator`, `zizmor-check` | +| `*.json` | `semgrep-check` | +| `*.py` | `check:python` | +| `*.dart` | `check:dart` | +| `*.zig` | `check:zig` | +| `*.cs` | `check:dotnet` | +| `*.java` | `check:java` | + +`dprint-fmt` / `dprint-check` cover `*.md`, `*.yaml`, `*.json`/`*.jsonc`, +`*.ts`/`*.js`, `*.css`, `*.html`, `*.java`, and `Dockerfile*`; `hadolint-check` +also lints Dockerfiles, and `link-check` scans `*.md` + `*.rs`. Every file is +covered by `editorconfig-check` and `typos`, file and directory names by +`ls-lint-check`, and `*.yml` is rejected by `semgrep-check` (use `*.yaml`). + ## Architecture This is a WebSocket-based edge computing framework. @@ -277,8 +309,11 @@ available linters: TOML/text (e.g. `mise-config.yaml` lints `.mise/config*.toml`). - **taplo** JSON-schemas (`config/taplo/`) — TOML structure, applied via `taplo lint --schema` in `taplo-check`. -- plus hadolint, editorconfig-checker, typos, and action-validator for their - domains. +- **conftest** (`config/conftest/policy/`) — Rego policies over the combined + TOML/YAML config set, for cross-file checks the schema linters can't express. +- plus hadolint, ls-lint (file/dir naming), zizmor (Actions security), ryl + (YAML), lychee (links), editorconfig-checker, typos, and action-validator for + their domains. ast-grep has no TOML grammar, so it **cannot** lint TOML — use a taplo schema or a semgrep `generic` rule there. If none of the above can express a check, diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index dd4dfb0..b9df316 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -100,12 +100,14 @@ ENV WASM_PACK_CACHE=C:\Temp\wasm-pack # mean no source builds); the gnu Docker path does. One job at a time avoids it. ENV MISE_JOBS=1 -# Verbose mise + full Rust backtraces while the Windows build is being brought -# up: surface tool/task resolution, the exact bash mise launches, and any panic -# in the CI log. Noisy but invaluable while diagnosing the bare-Nano-Server path. -ENV MISE_VERBOSE=1 ` - MISE_LOG_LEVEL=debug ` - RUST_BACKTRACE=full +# Full Rust backtraces so any panic surfaces in the CI log. +ENV RUST_BACKTRACE=full + +# Verbose mise is disabled -- it floods the log with tool/task resolution and the +# exact bash it launches. Re-enable it (noisy but invaluable) when diagnosing a +# Windows build failure on the bare-Nano-Server path: +# ENV MISE_VERBOSE=1 ` +# MISE_LOG_LEVEL=debug # mise.zip is staged in the build context by the windows job in # .github/workflows/docker.yaml (which pins the version) and copied in here. We diff --git a/README.md b/README.md index 2f4d862..3cb0df5 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,19 @@ mise install-all The `preinstall` task will advise if there are any required dependencies are are missing, such as Xcode Command Line Tools on MacOS. +### Install failures + +`mise install` runs tool installs in parallel. If they fail intermittently — a +download race, or a `cargo:` source build colliding with another — serialize +them with `MISE_JOBS=1`: + +```bash +MISE_JOBS=1 mise install-all +``` + +This is the same workaround both Docker builds bake in, so reach for it first if +a local install or build misbehaves. + ## Contributing Use `mise run fmt-all` and `mise run check-all` to run formatters and checkers. diff --git a/config/conftest/policy/cargo.rego b/config/conftest/policy/cargo.rego index 3073b91..f56ac86 100644 --- a/config/conftest/policy/cargo.rego +++ b/config/conftest/policy/cargo.rego @@ -190,3 +190,13 @@ deny contains msg if { is_dep_name(file, feat) msg := sprintf("%s: feature %q shares its name with a dependency; rename it", [file.path, feat]) } + +# Dependency overrides ([patch]/[replace]) belong in the root manifest, where +# they apply workspace-wide and stay in one place; a member can't override deps. +deny contains msg if { + some file in input + is_member(file) + some table in {"patch", "replace"} + file.contents[table] + msg := sprintf("%s: [%s] belongs in the root manifest, not a member crate", [file.path, table]) +} diff --git a/config/conftest/policy/mise.rego b/config/conftest/policy/mise.rego index 2609684..0cb09c3 100644 --- a/config/conftest/policy/mise.rego +++ b/config/conftest/policy/mise.rego @@ -57,3 +57,23 @@ deny contains msg if { startswith(name, "ubi:") msg := sprintf("%s: tool %q uses the deprecated ubi backend; use http: instead", [file.path, name]) } + +# Tools should work on every OS (CLAUDE.md "Tools must work on every OS"). Any +# os-scoped [tools] entry must be a genuinely platform-specific one in this list. +allowed_os_scoped_tool := { + "chromedriver", + "pipx", + "npm:pnpm", + "pnpm", + "github:christianhelle/openapi2zig", +} + +deny contains msg if { + some file in input + is_mise(file) + some name, spec in file.contents.tools + is_object(spec) + spec.os + not allowed_os_scoped_tool[name] + msg := sprintf("%s: tool %q is os-scoped; tools must work on every OS (or allowlist it)", [file.path, name]) +} diff --git a/config/ls-lint.yaml b/config/ls-lint.yaml new file mode 100644 index 0000000..bf4187e --- /dev/null +++ b/config/ls-lint.yaml @@ -0,0 +1,43 @@ +# ls-lint file/directory naming linter (https://ls-lint.org), run via +# `mise run ls-lint-check`. ls-lint walks the working tree (not git), so the +# build/vendor/codegen trees are ignored below and only hand-written source is +# linted. Each rule lists the casings actually used in the repo today. +ls: + # Directories are kebab-case (most) or snake_case (a few generated-style + # names); the regex alternative permits leading-dot tool dirs (.github, .mise). + .dir: kebab-case | snake_case | regex:\.[a-z]+ + + .rs: snake_case | kebab-case + .py: snake_case | regex:__[a-z]+__ + .js: snake_case + .zig: snake_case + .dart: snake_case + .wit: snake_case | kebab-case + .rego: snake_case | kebab-case + .yaml: snake_case | kebab-case + .cs: PascalCase + .java: PascalCase + +ignore: + - .git + - target + - node_modules + - generated + # Bundled data/model assets -- names are not hand-chosen. + - data + # Build/vendor output that ls-lint would otherwise walk (it can't read + # .gitignore). The `**/` form matches each at any depth. + - "**/.venv" + - "**/__pycache__" + - "**/.pytest_cache" + - "**/.ruff_cache" + - "**/.zig-cache" + - "**/zig-out" + - "**/.dart_tool" + - "**/obj" + - "**/bin/Debug" + - "**/bin/Release" + # Built module artifacts (wheels, .wasm, generated JS) live under each pkg/. + - "**/pkg" + # Zig codegen templates dir whose name carries a literal `.in` suffix. + - "**/zig.in" diff --git a/config/zizmor.yaml b/config/zizmor.yaml new file mode 100644 index 0000000..8d9fb3d --- /dev/null +++ b/config/zizmor.yaml @@ -0,0 +1,16 @@ +# zizmor GitHub Actions audit config, applied via `mise run zizmor-check`. +# +# We pin actions by tag (`@v4`), not by commit hash: tags are readable and +# Dependabot keeps them current. So the `unpinned-uses` audit's default +# hash-pin policy is turned off here -- `any` accepts a tag/branch ref. +rules: + unpinned-uses: + config: + policies: + "*": any + obfuscation: + # test.yaml's matrix `include` is built with `fromJSON(... format(...))` + # precisely because the OS list depends on `github.event_name` / the + # workflow_dispatch input -- it cannot "be reduced to a constant". + ignore: + - test.yaml From 9fc3fe500f4e733fbcb934440c36ef79db5ff5f3 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 17:10:21 +0800 Subject: [PATCH 59/60] java and C linting --- .github/workflows/docker-windows.yaml | 2 +- .mise/config.java.toml | 6 ++++ .mise/config.linux.toml | 4 +++ .mise/config.toml | 3 ++ .mise/config.zig.toml | 43 +++++++++++++++++++++--- CLAUDE.md | 6 ++-- config/clang-format.yaml | 6 ++++ config/clang-tidy.yaml | 5 +++ pom.xml | 35 +++++++++++++++++++ services/ws-modules/zig-data1/src/util.c | 3 +- 10 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 config/clang-format.yaml create mode 100644 config/clang-tidy.yaml diff --git a/.github/workflows/docker-windows.yaml b/.github/workflows/docker-windows.yaml index c4bea98..7c3185a 100644 --- a/.github/workflows/docker-windows.yaml +++ b/.github/workflows/docker-windows.yaml @@ -22,7 +22,7 @@ defaults: jobs: build: runs-on: windows-2022 - timeout-minutes: 80 + timeout-minutes: 120 env: # The classic Windows builder can't substitute build-args into the Dockerfile's RUN, # and mise's prebuilt "latest" zip is stale (2026.3.0, too old for the config). diff --git a/.mise/config.java.toml b/.mise/config.java.toml index a5cbf11..d1d66d3 100644 --- a/.mise/config.java.toml +++ b/.mise/config.java.toml @@ -15,6 +15,12 @@ MAVEN_ARGS = "--no-transfer-progress" description = "Build the java-data1 workflow module" run = "mvn package" +# Namespaced aggregator picked up by the default config's globbed `check`. The +# compile triggers maven-compiler-plugin with -Xlint:all -Werror + Error Prone. +[tasks."check:java"] +description = "Run Java checks (javac -Xlint:all -Werror, Error Prone)" +run = "mvn -q compile" + [tasks."prefetch:java"] description = "Prefetch Java (Maven) dependencies" run = "mvn dependency:resolve --quiet" diff --git a/.mise/config.linux.toml b/.mise/config.linux.toml index 151f4b8..bf30ccb 100644 --- a/.mise/config.linux.toml +++ b/.mise/config.linux.toml @@ -23,6 +23,10 @@ b_clang_sysroot = "{{ vars.a_conda_clangxx }}/{{ vars.a_conda_arch }}-conda-linu # in CI). Using conda's sysroot rather than the host's /usr/include keeps bindgen # self-contained. c_bindgen_args = "-resource-dir {{ vars.b_clang_resource }} --sysroot={{ vars.b_clang_sysroot }}" +# clang-tidy gets the same resource-dir + conda sysroot as bindgen: the sysroot +# also stops clang scanning the host's multiple /usr/lib/gcc installs for +# libstdc++ (which otherwise warns, and -warnings-as-errors fails the run). +clang_resource_arg = "{{ vars.c_bindgen_args }}" [env] # clang-sys finds conda's libclang.so via LIBCLANG_PATH; bindgen gets conda's diff --git a/.mise/config.toml b/.mise/config.toml index 16ccc27..458a098 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -78,6 +78,9 @@ yq = "latest" zizmor = "latest" [vars] +# clang-tidy's resource-dir arg (points clang at its builtin headers, e.g. +# stddef.h). Empty default; config.linux.toml sets it from conda:clangxx. +clang_resource_arg = "" conda_openssl = "{{ env.HOME }}/.local/share/mise/installs/conda-openssl/latest" # mise-managed CPython 3.13 (Linux/macOS); the PYO3_PYTHON default below. Windows # overrides PYO3_PYTHON with its own py313_win in config.windows.toml. diff --git a/.mise/config.zig.toml b/.mise/config.zig.toml index a2bc4d1..0404a45 100644 --- a/.mise/config.zig.toml +++ b/.mise/config.zig.toml @@ -7,6 +7,14 @@ # REST-client generation step in et-int-gen detects the missing binary at # runtime and is a no-op when it's not on PATH. "github:christianhelle/openapi2zig" = { version = "latest", os = ["linux/x64", "macos", "windows"] } +# C tooling for the zig-data1 module's hand-written C (services/ws-modules/ +# zig-data1/src/*.c). Two conda packages because mise hands the `clang-format` +# name to conda:clang-format, so conda:clang-tools omits it and only supplies +# clang-tidy. Both are conda-forge's native LLVM binaries. cpplint (Google C++ +# style) has no prebuilt binary -- pipx is its only distribution. +"conda:clang-format" = "latest" +"conda:clang-tools" = "latest" +"pipx:cpplint" = "latest" zig = "latest" [tasks.zig-check] @@ -15,14 +23,41 @@ run = "zig fmt --check services/ws-modules/ generated/zig-rest/" [tasks.zig-fmt] run = "zig fmt services/ws-modules/ generated/zig-rest/" +[tasks.clang-format] +description = "Format C sources (clang-format)" +run = "git ls-files '*.c' '*.h' | xargs clang-format -i --style=file:config/clang-format.yaml" + +[tasks.clang-format-check] +description = "Check C source formatting (clang-format)" +run = "git ls-files '*.c' '*.h' | xargs clang-format --dry-run --Werror --style=file:config/clang-format.yaml" + +[tasks.clang-tidy-check] +description = "Lint C sources (clang-tidy)" +# Flags after `--` are the compile args; clang_resource_arg points clang at its +# builtin headers (set per host -- see config.linux.toml). {{ }} renders to empty +# where unset, leaving a bare `--`. +run = """ +git ls-files '*.c' '*.h' | + xargs -I{} clang-tidy --config-file=config/clang-tidy.yaml {} -- {{ vars.clang_resource_arg }} +""" +shell = "bash -euo pipefail -c" + +[tasks.cpplint-check] +description = "Lint C sources for Google C++ style (cpplint)" +run = """ +git ls-files '*.c' '*.h' | + xargs cpplint --quiet --linelength=120 --filter=-legal/copyright,-build/include_subdir +""" +shell = "bash -euo pipefail -c" + # Namespaced aggregators picked up by the default config's globbed `check`/`fmt`. [tasks."check:zig"] -depends = ["zig-check"] -description = "Run Zig checks (format-check)" +depends = ["zig-check", "clang-format-check", "clang-tidy-check", "cpplint-check"] +description = "Run Zig + C checks (zig fmt-check, clang-format, clang-tidy, cpplint)" [tasks."fmt:zig"] -depends = ["zig-fmt"] -description = "Format Zig sources" +depends = ["zig-fmt", "clang-format"] +description = "Format Zig + C sources" [tasks.build-ws-zig-data1-module] description = "Build the zig-data1 workflow WASM module" diff --git a/CLAUDE.md b/CLAUDE.md index fbf0b3e..c7a793d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,6 +106,7 @@ them all; guest rows need their `MISE_ENV` loaded). | `*.py` | `ruff-fmt` | | `*.dart` | `fmt:dart` | | `*.zig` | `fmt:zig` | +| `*.c` | `clang-format` | | `*.cs` | `fmt:dotnet` | | File type | Check task(s) | @@ -117,6 +118,7 @@ them all; guest rows need their `MISE_ENV` loaded). | `*.py` | `check:python` | | `*.dart` | `check:dart` | | `*.zig` | `check:zig` | +| `*.c` | `clang-format-check`, `clang-tidy-check`, `cpplint-check` | | `*.cs` | `check:dotnet` | | `*.java` | `check:java` | @@ -312,8 +314,8 @@ available linters: - **conftest** (`config/conftest/policy/`) — Rego policies over the combined TOML/YAML config set, for cross-file checks the schema linters can't express. - plus hadolint, ls-lint (file/dir naming), zizmor (Actions security), ryl - (YAML), lychee (links), editorconfig-checker, typos, and action-validator for - their domains. + (YAML), lychee (links), clang-format / clang-tidy / cpplint (C, in the zig + config), editorconfig-checker, typos, and action-validator for their domains. ast-grep has no TOML grammar, so it **cannot** lint TOML — use a taplo schema or a semgrep `generic` rule there. If none of the above can express a check, diff --git a/config/clang-format.yaml b/config/clang-format.yaml new file mode 100644 index 0000000..e09a977 --- /dev/null +++ b/config/clang-format.yaml @@ -0,0 +1,6 @@ +# clang-format style for C sources, applied via `mise run clang-format` (and +# clang-format-check). Passed to clang-format as --style=file:. LLVM base +# with the repo's 120-column limit and 4-space indent (matches the C sources). +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 diff --git a/config/clang-tidy.yaml b/config/clang-tidy.yaml new file mode 100644 index 0000000..aaf5045 --- /dev/null +++ b/config/clang-tidy.yaml @@ -0,0 +1,5 @@ +# clang-tidy checks for C sources, applied via `mise run clang-tidy-check` +# (passed as --config-file=). Bug-finding analysis only; opinionated +# readability/style checks are left off to avoid churn on the small C surface. +Checks: "clang-analyzer-*,bugprone-*,performance-*,portability-*" +WarningsAsErrors: "*" diff --git a/pom.xml b/pom.xml index e7e46f3..85c603c 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,41 @@ ${project.basedir}/services/ws-modules/java-data1/src/main/java + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + true + + -Xlint:all + -Werror + -XDcompilePolicy=simple + --should-stop=ifError=FLOW + -Xplugin:ErrorProne + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + + + + com.google.errorprone + error_prone_core + 2.49.0 + + + + org.teavm teavm-maven-plugin diff --git a/services/ws-modules/zig-data1/src/util.c b/services/ws-modules/zig-data1/src/util.c index 57cfea0..5a70034 100644 --- a/services/ws-modules/zig-data1/src/util.c +++ b/services/ws-modules/zig-data1/src/util.c @@ -4,6 +4,7 @@ // Returns the sum of all bytes in buf, mod 256. uint8_t byte_sum(const uint8_t *buf, size_t len) { uint8_t acc = 0; - for (size_t i = 0; i < len; i++) acc += buf[i]; + for (size_t i = 0; i < len; i++) + acc += buf[i]; return acc; } From ad28489322d2588870d104d4b3630a12a139fa8e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 14 Jun 2026 17:58:19 +0800 Subject: [PATCH 60/60] disable cpplint on nano --- Dockerfile.nanoserver | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index b9df316..b4ec877 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -202,14 +202,14 @@ ENV PATH="C:\Users\ContainerAdministrator\AppData\Local\mise\installs\python\3.1 RUN (if exist C:\token\gh_token set /p GITHUB_TOKEN=