diff --git a/.dockerignore b/.dockerignore index dd97eb3..97a1453 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ **/target/ **/.DS_Store services/ws-wasm-agent/pkg/ +services/ws-modules/pywasm1/pkg/ services/ws-server/static/models/ **/.zig-cache/ **/zig-out/ diff --git a/.editorconfig b/.editorconfig index 5276445..c51eb0f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -79,3 +79,12 @@ max_line_length = unset # The generator is the source of truth; we don't edit its output by hand. [generated/zig-rest/**] max_line_length = unset + +# config/upstream-cache/data.toml stores per-asset metadata including the canonical +# download URL (`https://github.com/edge-toolkit/core/releases/download//`) +# and the upstream project URL. Both routinely exceed 120 chars and can't be +# meaningfully line-wrapped in TOML (a string value can't span lines without +# multi-line-string syntax that would change the parsed value). Loosen to 200. +[config/upstream-cache/data.toml] +max_line_length = 200 + diff --git a/.github/actions/install-mise-tools/action.yaml b/.github/actions/install-mise-tools/action.yaml new file mode 100644 index 0000000..afee2dc --- /dev/null +++ b/.github/actions/install-mise-tools/action.yaml @@ -0,0 +1,74 @@ +--- +name: Install mise tools +description: >- + Install mise (via install-mise), then trust the .mise config, and install + the toolchain named by the workflow's MISE_ENV (with one retry on transient + HTTP flakes). Shared by check.yaml and test.yaml; the caller must run + `actions/checkout` before this so .mise/config*.toml exists for `mise trust`. + +inputs: + github-token: + description: >- + GitHub token forwarded as `$GITHUB_TOKEN` to the mise/preinstall steps. + Raises the api.github.com rate-limit ceiling on aqua's attestation + lookups + tool-asset fetches. + required: true + extra-tools: + description: >- + Comma-separated tools to co-install via taiki-e/install-action in the + same step as mise. Forwarded to install-mise's `extra-tools` input. + NOT used for `aube` -- install-action's manifest expects an `aubr` + binary that recent aube releases don't ship, so the install resolves + to a cargo source-build that flakes on crates.io SSL. The aube path + below uses the original `mise run setup-aube` (npm-backed, allowed + to fail) which has been the reliable install method. + required: false + default: "" + +runs: + using: composite + steps: + - name: Install mise binary + uses: ./.github/actions/install-mise + with: + extra-tools: ${{ inputs.extra-tools }} + + - name: Show MISE_ENV + shell: bash + run: echo "MISE_ENV=$MISE_ENV" + + # Optional npm backend, installed before the main `mise install`. + # Only useful when js env is loaded (it's the backend for npm:* tools, all + # of which live in config.js.toml); skip otherwise to avoid the install + # cost on workflows that don't need any npm: install. + # See [tasks.setup-aube] in .mise/config.toml for the full rationale. + - name: Install aube (optional npm backend, allowed to fail) + if: contains(env.MISE_ENV, 'js') + continue-on-error: true + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + run: mise run setup-aube + + - name: Install mise tools + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + run: | + mise run preinstall + # Retry once on transient api.github.com 5xx during the aqua attestation + # / asset fetches: the first attempt re-uses what's already on disk, so + # the retry usually clears single-tool HTTP flakes in seconds. + mise install || { echo "::warning::mise install failed, retrying once"; mise install; } + + # Delete mise's auto-generated `mise.cmd` shim on Windows so cmd's PATH + # search falls through to the real `mise.exe` at `~/.cargo/bin/mise.exe` + # (staged by taiki-e/install-action). Must run AFTER `mise install` -- + # that's the step that creates the shims. + - name: Remove self-recursive mise.cmd shim on Windows + if: runner.os == 'Windows' + shell: bash + run: | + shims="$LOCALAPPDATA/mise/shims" + rm -fv "$shims/mise.cmd" "$shims/mise" + ls "$shims" | grep -i '^mise' || true diff --git a/.github/actions/install-mise/action.yaml b/.github/actions/install-mise/action.yaml new file mode 100644 index 0000000..d697f96 --- /dev/null +++ b/.github/actions/install-mise/action.yaml @@ -0,0 +1,48 @@ +--- +name: Install mise +description: >- + Install the mise binary (and cargo-binstall as its source-build backend) + via taiki-e/install-action, and on Windows add mise's file-shim dir to + PATH so subsequent steps can resolve mise-managed tools by bare name. + Shared by every workflow that runs `mise` on the runner. For the heavier + "also install everything named by MISE_ENV" flow, use install-mise-tools. + +inputs: + extra-tools: + description: >- + Comma-separated tools to co-install via taiki-e/install-action in the + same step as mise (cheaper than a second install-action call). Use the + same syntax as install-action's `tool:` input. + required: false + default: "" + +runs: + using: composite + steps: + - name: Install mise (and any extra tools) + uses: taiki-e/install-action@v2 + with: + tool: cargo-binstall,mise@2026.6.5${{ inputs.extra-tools && ',' || '' }}${{ inputs.extra-tools }} + + # taiki-e/install-action's mise manifest stages only `mise.exe`; mise's + # per-tool `.cmd` "file" shims under `%LOCALAPPDATA%\mise\shims` aren't + # on PATH by default, so tool lookups inside `mise run` fail with + # `'dart' / 'mvn' / 'recur' / 'wasm-pack' / 'zig' is not recognized…`. + # We append the shims dir for subsequent steps. The companion + # `mise-shim.exe` (single-binary shim) ALSO ships in the Windows zip, + # but staging it triggers a CreateProcess `ERROR_FILENAME_EXCED_RANGE` + # (os error 206) when mise-shim re-execs mise.exe with the runner's + # already-long PATH inherited -- the file-shim fallback doesn't have + # this problem (each .cmd shim invokes `mise exec --` plainly via cmd). + - name: Add mise shims dir to PATH on Windows + if: runner.os == 'Windows' + shell: bash + run: echo "$LOCALAPPDATA\mise\shims" >> "$GITHUB_PATH" + + # Marks the checked-out .mise/config*.toml as trusted so subsequent mise + # invocations don't error out with "Config files are not trusted". The + # caller must run actions/checkout BEFORE this composite action so the + # config files are on disk. + - name: Trust mise config + shell: bash + run: mise trust diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index e18059f..ba13f09 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -16,7 +16,10 @@ concurrency: defaults: run: - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig jobs: check: @@ -29,35 +32,10 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Install mise - uses: taiki-e/install-action@v2 + - name: Install mise + tools + uses: ./.github/actions/install-mise-tools with: - tool: cargo-binstall,mise@2026.6.5 - - - name: Select all language envs - run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - # Optional npm backend, installed before the main `mise install`. - # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - - name: Install aube (optional npm backend, allowed to fail) - continue-on-error: true - run: | - mise settings experimental=true - mise run setup-aube - env: - GITHUB_TOKEN: ${{ github.token }} - MISE_HTTP_TIMEOUT: "120" - - - name: Install mise tools - run: | - mise run preinstall - mise install - env: - GITHUB_TOKEN: ${{ github.token }} - # GitHub release downloads occasionally take longer than mise's - # default 30s HTTP timeout; bump it so transient network slowness - # doesn't fail the whole `mise install` step. - MISE_HTTP_TIMEOUT: "120" + github-token: ${{ github.token }} - name: Prefetch Rust dependencies run: mise run prefetch:rust diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml index eea0754..f021416 100644 --- a/.github/workflows/dependencies.yaml +++ b/.github/workflows/dependencies.yaml @@ -10,6 +10,7 @@ name: dependencies - config/deny.toml - config/osv-scanner.toml - .github/workflows/dependencies.yaml + - .github/actions/install-mise/** workflow_dispatch: permissions: @@ -21,10 +22,19 @@ concurrency: defaults: run: - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig jobs: dependencies: + # `ubuntu-latest`, not `ubuntu-slim`: although every step in this workflow + # is metadata/network-bound (no Rust compile), `taiki-e/install-action` + # itself assumes a standard runner-image FHS (e.g. ~/.cargo/bin) and + # fails on slim's bare container with `I/O Error: No such file or + # directory` while staging cargo-unmaintained. Slim only fits workflows + # whose first step doesn't need that scaffolding. runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -34,13 +44,13 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Install tools - uses: taiki-e/install-action@v2 + - name: Install mise + dependency-scan tools + uses: ./.github/actions/install-mise with: - tool: cargo-deny,cargo-unmaintained,coreutils,mise@2026.6.5,osv-scanner,ripgrep + extra-tools: cargo-deny,cargo-unmaintained,coreutils,osv-scanner,ripgrep - - name: Trust mise config - run: mise trust + - name: Show MISE_ENV + run: echo "MISE_ENV=$MISE_ENV" - name: Generate config/osv-scanner.toml from config/deny.toml run: mise run gen:osv-scanner diff --git a/.github/workflows/docker-linux.yaml b/.github/workflows/docker-linux.yaml index cd77e89..b549bb8 100644 --- a/.github/workflows/docker-linux.yaml +++ b/.github/workflows/docker-linux.yaml @@ -17,7 +17,10 @@ concurrency: defaults: run: - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig jobs: build: @@ -27,13 +30,15 @@ jobs: fail-fast: false matrix: base: - - ubuntu:22.04 - - ubuntu:24.04 - - ubuntu:26.04 + - amazonlinux:2023 - debian:bookworm - debian:trixie - fedora:42 - mcr.microsoft.com/azurelinux/base/core:3.0 + - opensuse/leap:15.6 + - ubuntu:22.04 + - ubuntu:24.04 + - ubuntu:26.04 name: build (${{ matrix.base }}) steps: - name: Checkout @@ -42,12 +47,18 @@ jobs: fetch-depth: 1 persist-credentials: false - # The image is huge (every language toolchain + prefetched models + a full - # debug build, incl. aws-lc-sys's large C objects), and its peak `target/` - # overruns a single runner disk. Reclaim the unused preinstalled SDKs and - # concatenate the freed root space with /mnt into one LVM volume mounted at - # Docker's data dir, then restart Docker so the build uses the combined space. - - name: Maximize build space (combine root + /mnt for Docker) + - name: Show MISE_ENV + run: echo "MISE_ENV=$MISE_ENV" + + # jlumbroso/free-disk-space (which test.yaml uses) only frees space on + # the existing root partition. This workflow needs more: the image build + # writes to /var/lib/docker, so we have to relocate Docker's data dir + # onto an LVM volume that concatenates the freed root space with /mnt. + # easimon is the only maintained action that does that LVM remount, so + # we stay on it here despite its rougher edges (hard-fails on runners + # without /mnt or without a Docker daemon -- not a concern for this + # workflow's ubuntu-latest matrix, which has both). + - name: Maximize build space uses: easimon/maximize-build-space@v10 with: root-reserve-mb: 4096 @@ -68,7 +79,10 @@ jobs: BASE_IMAGE: ${{ matrix.base }} DOCKER_BUILDKIT: "1" GITHUB_TOKEN: ${{ github.token }} - run: docker build --target test --build-arg BASE_IMAGE --secret id=gh_token,env=GITHUB_TOKEN -t et-test . + run: | + args="--target test --build-arg BASE_IMAGE --build-arg MISE_ENV" + args="$args --secret id=gh_token,env=GITHUB_TOKEN -t et-test" + docker build $args . # The check stage needs `.git/` + every tracked Dockerfile. The main # `.dockerignore` excludes both (`**/Dockerfile*` so Dockerfiles in @@ -90,10 +104,57 @@ jobs: mkdir -p "$CTX" cp -r .git "$CTX/" git ls-files '*Dockerfile' '*Dockerfile.*' | xargs -I{} cp --parents {} "$CTX/" - docker build --target check --build-arg BASE_IMAGE --build-context extras=$CTX --secret id=gh_token -t $TAG . + args="--target check --build-arg BASE_IMAGE --build-arg MISE_ENV" + args="$args --build-context extras=$CTX --secret id=gh_token -t $TAG" + docker build $args . + + # Capture image + intermediate stage sizes after the builds (and after + # the test/check runs leave any -rm'd container layers behind). `always()` + # so we still get the breakdown when a downstream step ran the disk dry; + # the typical failure mode (test-ws-web-runner compile inside `docker run`) + # leaves the host's image catalogue intact and this step is what surfaces + # which layer ballooned. -a includes intermediate/dangling layers each + # stage produces; `system df -v` totals per cache type (Images, Build + # Cache, Containers, Volumes). + - name: Docker disk usage (debug) + if: always() + run: | + echo "::group::docker images -a" + docker images -a + echo "::endgroup::" + echo "::group::docker system df -v" + docker system df -v + echo "::endgroup::" + echo "::group::df -h /var/lib/docker" + df -h /var/lib/docker || true + echo "::endgroup::" + # In-image large-file report: which paths inside the et-test image + # are eating space? `du --threshold=10M --max-depth=2` for a + # top-level subtree map (catches install dirs, tool caches, /var + # stragglers); `find ... -size +50M` for the individual + # heavyweights. Run in a throwaway container so the host's + # writable layer doesn't tilt the numbers; find/du stderr noise + # (unreadable /proc entries etc.) prints to the log but doesn't + # affect the sort pipeline. + echo "::group::et-test: largest dirs (depth 2, >=10M)" + docker run --rm et-test sh -c "du -h --threshold=10M --max-depth=2 / | sort -h | tail -50" + echo "::endgroup::" + echo "::group::et-test: individual files >50M" + docker run --rm et-test sh -c "find / -xdev -type f -size +50M -exec du -h {} + | sort -h | tail -50" + echo "::endgroup::" + + # Two `docker run` invocations: each starts a fresh container with an + # empty writable layer, so the per-phase compile target/ doesn't share + # disk with the other phase. Splits cargo-test's two halves (workspace + # minus et-ws-web-runner, then et-ws-web-runner serialized) for clearer + # attribution when a tight-disk runner fails. test-ws-web-runner + # already skips itself on hosts that can't satisfy its Deno-runtime + # requirements, so this step is a no-op on those platforms. + - name: Run cargo-test-other + run: docker run --rm et-test mise run cargo-test-other - - name: Run the test suite - run: docker run --rm et-test + - name: Run test-ws-web-runner + run: docker run --rm et-test mise run test-ws-web-runner - name: Run mise check run: docker run --rm et-check diff --git a/.github/workflows/docker-windows.yaml b/.github/workflows/docker-windows.yaml index fa62d52..38524e4 100644 --- a/.github/workflows/docker-windows.yaml +++ b/.github/workflows/docker-windows.yaml @@ -6,6 +6,7 @@ name: docker-windows paths: - .github/workflows/docker-windows.yaml - Dockerfile.nanoserver + - Dockerfile.windows workflow_dispatch: permissions: @@ -17,7 +18,10 @@ concurrency: defaults: run: - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig jobs: build: @@ -25,11 +29,27 @@ jobs: strategy: fail-fast: false matrix: + # `base` controls which Dockerfile is built; `runner` picks the host + # GHA runner. `nanoserver` is the minimal ~120 MB lane; `servercore` + # is the heavier ~1.25 GB lane with a fuller Windows runtime (used + # by Dockerfile.windows). Both run on each Windows runner version + # so we exercise the ltsc2022/ltsc2025 base-image pair against the + # matching runner kernel. Workflow-level MISE_ENV is the standard + # set; Dockerfile.nanoserver pins its own narrower MISE_ENV via an + # ENV that overrides the inherited ARG (Nano can't install python + # or dotnet). include: - runner: windows-2022 + base: nanoserver + - runner: windows-2025 + base: nanoserver + build_arg: "--build-arg WINDOWS_VERSION=ltsc2025" + - runner: windows-2022 + base: servercore - runner: windows-2025 + base: servercore build_arg: "--build-arg WINDOWS_VERSION=ltsc2025" - name: build (${{ matrix.runner }}) + name: build (${{ matrix.runner }}, ${{ matrix.base }}) timeout-minutes: 120 env: # The classic Windows builder on windows-2022 can't substitute build-args into the Dockerfile's RUN, @@ -43,6 +63,9 @@ jobs: fetch-depth: 1 persist-credentials: false + - name: Show MISE_ENV + run: echo "MISE_ENV=$MISE_ENV" + # Hosted Windows runners don't reliably leave the Docker daemon running, so # the build can fail connecting to the docker_engine pipe. Start it (no-op # if already running) and confirm connectivity before building. @@ -51,11 +74,91 @@ jobs: sc query docker | grep -q RUNNING || net start docker docker version + # Free runner disk before the build. The Server Core lane has been hitting + # "There is not enough space on the disk" during the conda-clang-tools + # install inside `mise install`. `docker system prune -af` clears any + # unused images/containers from earlier matrix entries on the same + # runner; `docker builder prune -af` clears BuildKit's layer cache. + # Disk-free is logged before + after so we can see how much each prune + # actually reclaims on this runner image -- adjust the cleanup set up or + # down based on the delta. + - name: Free disk space before docker build + env: + PS_DISKFREE: Get-PSDrive C | Select-Object -Property Used,Free | Format-List + run: | + freespace() { powershell.exe -NoProfile -Command "$PS_DISKFREE"; } + echo "::group::disk-free BEFORE prune" + freespace + echo "::endgroup::" + echo "::group::docker system prune -af" + docker system prune -af + echo "::endgroup::" + echo "::group::docker builder prune -af" + docker builder prune -af + echo "::endgroup::" + echo "::group::disk-free AFTER prune" + freespace + echo "::endgroup::" + - name: Prepare mise and Github token for the build context run: | v="${{ env.MISE_VERSION }}" curl -fsSL -o mise.zip "https://github.com/jdx/mise/releases/download/v$v/mise-v$v-windows-x64.zip" printf '%s' "${{ github.token }}" > gh_token + # Pick which Dockerfile each matrix lane builds. The build is split into + # `build-minimal` → diagnostics → `precompile` so the et-rp (rustpython) + # diagnostic runs between the two stages. Both `docker build` calls share + # BuildKit's layer cache; the second one rebuilds only the layers above + # `build-minimal`, so the split adds the diagnostic step's runtime, not + # a second full compile. + - name: Resolve Dockerfile + image tag + id: dockerfile + run: | + case "${{ matrix.base }}" in + nanoserver) dockerfile=Dockerfile.nanoserver ;; + servercore) dockerfile=Dockerfile.windows ;; + *) echo "unknown matrix.base: ${{ matrix.base }}" >&2; exit 2 ;; + esac + tag="et-windows:${{ matrix.runner }}-${{ matrix.base }}-build-minimal" + echo "dockerfile=$dockerfile" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Build stage build-minimal + env: + DOCKERFILE: ${{ steps.dockerfile.outputs.dockerfile }} + IMAGE_TAG: ${{ steps.dockerfile.outputs.tag }} + run: | + args="-f $DOCKERFILE ${{ matrix.build_arg }} --build-arg MISE_ENV" + args="$args --target build-minimal -t $IMAGE_TAG" + docker build $args . + + # Diagnostic: does the et-rp (rustpython) binary actually run inside this + # base image? servercore expected: works. nanoserver expected: fails with + # 0xc0000139 (STATUS_ENTRYPOINT_NOT_FOUND) when rustpython.exe tries to + # resolve symbols from Nano-stripped shell32/winmm/propsys. Captured here + # rather than only via the preinstall fallback so the comparison sits + # side-by-side in the same workflow run. continue-on-error so a Nano + # failure here doesn't pre-empt the precompile build that follows. + - name: Diagnose et-rp (rustpython) in build-minimal + continue-on-error: true + env: + IMAGE_TAG: ${{ steps.dockerfile.outputs.tag }} + run: | + echo "::group::which rustpython" + docker run --rm "$IMAGE_TAG" cmd /c "mise where http:et-rp & where rustpython" + echo "::endgroup::" + echo "::group::rustpython --version" + docker run --rm "$IMAGE_TAG" cmd /c "mise exec -- rustpython --version" + echo "::endgroup::" + echo "::group::rustpython -c hello" + docker run --rm "$IMAGE_TAG" cmd /c "mise exec -- rustpython -c \"print('et-rp ok')\"" + echo "::endgroup::" + - name: Build stage precompile - run: docker build -f Dockerfile.nanoserver ${{ matrix.build_arg }} --target precompile . + env: + DOCKERFILE: ${{ steps.dockerfile.outputs.dockerfile }} + run: | + args="-f $DOCKERFILE ${{ matrix.build_arg }} --build-arg MISE_ENV" + args="$args --target precompile" + docker build $args . diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1a09c89..9b0605e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,10 +12,12 @@ name: test options: - all - ubuntu-latest + - ubuntu-26.04 - ubuntu-24.04-arm - macos-latest - macos-26-intel - windows-latest + - windows-11-arm permissions: contents: read @@ -26,18 +28,39 @@ concurrency: defaults: run: - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: "dart,dotnet,java,js,python,rust,zig" jobs: rust: runs-on: ${{ matrix.os }} timeout-minutes: ${{ matrix.timeout }} + # Override the workflow-level `shell: bash` default for Windows runs to + # use the busybox-w32 ash that mise installs in `_setup_all` (the same + # shell `MISE_BASH_PATH` already points at for `shell = "bash"` tasks on + # the Windows Docker images). Avoids Git Bash's MSYS fork-emulation + # cygheap pathology -- see OLD_NOTES.md for the captured error + # signatures and the diagnostic harness that found the root cause. The + # path matches the version pin in `.mise/config.windows.toml`'s + # [tools."http:busybox"] entry. The `Set HOME = USERPROFILE on Windows` + # step below explicitly overrides back to `shell: bash` because it + # runs BEFORE `Install mise + tools` (which is what installs + # busybox-w32). All later run-bodies use ash. + defaults: + run: + shell: >- + ${{ startsWith(matrix.os, 'windows') && + 'C:\Users\runneradmin\AppData\Local\mise\installs\http-busybox\1.37.0\ash.exe -euo pipefail {0}' + || 'bash --noprofile --norc -euo pipefail {0}' }} strategy: fail-fast: false # `matrix` context isn't available in job-level `if:`, so we compute the # OS list right here in the strategy block (where `github.*` IS in scope): # pull_request → ubuntu-latest + ubuntu-24.04-arm + macos-latest - # (skip slow Intel Mac & Windows) + # + windows-latest (skip slow Intel Mac; + # ubuntu-26.04 + windows-11-arm are dispatch-only) # workflow_dispatch → the OS picked in the dropdown, or all of them # The combos are emitted as a fully-formed list of `{os, timeout}` # objects so the `include:` "unmatched entries append as new jobs" @@ -46,38 +69,29 @@ jobs: include: ${{ fromJSON( github.event_name == 'workflow_dispatch' && ( github.event.inputs.os == 'all' && format( - '[{0},{1},{2},{3},{4}]', - '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":30}', + '[{0},{1},{2},{3},{4},{5},{6}]', + '{"os":"ubuntu-latest","timeout":40}', + '{"os":"ubuntu-26.04","timeout":40}', + '{"os":"ubuntu-24.04-arm","timeout":40}', '{"os":"macos-latest","timeout":45}', '{"os":"macos-26-intel","timeout":60}', - '{"os":"windows-latest","timeout":60}' + '{"os":"windows-latest","timeout":60}', + '{"os":"windows-11-arm","timeout":60}' ) - || github.event.inputs.os == 'ubuntu-latest' && '[{"os":"ubuntu-latest","timeout":30}]' - || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":30}]' + || github.event.inputs.os == 'ubuntu-latest' && '[{"os":"ubuntu-latest","timeout":40}]' + || github.event.inputs.os == 'ubuntu-26.04' && '[{"os":"ubuntu-26.04","timeout":40}]' + || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":40}]' || github.event.inputs.os == 'macos-latest' && '[{"os":"macos-latest","timeout":45}]' || github.event.inputs.os == 'macos-26-intel' && '[{"os":"macos-26-intel","timeout":60}]' || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":60}]' + || github.event.inputs.os == 'windows-11-arm' && '[{"os":"windows-11-arm","timeout":60}]' ) - || format( - '[{0},{1},{2}]', - '{"os":"ubuntu-latest","timeout":30}', - '{"os":"ubuntu-24.04-arm","timeout":30}', - '{"os":"macos-latest","timeout":45}' - ) + || format('[{0},{1},{2},{3}]', + '{"os":"ubuntu-latest","timeout":40}', + '{"os":"ubuntu-24.04-arm","timeout":40}', + '{"os":"macos-latest","timeout":45}', + '{"os":"windows-latest","timeout":60}') ) }} - env: - # Windows-only: skip the two cargo: source-builds that need a fully-set-up - # gnullvm toolchain at install time. The Nano docker (Dockerfile.nanoserver) - # sets the same flag; on the windows-latest runner here the gnullvm - # rust-std isn't reliably in place when cargo install runs them, leading to - # the rustc diagnostic: - # error[E0463]: can't find crate for `core` - # = note: the `x86_64-pc-windows-gnullvm` target may not be installed - # Both are dev-only tools (cargo-expand: macro debugging; dart-typegen: - # used only by `mise run gen:dart-ws`, which the dart config skips on - # Windows anyway); neither is in the test suite's hot path. - MISE_DISABLE_TOOLS: ${{ startsWith(matrix.os, 'windows') && 'cargo:cargo-expand,cargo:dart-typegen' || '' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -85,14 +99,9 @@ jobs: fetch-depth: 1 persist-credentials: false - # Reclaim the large preinstalled toolchains we never use (~30 GB). - # `|| true` so a missing dir (e.g. on the arm image) is not fatal. - - name: Free up disk space (Linux) + - name: Free disk space if: runner.os == 'Linux' - run: | - sudo rm -rf /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/share/swift || true - sudo docker image prune --all --force || true - df -h / + uses: jlumbroso/free-disk-space@v1.3.1 - name: Install Mesa Vulkan drivers (lavapipe) if: runner.os == 'Linux' @@ -100,42 +109,79 @@ jobs: sudo apt-get update sudo apt-get install -y mesa-vulkan-drivers - - name: Install mise - uses: taiki-e/install-action@v2 - with: - tool: cargo-binstall,mise@2026.6.5 - - - name: Select all language envs - run: echo "MISE_ENV=$(mise run print-all-langs)" >> "$GITHUB_ENV" - - # Optional npm backend, installed before the main `mise install`. - # See [tasks.setup-aube] in .mise/config.toml for the full rationale. - - name: Install aube (optional npm backend, allowed to fail) - continue-on-error: true + # Empirical probe: does the GHA windows-latest VM populate HOME in its + # default env, or is the next step (Pre-mise: HOME = USERPROFILE) + # genuinely load-bearing? Prints HOME/USERPROFILE from both Git Bash + # (which auto-sets HOME from USERPROFILE on its own) and cmd (which + # doesn't, so its output reflects only what the parent env has). + # cmd resolves an unset var by leaving the literal `%HOME%` token in + # the output, so `HOME=[%HOME%]` is the unambiguous "unset" signal; + # `HOME=[]` means defined-but-empty; `HOME=[C:\Users\...]` means set. + # Remove this step once the answer is captured in OLD_NOTES.md. + - name: "Pre-mise: probe HOME on Windows" + if: runner.os == 'Windows' + timeout-minutes: 1 + # Same `shell:` rationale as the next step -- runs before + # `Install mise + tools` so we can't use the job's busybox default. + shell: bash --noprofile --norc -euo pipefail {0} run: | - mise settings experimental=true - mise run setup-aube - env: - GITHUB_TOKEN: ${{ github.token }} - MISE_HTTP_TIMEOUT: "120" + echo "=== Git Bash perspective ===" + echo " HOME=[$HOME] USERPROFILE=[$USERPROFILE]" + echo "=== cmd.exe perspective ===" + cmd.exe /c "echo HOME=[%HOME%] USERPROFILE=[%USERPROFILE%]" - - name: Install mise tools - run: | - mise run preinstall - mise install - env: - GITHUB_TOKEN: ${{ github.token }} - # Match check.yaml: bump mise's 30s HTTP timeout so GitHub-release - # downloads don't fail the install on transient slowness. - MISE_HTTP_TIMEOUT: "120" + # mise's Tera renderer expects $HOME to be set when evaluating env.HOME + # in [vars] (config.toml has e.g. `conda_openssl = "{{ env.HOME }}/.local + # /share/mise/installs/conda-openssl/3"`). MSYS bash auto-sets HOME from + # USERPROFILE; pwsh and cmd do not, so mise tasks invoked from a pwsh + # step fail with `Variable env.HOME not found in context while rendering + # __tera_one_off`. Plumb HOME = USERPROFILE so every subsequent step + # (incl. our pwsh diagnostic ones) sees it. + - name: "Pre-mise: HOME = USERPROFILE on Windows" + if: runner.os == 'Windows' + timeout-minutes: 1 + # Explicit `shell: bash` because the job-level defaults point at the + # busybox ash that's installed by the next step (Install mise + + # tools), so we can't use it here yet. Git Bash is on the runner + # image by default. + shell: bash --noprofile --norc -euo pipefail {0} + run: echo "HOME=$USERPROFILE" >> "$GITHUB_ENV" + + - name: Install mise + tools + uses: ./.github/actions/install-mise-tools + timeout-minutes: 20 + with: + github-token: ${{ github.token }} - name: Prefetch dependencies + timeout-minutes: 15 env: GITHUB_TOKEN: ${{ github.token }} run: mise run prefetch-ci + # GITHUB_TOKEN raises the rate-limit ceiling on mise's fetch of the + # rp-v rustpython_wasm tarball ([tools."http:rp-wasm"] → mise's + # http backend against github.com release assets). + # MISE_JOBS=1 on Windows serializes mise's task scheduler so the parallel + # `build-ws-*-module` deps don't race on cargo's `.cargo/registry/index` + # lock -- on Linux/macOS cargo waits gracefully, but on Windows it fails + # fast with `cargo metadata exited with an error: Blocking waiting for + # file lock on package cache` (observed in job 82448665293, where + # build-ws-graphics-info-module + build-ws-nfc-module + build-ws-har1- + # module all kicked off in parallel). Conditional via a separate step + # rather than a step-level `env:` because mise rejects an empty + # MISE_JOBS value ("cannot parse integer from empty string"), and GHA + # has no `if:` on individual env entries. + - name: Serialize mise tasks on Windows + if: runner.os == 'Windows' + run: echo "MISE_JOBS=1" >> "$GITHUB_ENV" + - name: Build WASM modules + timeout-minutes: 25 + env: + GITHUB_TOKEN: ${{ github.token }} run: mise run build-modules - name: Run tests + timeout-minutes: 30 run: mise run test diff --git a/.github/workflows/upstream-cache.yaml b/.github/workflows/upstream-cache.yaml new file mode 100644 index 0000000..fea1605 --- /dev/null +++ b/.github/workflows/upstream-cache.yaml @@ -0,0 +1,746 @@ +--- +name: upstream-cache + +# Manually-dispatched maintainer workflow that builds + publishes +# upstream-sourced binary artifacts to GitHub releases owned by this +# repo, so consumers (CI + workstations) can `mise install` prebuilts +# instead of compiling from source. One job per cached upstream; add +# more jobs as new upstreams need caching. +# +# - rustpython: native binary per host triple + the platform-agnostic +# wasm tarball, published to the rp-v release. Forward-stocked -- +# we publish even if [tools."http:et-rp".platforms.] isn't +# wired up yet, so consumers can migrate off cargo:rustpython +# whenever it's convenient. + +"on": + # pull_request is retained per the gha_combined policy ("every workflow + # must trigger on pull_request"), but every job below is gated by + # `if: false` so PR runs are no-ops. Drop the `if: false` (and re-add + # the path scope below) on the specific job you want to iterate on. + pull_request: + paths: + - .github/workflows/upstream-cache.yaml + - .github/actions/install-mise/** + - config/upstream-cache/data.toml + workflow_dispatch: + inputs: + os: + description: "OS(es) to publish for" + type: choice + default: all + options: + - all + - ubuntu-latest + - ubuntu-24.04-arm + - macos-latest + - macos-26-intel + - windows-latest + +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +env: + MISE_ENV: dart,dotnet,java,js,python,rust,zig + +jobs: + rustpython: + # Parked. Drop the `if: false` to rebuild + republish (e.g. on a + # version bump). Default is no PR run. + if: false + runs-on: ${{ matrix.os }} + # Job-level write so gh release upload can push assets; the workflow + # default is contents:read. + permissions: + contents: write + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # workflow_dispatch with a specific OS → just that one; everything + # else (workflow_dispatch=all, push, pull_request) → every platform. + os: ${{ fromJSON( + (github.event_name == 'workflow_dispatch' + && github.event.inputs.os != 'all' + && format('["{0}"]', github.event.inputs.os)) + || '["ubuntu-latest","ubuntu-24.04-arm","macos-latest","macos-26-intel","windows-latest"]' + ) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + # Cheap host-only detection: reads the pinned short SHA from + # config.toml via sed, derives the triple from matrix.os, and + # queries rp-v1 with the runner's preinstalled gh CLI. Runs before + # any install so the typical case (asset already present) skips + # disk-free, mise install, RustPython clone, and the cargo source + # build below entirely. `work=yes` when this matrix entry has + # something to do: a missing native asset for its triple, or (only + # on ubuntu-latest) a missing wasm asset. Subsequent steps gate on + # `work`. Logic mirrors the `rp-detect-missing` mise task (kept + # for local maintainer use, where mise's tools are already there). + - name: Detect missing rp-v1 assets + id: rp_check + env: + GITHUB_TOKEN: ${{ github.token }} + # MATRIX_OS via env (not `${{ matrix.os }}` inline) so zizmor's + # template-injection rule doesn't flag matrix expansion inside the + # bash run block. + MATRIX_OS: ${{ matrix.os }} + run: | + case "$MATRIX_OS" in + ubuntu-latest) triple=x86_64-unknown-linux-gnu ;; + ubuntu-24.04-arm) triple=aarch64-unknown-linux-gnu ;; + macos-latest) triple=aarch64-apple-darwin ;; + macos-26-intel) triple=x86_64-apple-darwin ;; + windows-latest) triple=x86_64-pc-windows-msvc ;; + *) echo "unknown matrix.os: $MATRIX_OS" >&2; exit 1 ;; + esac + # awk (portable across BSD/macOS + GNU/Linux): enter the + # [tools."http:et-rp"] section, capture its `version = "..."`, + # stop on the next `[` table header. BSD sed rejects the + # `/range/{s/...p}` one-liner -- "bad flag in substitute". + short=$(awk ' + /^\[tools\."http:et-rp"\]$/ { in_section=1; next } + in_section && /^\[/ { exit } + in_section && /^version = / { gsub(/^version = "|"$/, ""); print; exit } + ' .mise/config.toml) + native_asset="rustpython-${short}-${triple}.tar.gz" + wasm_asset="rustpython-wasm-${short}.tar.gz" + repo=edge-toolkit/core + assets=$(gh release view rp-v1 --repo "$repo" --json assets --jq '.assets[].name' 2>/dev/null || true) + native_missing=true; wasm_missing=true + if echo "$assets" | grep -Fxq "$native_asset"; then native_missing=false; fi + if echo "$assets" | grep -Fxq "$wasm_asset"; then wasm_missing=false; fi + work= + [ "$native_missing" = "true" ] && work=yes + [ "$wasm_missing" = "true" ] && [ "$MATRIX_OS" = "ubuntu-latest" ] && work=yes + { + echo "short=$short" + echo "triple=$triple" + echo "native_missing=$native_missing" + echo "wasm_missing=$wasm_missing" + echo "work=$work" + } >> "$GITHUB_OUTPUT" + echo "[rp-check] triple=$triple short=$short" + echo "[rp-check] native_missing=$native_missing wasm_missing=$wasm_missing work=${work:-no}" + if [ -z "$work" ]; then + echo "::notice::All rp-v1 assets present for $triple; skipping install + publish." + fi + + # Everything below this point runs only when there's actually + # something to upload. `repo.fork != true` skips fork PRs (GHA gives + # them a read-only GITHUB_TOKEN, so the upload would fail; building + # the asset would be wasted compute). + - name: Free up disk space (Linux) + if: |- + steps.rp_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + && runner.os == 'Linux' + run: | + sudo rm -rf /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/share/swift || true + sudo docker image prune --all --force || true + df -h / + + - name: Install mise + if: |- + steps.rp_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + uses: ./.github/actions/install-mise + + - name: Show MISE_ENV + if: |- + steps.rp_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + run: echo "MISE_ENV=$MISE_ENV" + + # MISE_ENV includes `maint` here so the publish task's deps + # (augtool, via http:augeas on Windows; system-installed + # elsewhere) and any other maint-only tools install in the same + # step as the language toolchains -- otherwise the publish step + # hits `bash: line ...: : command not found` because + # `mise run` doesn't auto-install on demand. + - name: Install mise tools + if: |- + steps.rp_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + run: | + mise run preinstall + # Retry once on transient api.github.com 5xx during the aqua attestation + # / asset fetches: the first attempt re-uses what's already on disk, so + # the retry usually clears single-tool HTTP flakes in seconds. + MISE_ENV="$MISE_ENV,maint" mise install || { + echo "::warning::mise install failed, retrying once" + MISE_ENV="$MISE_ENV,maint" mise install + } + env: + GITHUB_TOKEN: ${{ github.token }} + + # RP_SHORT comes via env (not `${{ }}` inside the run block) to keep + # zizmor's template-injection rule happy -- step outputs are treated + # as potentially attacker-controllable when expanded inside bash. + # `repo.fork != true` skips fork PRs (GHA gives them a read-only + # GITHUB_TOKEN, so the upload would fail; building the asset would + # be wasted compute). + - name: Clone RustPython at pinned SHA + if: |- + github.event.pull_request.head.repo.fork != true + && (steps.rp_check.outputs.native_missing == 'true' + || (steps.rp_check.outputs.wasm_missing == 'true' && matrix.os == 'ubuntu-latest')) + env: + RP_SHORT: ${{ steps.rp_check.outputs.short }} + run: | + git clone https://github.com/RustPython/RustPython.git "$HOME/rust/RustPython" + git -C "$HOME/rust/RustPython" checkout "$RP_SHORT" + + # Windows-only: source the MSVC dev env so `rc.exe` (Windows Resource + # Compiler) is on PATH for the rustpython build. The `embed_resource` + # crate in rustpython's build.rs targets msvc and otherwise warns: + # warning: rustpython@0.5.0: Failed to compile Windows resources: + # The system cannot find the path specified. (os error 3) + # The resulting .exe still runs (resource embedding is best-effort), + # but the icon + version metadata get dropped. ilammy/msvc-dev-cmd + # is the maintained equivalent of `call vcvars64.bat` for GHA. + - name: Set up MSVC dev env for rc.exe (Windows) + if: |- + matrix.os == 'windows-latest' + && github.event.pull_request.head.repo.fork != true + && steps.rp_check.outputs.native_missing == 'true' + uses: ilammy/msvc-dev-cmd@v1 + + # Native rustpython is per-triple: each matrix entry publishes its + # own host's binary if missing. + - name: Build + publish native rustpython for this host + if: |- + github.event.pull_request.head.repo.fork != true + && steps.rp_check.outputs.native_missing == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: MISE_ENV="$MISE_ENV,maint" mise run publish-rp-native-to-release + + # Wasm rustpython is platform-agnostic; gate to a single canonical + # runner so multiple matrix jobs don't race-clobber the same asset. + - name: Build + publish wasm rustpython (ubuntu-latest only) + if: |- + github.event.pull_request.head.repo.fork != true + && steps.rp_check.outputs.wasm_missing == 'true' + && matrix.os == 'ubuntu-latest' + env: + GITHUB_TOKEN: ${{ github.token }} + run: MISE_ENV="$MISE_ENV,maint" mise run publish-rp-wasm-to-release + + # Windows-only augeas build. Augeas has no upstream Windows binaries + # anywhere (not in MSYS2/Cygwin/aqua/conda); the only way to get a + # native gpg-style PE binary is to build it ourselves under MSYS2. + # This job mirrors the rustpython pattern: detect missing asset, build + # from source, tar the install dir, and upload to the augeas-v1 GitHub + # release (which mise's `http:augeas` tool entry points at, per + # platforms.windows-x64). + # + # Source is the edge-toolkit augeas fork's `win` branch rather than the + # upstream 1.14.1 release tarball: the Windows port needs changes across + # many files (vendored glob, _WIN32 guards in augtool/transform, gnulib + # fsync, libtool -no-undefined, MIN/MAX fallbacks, a .gitattributes LF + # fix, etc.) that the released tarball doesn't carry. The fork builds + # green on its own CI across Linux/macOS/Windows, so we build it directly + # instead of maintaining a sprawling patch against the tarball. + augeas: + # Parked. Drop the `if: false` to rebuild + republish (e.g. on a + # version bump or a fork-branch update). Default is no PR run. + if: false + # All five platforms build from the same jayvdb/augeas `win` branch + # source -- one fork, one set of portability patches, identical + # binaries across Linux/macOS/Windows. The fork's own build.yml is + # the recipe template (./autogen.sh --disable-gnulib-tests on Linux/ + # macOS; MSYS2 + mingw-w64 CFLAGS workarounds on Windows). + runs-on: ${{ matrix.runner }} + permissions: + contents: write + timeout-minutes: 60 + env: + AUG_VERSION: "1.14.1" + AUG_RELEASE_TAG: "augeas-v1" + AUG_FORK_REPO: "https://github.com/jayvdb/augeas.git" + AUG_REF: "win" + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + triple: x86_64-unknown-linux-gnu + archive_ext: tar.xz + - runner: ubuntu-24.04-arm + triple: aarch64-unknown-linux-gnu + archive_ext: tar.xz + - runner: macos-latest + triple: aarch64-apple-darwin + archive_ext: tar.xz + - runner: macos-26-intel + triple: x86_64-apple-darwin + archive_ext: tar.xz + - runner: windows-latest + triple: x86_64-pc-windows-mingw + archive_ext: tar.gz + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Detect missing augeas asset + id: aug_check + env: + GITHUB_TOKEN: ${{ github.token }} + MATRIX_TRIPLE: ${{ matrix.triple }} + MATRIX_EXT: ${{ matrix.archive_ext }} + run: | + # No "augeas-" project prefix on the filename -- the release tag + # (`augeas-v1`) already encodes the project. Saves URL length and + # keeps the `-.` shape uniform with the + # dart-typegen + gnupg-w32 assets. + asset="${AUG_VERSION}-${MATRIX_TRIPLE}.${MATRIX_EXT}" + repo=edge-toolkit/core + jq='.assets[].name' + assets=$(gh release view "$AUG_RELEASE_TAG" --repo "$repo" --json assets --jq "$jq" 2>/dev/null || true) + if echo "$assets" | grep -Fxq "$asset"; then + echo "::notice::$asset already in $AUG_RELEASE_TAG; skipping" + echo "work=" >> "$GITHUB_OUTPUT" + else + echo "work=yes" >> "$GITHUB_OUTPUT" + echo "asset=$asset" >> "$GITHUB_OUTPUT" + fi + + # Linux: apt-get the build deps, identical set to the fork's build.yml + # `linux` job. Skipped on the macOS/Windows runners. + - name: Install Linux build deps + if: |- + steps.aug_check.outputs.work == 'yes' + && runner.os == 'Linux' + run: | + sudo apt-get update + pkgs="autoconf automake bison flex libtool-bin libxml2-dev libreadline-dev pkg-config" + sudo apt-get install -y --no-install-recommends $pkgs + + # macOS: brew the build deps, mirroring the fork's build.yml `macos` job. + # CPPFLAGS/LDFLAGS point configure at GNU readline (libedit lacks + # rl_crlf/rl_replace_line/rl_char_is_quoted_p); modern bison required. + - name: Install macOS build deps + if: |- + steps.aug_check.outputs.work == 'yes' + && runner.os == 'macOS' + run: | + brew install autoconf automake libtool readline libxml2 pkg-config bison + + # Windows: MSYS2 toolchain + autotools, mirroring the fork's `windows` + # job. `msystem: MINGW64` targets x86_64-w64-mingw32. + - name: Set up MSYS2 (Windows) + if: |- + steps.aug_check.outputs.work == 'yes' + && runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: |- + base-devel + git + autoconf + automake + libtool + bison + flex + mingw-w64-x86_64-toolchain + mingw-w64-x86_64-libxml2 + mingw-w64-x86_64-readline + mingw-w64-x86_64-pkgconf + + # Unix build (Linux + macOS). Both use the same ./autogen.sh recipe + # with --disable-gnulib-tests (gnulib's bundled test harness fails to + # build against modern toolchains; unrelated to augeas itself). macOS + # needs the homebrew bison/readline overrides; Linux uses system libs. + - name: Build + package augeas (Unix) + if: |- + steps.aug_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + && runner.os != 'Windows' + env: + ASSET: ${{ steps.aug_check.outputs.asset }} + run: | + set -euo pipefail + srcdir="$RUNNER_TEMP/augeas-src" + installdir="$RUNNER_TEMP/install" + mkdir -p "$installdir" + clone_args="--depth 1 --branch $AUG_REF --recurse-submodules" + git clone $clone_args "$AUG_FORK_REPO" "$srcdir" + cd "$srcdir" + if [ "$RUNNER_OS" = "macOS" ]; then + bison_prefix=$(brew --prefix bison) + readline_prefix=$(brew --prefix readline) + libxml2_prefix=$(brew --prefix libxml2) + export PATH="$bison_prefix/bin:$PATH" + export PKG_CONFIG_PATH="$libxml2_prefix/lib/pkgconfig:$readline_prefix/lib/pkgconfig" + export CPPFLAGS="-I$readline_prefix/include" + export LDFLAGS="-L$readline_prefix/lib" + fi + autogen_args="--prefix=$installdir --disable-static --disable-gnulib-tests" + autogen_args="$autogen_args --disable-dependency-tracking" + ./autogen.sh $autogen_args + make -j"$(nproc 2>/dev/null || sysctl -n hw.ncpu)" + make install + # Relocate. libtool bakes the absolute --prefix into binaries + # and dylibs: Linux RUNPATH=/home/runner/work/_temp/install/lib + # and macOS LC_ID_DYLIB/LC_LOAD_DYLIB = + # /Users/runner/work/_temp/install/lib/libaugeas.0.dylib. Both + # break the moment we ship the binary elsewhere (dyld error + # "Library not loaded: /Users/runner/work/_temp/install/lib/ + # libaugeas.0.dylib"). Post-process to relocatable form so the + # tarball is consumable from any directory. + cd "$installdir" + if [ "$RUNNER_OS" = "Linux" ]; then + sudo apt-get install -y --no-install-recommends patchelf + for f in bin/* lib/lib*.so*; do + [ -f "$f" ] && [ ! -L "$f" ] || continue + patchelf --set-rpath '$ORIGIN/../lib' "$f" 2>/dev/null || true + done + else + # macOS: relocate internal lib install_names + cross-refs to + # @rpath, add @loader_path/../lib rpath to binaries, and + # bundle the Homebrew deps (readline + libxml2 + their own + # deps) so consumers don't need brew installed. + our_libs="" + for lib in lib/libaugeas*.dylib lib/libfa*.dylib; do + [ -f "$lib" ] && [ ! -L "$lib" ] || continue + base=$(basename "$lib") + install_name_tool -id "@rpath/$base" "$lib" + our_libs="$our_libs $base" + done + for f in lib/libaugeas*.dylib lib/libfa*.dylib bin/*; do + [ -f "$f" ] && [ ! -L "$f" ] || continue + file "$f" | grep -q "Mach-O" || continue + for old in $(otool -L "$f" 2>/dev/null | awk 'NR>1 {print $1}'); do + base=$(basename "$old") + for ours in $our_libs; do + if [ "$base" = "$ours" ]; then + install_name_tool -change "$old" "@rpath/$base" "$f" + fi + done + done + done + for f in bin/*; do + [ -f "$f" ] && [ ! -L "$f" ] || continue + file "$f" | grep -q "Mach-O" || continue + install_name_tool -add_rpath @loader_path/../lib "$f" 2>/dev/null || true + done + # Bundle Homebrew dylib closure. otool -L on augtool tells us + # which absolute brew paths got baked in; copy each, rewrite + # its install_name + nested refs, and rewrite consumers. + brew_filter='NR>1 && $1 ~ /^\/(usr\/local|opt\/homebrew)\// {print $1}' + brew_deps_otool=$(otool -L bin/augtool lib/libaugeas*.dylib lib/libfa*.dylib 2>/dev/null) + queue=$(echo "$brew_deps_otool" | awk "$brew_filter" | sort -u) + seen="" + while [ -n "$queue" ]; do + next="" + for dep in $queue; do + base=$(basename "$dep") + # space-delimited seen list; membership = substring with sentinels + case " $seen " in *" $base "*) continue ;; esac + seen="$seen $base" + [ -f "$dep" ] || continue + cp -L "$dep" "lib/$base" + chmod +w "lib/$base" + install_name_tool -id "@rpath/$base" "lib/$base" + # add new transitive brew deps to the queue + nested_otool=$(otool -L "lib/$base" 2>/dev/null) + nested_refs=$(echo "$nested_otool" | awk "$brew_filter") + for nested in $nested_refs; do + next="$next $nested" + done + done + queue=$(echo "$next" | tr ' ' '\n' | sort -u | grep -v '^$' || true) + done + # Rewrite every Mach-O file's refs to brew paths -> @rpath/ + for f in lib/*.dylib bin/*; do + [ -f "$f" ] && [ ! -L "$f" ] || continue + file "$f" | grep -q "Mach-O" || continue + f_otool=$(otool -L "$f" 2>/dev/null) + for old in $(echo "$f_otool" | awk "$brew_filter"); do + install_name_tool -change "$old" "@rpath/$(basename "$old")" "$f" + done + done + # Bundled dylibs in lib/ need their own rpath ($ORIGIN + # equivalent) so they can resolve each other. + for lib in lib/*.dylib; do + [ -f "$lib" ] && [ ! -L "$lib" ] || continue + install_name_tool -add_rpath @loader_path "$lib" 2>/dev/null || true + done + fi + # tar.xz: smaller than .gz, mise's http backend handles both. + (cd "$installdir" && tar -cJf "$RUNNER_TEMP/$ASSET" .) + ls -lh "$RUNNER_TEMP/$ASSET" + + # Windows build under MSYS2 -- separate step because shell differs + # (msys2 {0}), tar parses Windows-form paths as `host:path` (workarounds + # via cygpath -m + --force-local), and we bundle mingw runtime DLLs so + # the tarball is self-contained on a vanilla Windows host. + - name: Build + package augeas (Windows) + if: |- + steps.aug_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + && runner.os == 'Windows' + shell: msys2 {0} + env: + ASSET: ${{ steps.aug_check.outputs.asset }} + run: | + set -euo pipefail + # MSYS2 bash interprets `D:\a\_temp` (Windows-form $RUNNER_TEMP) + # as escape sequences -- libtool's `-rpath must be absolute` + # check then rejects the mangled result. cygpath -m gives the + # Windows-form path with forward slashes (e.g. `D:/a/_temp`). + runner_temp_m=$(cygpath -m "$RUNNER_TEMP") + srcdir="$runner_temp_m/augeas-src" + installdir="$runner_temp_m/install" + mkdir -p "$installdir" + clone_args="--depth 1 --branch $AUG_REF --recurse-submodules" + git clone $clone_args "$AUG_FORK_REPO" "$srcdir" + cd "$srcdir" + # CFLAGS, mirroring the fork's build.yml windows job: + # -Wno-implicit-function-declaration: mingw-w64 gcc 14+ promoted + # this from warning to error; some autoconf probes still trip it. + # -Duint=unsigned: augeas headers use the BSD-ism `uint`, which + # mingw-w64's doesn't define unconditionally. + export CFLAGS="-Wno-implicit-function-declaration -Duint=unsigned" + autogen_args="--prefix=$installdir --disable-static --disable-debug" + autogen_args="$autogen_args --disable-dependency-tracking" + ./autogen.sh $autogen_args + make -j"$(nproc)" + make install + # bundle the mingw runtime DLLs augeas links against so the + # tarball is self-contained on a vanilla Windows host. + dlls="libxml2-2.dll libreadline8.dll libncursesw6.dll" + dlls="$dlls libwinpthread-1.dll libgcc_s_seh-1.dll libstdc++-6.dll" + dlls="$dlls libiconv-2.dll libtermcap-0.dll zlib1.dll" + for dll in $dlls; do + src="/mingw64/bin/$dll" + if [ -f "$src" ]; then + cp -v "$src" "$installdir/bin/" + fi + done + # tar parses any colon in the output path as `host:path` + # ("Cannot connect to D: resolve failed"); --force-local + # disables that. cd into installdir keeps the input arg as `.`. + (cd "$installdir" && tar --force-local -czf "$runner_temp_m/$ASSET" .) + ls -lh "$runner_temp_m/$ASSET" + + - name: Publish augeas asset to release + if: |- + steps.aug_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + env: + GITHUB_TOKEN: ${{ github.token }} + ASSET: ${{ steps.aug_check.outputs.asset }} + run: | + gh release upload "$AUG_RELEASE_TAG" --repo edge-toolkit/core --clobber "$RUNNER_TEMP/$ASSET" + + # Windows-only dart-typegen cargo build. `cargo:dart-typegen` is in + # `MISE_DISABLE_TOOLS` on the Windows test/build lanes because its + # cargo source-build against the gnullvm rust host trips + # `error[E0463]: can't find crate for 'core'`. Solution: pre-build it + # under the runner's stable-msvc toolchain (already installed on the + # windows-latest GHA image) and publish the binary so consumers + # `http:` install instead of source-compiling. Same shape as the + # augeas job above. + dart-typegen: + # Parked. Drop the `if: false` to rebuild + republish (e.g. on a + # version bump). Default is no PR run. + if: false + runs-on: windows-latest + permissions: + contents: write + timeout-minutes: 30 + env: + DTG_VERSION: "0.1.13" + DTG_RELEASE_TAG: "dart-typegen-v1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Detect missing dart-typegen asset + id: dtg_check + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + triple=x86_64-pc-windows-msvc + # No "dart-typegen-" project prefix; release tag already names it. + asset="${DTG_VERSION}-${triple}.tar.gz" + repo=edge-toolkit/core + jq='.assets[].name' + assets=$(gh release view "$DTG_RELEASE_TAG" --repo "$repo" --json assets --jq "$jq" 2>/dev/null || true) + if echo "$assets" | grep -Fxq "$asset"; then + echo "::notice::$asset already in $DTG_RELEASE_TAG; skipping" + echo "work=" >> "$GITHUB_OUTPUT" + else + echo "work=yes" >> "$GITHUB_OUTPUT" + echo "triple=$triple" >> "$GITHUB_OUTPUT" + echo "asset=$asset" >> "$GITHUB_OUTPUT" + fi + + # The GHA windows-latest image ships rustup with the + # stable-x86_64-pc-windows-msvc toolchain pre-installed; no + # explicit setup-rust step is needed. + - name: Build dart-typegen via cargo install + if: |- + steps.dtg_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + env: + ASSET: ${{ steps.dtg_check.outputs.asset }} + run: | + set -euo pipefail + installdir="$RUNNER_TEMP/install" + mkdir -p "$installdir" + # Force the msvc target -- the runner default-host is msvc, + # but a stray default-toolchain (e.g. on a custom runner) + # could otherwise pick gnullvm and trip the same build failure + # we're working around. + cargo_args="--version $DTG_VERSION --root $installdir" + cargo_args="$cargo_args --target x86_64-pc-windows-msvc --locked" + cargo install dart-typegen $cargo_args + ls "$installdir/bin" + # `cd` into the dir + tar a relative path -- Git Bash's bundled + # tar parses `D:\a\_temp/...` as `host:path` (the colon) and + # then errors out with `Cannot connect to D: resolve failed`. + # Relative paths avoid the colon entirely. + (cd "$installdir/bin" && tar czf "../../$ASSET" dart-typegen.exe) + ls -lh "$RUNNER_TEMP/$ASSET" + + # Release tag must already exist -- run `MISE_ENV=maint mise run + # bootstrap-dart-typegen-release` locally once before the first + # dispatch. The bootstrap task is idempotent. + - name: Publish dart-typegen asset to release + if: |- + steps.dtg_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + env: + GITHUB_TOKEN: ${{ github.token }} + ASSET: ${{ steps.dtg_check.outputs.asset }} + run: | + gh release upload "$DTG_RELEASE_TAG" --repo edge-toolkit/core --clobber "$RUNNER_TEMP/$ASSET" + + # Re-package GnuPG's "Simple installer for Windows" (an NSIS-wrapped + # `.exe`) as a flat tarball so mise's http: backend can extract it + # natively. The upstream installer at + # https://gnupg.org/ftp/gcrypt/binary/gnupg-w32-.exe is a 5.5 MB + # NSIS installer; mise's built-in extraction (`sevenz_rust2`) handles + # canonical .7z but not the NSIS wrapper, so we extract once in CI + # via the `7z` binary (p7zip-full, preinstalled on ubuntu-latest) + # and host the resulting tarball. Output is a flat `bin/`+`share/`+ + # `lib/`+`include/` layout -- a self-contained native-PE gpg/gpg-agent/ + # dirmngr/gpgsm suite with no `Library/usr/bin/` nesting, no msys2- + # runtime autoload trap (every binary is PE32+ x86-64, not Cygwin). + # Runs on ubuntu-latest because 7zz extraction is fully cross-platform + # and ubuntu jobs are cheaper than windows ones. + gnupg-w32: + # Parked. Drop the `if: false` to rebuild + republish (e.g. on a + # version bump). Default is no PR run. + if: false + runs-on: ubuntu-latest + permissions: + contents: write + timeout-minutes: 15 + env: + # Bump in lockstep with upstream's "Simple installer" page at + # https://gnupg.org/download/ -- the .exe filename embeds both + # the version and a build date stamp. Keep the date stamp here + # to pin to the exact upstream build (a re-release at the same + # version would otherwise silently drift our cached asset). + GNUPG_VERSION: "2.5.20" + GNUPG_BUILD_STAMP: "20260513" + GNUPG_RELEASE_TAG: "gnupg-w32-v1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Detect missing gnupg-w32 asset + id: gpg_check + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # No "gnupg-w32-" project prefix; release tag already names it. + asset="${GNUPG_VERSION}_${GNUPG_BUILD_STAMP}-x86_64-pc-windows.tar.gz" + repo=edge-toolkit/core + jq='.assets[].name' + assets=$(gh release view "$GNUPG_RELEASE_TAG" --repo "$repo" --json assets --jq "$jq" 2>/dev/null || true) + if echo "$assets" | grep -Fxq "$asset"; then + echo "::notice::$asset already in $GNUPG_RELEASE_TAG; skipping" + echo "work=" >> "$GITHUB_OUTPUT" + else + echo "work=yes" >> "$GITHUB_OUTPUT" + echo "asset=$asset" >> "$GITHUB_OUTPUT" + fi + + # No mise needed -- curl, 7z (p7zip-full), tar, and gh are all + # preinstalled on ubuntu-latest. We tried `MISE_ENV=maint mise + # exec -- xh` first (per the "use mise tools" preference) but + # mise 2026.6.5 on the runner failed to resolve xh from its aqua + # install dir layout (`/xh-v0.25.3-/xh`); the + # newer 2026.6.10 mise locally handles it, but pinning mise just + # for this job is more setup than the tool savings warrant. + + - name: Extract + repackage upstream installer + if: |- + steps.gpg_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + env: + ASSET: ${{ steps.gpg_check.outputs.asset }} + run: | + set -euo pipefail + work="$RUNNER_TEMP/gnupg-w32" + mkdir -p "$work" "$work/extracted" + cd "$work" + installer="gnupg-w32-${GNUPG_VERSION}_${GNUPG_BUILD_STAMP}.exe" + url="https://gnupg.org/ftp/gcrypt/binary/${installer}" + curl -sLO "$url" + 7z x -y -o"extracted" "$installer" + # Strip NSIS install-staging cruft: `$PLUGINSDIR/` is NSIS's + # own internal use, and the `*.tmp` files in bin/ are NSIS's + # mid-install copies of binaries (cleaned up by a working + # installer run, redundant in our extracted form). + rm -rf "extracted/\$PLUGINSDIR" + find extracted/bin -name '*.tmp' -delete + ls extracted/bin + # Flatten into tarball: bin/ + lib/ + share/ + include/ keep + # the install layout consumers expect; mise's http backend + # respects strip_components so we set the tar root to that. + tar czf "$RUNNER_TEMP/$ASSET" -C extracted . + ls -lh "$RUNNER_TEMP/$ASSET" + sha256sum "$RUNNER_TEMP/$ASSET" + + # Release tag must already exist -- run `MISE_ENV=maint mise run + # bootstrap-gnupg-w32-release` locally once before the first + # dispatch. The bootstrap task is idempotent. + - name: Publish gnupg-w32 asset to release + if: |- + steps.gpg_check.outputs.work == 'yes' + && github.event.pull_request.head.repo.fork != true + env: + GITHUB_TOKEN: ${{ github.token }} + ASSET: ${{ steps.gpg_check.outputs.asset }} + run: | + gh release upload "$GNUPG_RELEASE_TAG" --repo edge-toolkit/core --clobber "$RUNNER_TEMP/$ASSET" diff --git a/.gitignore b/.gitignore index 9284041..c94bd3d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ target/ .DS_Store services/ws-wasm-agent/pkg/ +services/ws-modules/pywasm1/pkg/ services/ws-server/static/models/ .zig-cache/ zig-out/ diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 5934e59..958c442 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -7,13 +7,26 @@ dart_bucket = "https://storage.googleapis.com/storage/v1/b/dart-archive/o" dart_release = "https://storage.googleapis.com/dart-archive/channels/stable/release" [tools] -# cargo: backend -- dart-typegen has no prebuilt binary (crates.io only). -"cargo:dart-typegen" = "latest" +# cargo: backend -- dart-typegen has no prebuilt binary on crates.io. +# Windows uses the pre-built `http:dart-typegen` (config.windows.toml) +# instead, so this entry is os-scoped to non-Windows. On Windows the +# upstream cargo: source-build trips +# `error[E0463]: can't find crate for 'core'` against the gnullvm rust +# host (see upstream-cache.yaml's dart-typegen job for the producer). +"cargo:dart-typegen" = { version = "latest", os = ["linux", "macos"] } # dart isn't an aqua/registry tool — install it straight from Google's dart # archive, discovering the latest stable from the bucket listing. `{{ version }}` # etc. resolve in the tool-url context; the static host/path prefix is a var. +# +# `strip_components = 1` makes mise drop the `dart-sdk/` wrapper every dartsdk +# zip uses (Windows/macOS/Linux ship the same `dart-sdk/` shape). mise +# normally auto-detects single-top-level-dir archives and strips, but that +# auto-detect doesn't fire reliably on Windows ZIPs, leaving the binary at +# `/dart-sdk/bin/dart.exe` instead of the `/bin/dart.exe` +# mise's PATH activation expects. Explicit beats auto. [tools.dart] +strip_components = 1 url = "{{ vars.dart_release }}/{{ version }}/sdk/dartsdk-{{ os() }}-{{ arch() }}-release.zip" version = "latest" version_expr = ''' diff --git a/.mise/config.dotnet.toml b/.mise/config.dotnet.toml index 93dec23..5fe9ce0 100644 --- a/.mise/config.dotnet.toml +++ b/.mise/config.dotnet.toml @@ -27,44 +27,81 @@ description = "Format .NET sources" description = "Build the dotnet-data1 C# WASM workflow module" dir = "services/ws-modules/dotnet-data1" run = ''' +# Windows lane is skipped: MSBuild's BrowserWasmApp.targets link step's +# `` does NOT inherit the bash-level PATH +# prepend (validated empirically -- the emcc-debug step in test.yaml +# shows `cmd.exe %PATH%` sees the emscripten dir after the prepend, yet +# the link still trips `'emcc' is not recognized` / MSB3073 exit 9009), +# and `-p:EmscriptenLocation=...\Sdk` fixes the compile target but not +# the link. Neither lever the .NET WebAssembly SDK exposes lets us +# route bare-`emcc` in the link to the workload-installed pack. +# tests/modules.rs's `dotnet_data1_pkg_built()` matches this skip so the +# downstream test on Windows logs `skipping ...: pkg/ not built` instead +# of failing. Drop this guard once .NET's SDK either honors PATH or +# documents a property that overrides the link command. +if [ "${OS:-}" = "Windows_NT" ]; then + echo "build-ws-dotnet-data1-module: skipped on Windows (MSBuild link ignores our PATH; emcc unresolvable)" + exit 0 +fi + +# `set -x` so every command (incl. the find / dirname chain that resolves +# $(EmscriptenLocation), the final dotnet publish invocation with its +# -p:EmscriptenLocation=... arg, and the cp of the publish artefacts) is +# echoed to the log. The Windows path of this task is fiddly and the CI +# log is the only way we get to see it; leave the trace on until the +# Windows build is reliably green. +set -x + dotnet workload install wasm-tools --skip-manifest-update # Windows-only: `dotnet workload install` drops the Emscripten SDK pack under # %LOCALAPPDATA%/mise/dotnet-root/packs/Microsoft.NET.Runtime.Emscripten.*/*/tools/emscripten/, -# but does not add that dir to PATH (Linux/macOS get it via dotnet's -# workload-install shell-rc setup). `dotnet publish`'s MSBuild targets then -# shell out to bare `emcc` and cmd reports verbatim: -# 'emcc' is not recognized as an internal or external command, -# operable program or batch file. -# Discover the pack dir and prepend to PATH so the publish subprocess finds -# emcc.bat. Glob via `find` so a future Emscripten / pack version bump -# rolls forward without touching this task. +# but does not surface that path to the build the way Linux/macOS do via +# dotnet's workload-install shell-rc setup. MSBuild's BrowserWasmApp.targets: +# - Compile target uses `$(EmscriptenLocation)\..\tools\emscripten\emcc` +# (absolute path; works as long as we set the property). +# - LINK target's `` calls bare `emcc @rsp` and relies on PATH, +# which we have to prepend in bash AND keep propagating through +# `dotnet publish` -> MSBuild -> cmd.exe. Without that the link errors: +# 'emcc' is not recognized as an internal or external command, +# operable program or batch file. +# error MSB3073: The command "emcc "@...emcc-default.rsp" ..." exited with code 9009. +# Belt-and-braces: do BOTH the `-p:EmscriptenLocation=...\Sdk` property +# (compile) AND the PATH prepend (link). The PATH prepend uses MSYS Unix +# form (`/c/Users/...`); when bash spawns dotnet.exe MSYS converts the +# whole `PATH` (`:`->`;`, `/c/Users/...` -> `C:\Users\...`) -- a +# `C:\Users\...` entry would otherwise escape the conversion and the +# trailing `:` would glue it to the next entry, leaving cmd.exe (run by +# MSBuild's ) without the emscripten dir. Glob via `find` so an +# Emscripten / pack version bump rolls forward. echo "[emcc-find] OS=${OS:-} MSYSTEM=${MSYSTEM:-}" +publish_args="" if [ "${OS:-}" = "Windows_NT" ]; then echo "[emcc-find] LOCALAPPDATA=${LOCALAPPDATA:-}" emcc_bat="$(find "$LOCALAPPDATA/mise/dotnet-root/packs" -name 'emcc.bat' 2>/dev/null | head -1)" echo "[emcc-find] emcc_bat=${emcc_bat:-}" - if [ -n "$emcc_bat" ]; then - emcc_dir="$(dirname "$emcc_bat")" - # `cygpath -w` (Git Bash) converts /c/Users/... to C:\Users\... so the - # prepended PATH entry resolves in cmd.exe when MSBuild's shells - # out to bare `emcc`. Without conversion, Git Bash hands Windows - # processes a PATH where this dir may not be recognised in the form - # cmd's resolver expects. - if command -v cygpath >/dev/null 2>&1; then - emcc_dir_win="$(cygpath -w "$emcc_dir")" - echo "[emcc-find] cygpath conversion: $emcc_dir -> $emcc_dir_win" - emcc_dir="$emcc_dir_win" - fi - export PATH="$emcc_dir:$PATH" - echo "[emcc-find] prepended to PATH: $emcc_dir" - else + if [ -z "$emcc_bat" ]; then echo "build-ws-dotnet-data1-module: emcc.bat not found under \$LOCALAPPDATA/mise/dotnet-root/packs" >&2 exit 1 fi + emcc_dir="$(dirname "$emcc_bat")" + # `$(EmscriptenLocation)` is the sibling `Sdk\` dir of the `tools\` dir + # that holds emcc.bat. emcc_bat sits at `/tools/emscripten/emcc.bat`, + # so two dirnames up + `/Sdk` is it. MSBuild wants Windows form. + emsdk_loc="$(dirname "$(dirname "$emcc_dir")")/Sdk" + if command -v cygpath >/dev/null 2>&1; then + emsdk_loc="$(cygpath -w "$emsdk_loc")" + emcc_dir="$(cygpath -u "$emcc_dir")" + fi + export PATH="$emcc_dir:$PATH" + publish_args="-p:EmscriptenLocation=$emsdk_loc" + echo "[emcc-find] prepended to PATH: $emcc_dir" + echo "[emcc-find] $publish_args" fi -dotnet publish -c Release +# $publish_args holds zero or one CLI flag; word-splitting is intentional. +# shellcheck disable=SC2086 +dotnet publish -c Release $publish_args PUBLISH=bin/Release/net10.0/publish/wwwroot/_framework cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/ diff --git a/.mise/config.java.toml b/.mise/config.java.toml index d1d66d3..02a3a65 100644 --- a/.mise/config.java.toml +++ b/.mise/config.java.toml @@ -1,26 +1,86 @@ # Java toolchain + tasks. Loaded when MISE_ENV includes `java`. [tools] -java = "latest" -maven = "latest" +# Pinned (not "latest") because the JAVA_HOME path below + config.windows.toml's +# maven_bin _.path entry both template install-dir segments with version. Bump +# in lockstep, AND on bump verify the Windows OpenJDK wrapper name in the +# java_home var below (build-number suffix changes with each GA release). +java = "26.0.1" +maven = "3.9.16" + +[vars] +# JDK install root for JAVA_HOME (see [env] below). mise's java plugin +# extracts the OpenJDK archive flat on every platform (Linux/macOS tarball +# and Windows zip alike -- `bin/java[.exe]` sits directly under the install +# dir; no `jdk-X.Y.Z+N\` wrapper). Default base directory is mise's +# per-platform data dir (`~/.local/share/mise` on Unix, `%LOCALAPPDATA%\mise` +# on Windows); MISE_DATA_DIR overrides on either. +# +# Windows-only slash normalization: %LOCALAPPDATA% arrives in mise's tera +# context with forward slashes (`C:/Users/.../AppData/Local`) but our +# literal segments use backslashes; the resulting mixed-slash path passes +# busybox `ls` but mvn.cmd's `if not exist "%JAVA_HOME%\bin\java.exe"` +# rejects it. The `| replace(from="/", to=`\`)` pass on the prefix +# normalizes everything to a single backslash. Note the backtick-quoted +# string for the replacement: tera 1.20's pest grammar treats strings +# verbatim (no `\\`-as-`\` escape), so backticks + a single literal +# backslash are the way to express a one-char `\` argument. +java_home = '''{% if os() == "windows" -%} +{%- set win_base = get_env(name="LOCALAPPDATA", default="") ~ "\mise" -%} +{{- get_env(name="MISE_DATA_DIR", default=win_base) | replace(from="/", to=`\`) -}} +\installs\java\26.0.1 +{%- else -%} +{{- get_env(name="MISE_DATA_DIR", default=env.HOME ~ "/.local/share/mise") -}} +/installs/java/26.0.1 +{%- endif %}''' [env] +# mvn.cmd's first action is `if "%JAVA_HOME%" == "" goto :error` -- it +# REQUIRES JAVA_HOME in the env (PATH-discovered java.exe is not enough). +# Linux/macOS mvn shells are more relaxed but still benefit from a pinned +# JAVA_HOME (otherwise java-toolchains.xml lookups vary by host). +JAVA_HOME = "{{ vars.java_home }}" # Silence Maven's per-artifact download/upload progress lines (the # "Downloading from central:" / "Downloaded" spam). Build output and real # errors still print. MAVEN_ARGS is honored by Maven 3.9+ and prepends to # every `mvn` invocation. MAVEN_ARGS = "--no-transfer-progress" +# Cross-platform mvn invocation. On Windows we must NOT let busybox bash +# resolve `mvn`: the maven distribution ships both files side-by-side in +# bin/ (`mvn` is a Unix shell script; `mvn.cmd` is the Windows wrapper), +# and busybox-w32's PATH lookup picks the bare `mvn` first via +# `file_is_executable`, parses its `#!/bin/sh` shebang, and runs it +# through busybox's own sh applet -- but the Unix script's JAVACMD +# composition (`JAVACMD="$JAVA_HOME/bin/java"`, no `.exe`) then fails +# its `[ -x "$JAVACMD" ]` test because on Windows the file is named +# `bin/java.exe`. Bare `mvn.cmd` (and `cmd.exe /c mvn`) both fail in +# the all-langs MISE_ENV with "'mvn[.cmd]' is not recognized" -- the +# wrapping cmd.exe cannot see maven\bin in its inherited PATH (likely +# a length/encoding issue introduced by the wider tool set; works fine +# with MISE_ENV=java alone). Use the absolute path to `mvn.cmd` +# instead: busybox-w32's spawnve detects `.cmd` and auto-wraps with +# cmd.exe, which runs the full path directly with no PATH lookup +# required. `vars.maven_bin` is defined in config.windows.toml; vars +# merge across all loaded configs. +MVN = '''{% if os() == "windows" -%} +{{- vars.maven_bin -}}\mvn.cmd +{%- else -%} +mvn +{%- endif %}''' [tasks.build-ws-java-data1-module] description = "Build the java-data1 workflow module" -run = "mvn package" +run = "$MVN package" +shell = "bash -euo pipefail -c" # Namespaced aggregator picked up by the default config's globbed `check`. The # compile triggers maven-compiler-plugin with -Xlint:all -Werror + Error Prone. [tasks."check:java"] description = "Run Java checks (javac -Xlint:all -Werror, Error Prone)" -run = "mvn -q compile" +run = "$MVN -q compile" +shell = "bash -euo pipefail -c" [tasks."prefetch:java"] description = "Prefetch Java (Maven) dependencies" -run = "mvn dependency:resolve --quiet" +run = "$MVN dependency:resolve --quiet" +shell = "bash -euo pipefail -c" diff --git a/.mise/config.js.toml b/.mise/config.js.toml new file mode 100644 index 0000000..f00986f --- /dev/null +++ b/.mise/config.js.toml @@ -0,0 +1,132 @@ +# JavaScript / npm env (MISE_ENV=js). +# +# Tools that resolve through mise's `npm:` backend live here so the modules +# that consume them (har1, face-detection — both have `onnxruntime-web` as a +# Cargo dep and load the JS package at runtime via /modules/onnxruntime-web) +# only build when the `js` env is loaded. The aqua-backed `pnpm` row stays +# in the default config; the `npm:pnpm` row below is the macos/x64 fallback +# (no aqua darwin/amd64 prebuilt). oxlint + oxfmt also live here -- they +# only know JS/TS (the framework-file support is just the `