diff --git a/.cargo/config.toml b/.cargo/config.toml index 69ee514ab6a..5a1b17391b2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,6 @@ [alias] +xtask = "run --package xtask --" +build-test-shader = "xtask test-build" compiletest = "run --release -p compiletests --" difftest = "run --release -p difftests --" run-wasm = ["run", "--release", "-p", "run-wasm", "--"] diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 83b0f02b7c7..28f6a2c9917 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Vulkan SDK uses: jakoch/install-vulkan-sdk-action@v1 with: @@ -55,7 +55,7 @@ jobs: run: cargo test -p rustc_codegen_spirv --release --no-default-features --features "use-installed-tools" - name: workspace test (excluding examples) - run: cargo test --release --workspace --exclude "example-runner-*" --no-default-features --features "use-installed-tools,clap" + run: cargo test --release --workspace --exclude "example-runner-*" --exclude "cargo-gpu*" --no-default-features --features "use-installed-tools,clap" # Examples - name: cargo check examples @@ -84,7 +84,7 @@ jobs: target: [ aarch64-linux-android ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Vulkan SDK uses: jakoch/install-vulkan-sdk-action@v1 with: @@ -128,7 +128,7 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Vulkan SDK uses: jakoch/install-vulkan-sdk-action@v1 with: @@ -151,7 +151,7 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install Vulkan SDK uses: jakoch/install-vulkan-sdk-action@v1 with: @@ -180,11 +180,115 @@ jobs: - name: difftests run: cargo run -p difftests --release --no-default-features --features "use-installed-tools" + cargo-gpu-os: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + runs-on: ${{ matrix.os }} + env: + RUST_LOG: debug + steps: + - uses: actions/checkout@v6 + - uses: taiki-e/install-action@v2 + with: + tool: nextest + - run: cargo fetch --locked + - name: shader-crate-template cargo fetch --locked + run: | + cd ./crates/shader-crate-template + cargo fetch --locked + - run: cargo nextest run -p cargo-gpu -p cargo-gpu-install + - name: Run a full build + run: cargo xtask test-build + + cargo-gpu-backwards-compat: + # The dependency is technically unnecessary, this is just to delay the launch of this job, so that github prefers + # launching long-running jobs first. Currently, these jobs take only about 7min compared to 22min of others. + needs: [cargo-deny] + strategy: + fail-fast: false + matrix: + include: + # As well as testing on each OS, we also want to test to make sure we're still supporting + # older versions of `rust-gpu`. However, we can assume that these tests are already okay + # across platforms, so we only need to test on Linux, the chepeast in terms of minutes. + # + # `0.7.0` currently fails building `spirv-builder-cli` with: + # """ + # package `is_terminal_polyfill v1.70.1` cannot be built because it requires rustc + # 1.70.0 or newer, while the currently active rustc version is 1.69.0-nightly + # """ + # It's probably easily fixable. But also `0.7.0` was released in April 2023, so there's + # unlikely many users of it? + + # 0.8.0 started failing as well due to `zmij v1.0.20` requiring rustc 1.71 or newer + # - rust-gpu-version: 0.8.0 + # glam-version: 0.24.2 + - rust-gpu-version: 0.9.0 + glam-version: 0.24.2 + + # target spec introduction + # last version before + # * fails: compiler too old, `serde` using `#[diagnostic]` + # - rust-gpu-version: cc752312c3de6813a41189e46476d5c1be5e0bbe + # glam-version: 0.30.7 + # first version requiring target specs + # * fails: target spec mismatch! + # * resolution: Since this is just a few commits, I'd be fine ignoring it. + # - rust-gpu-version: 02cefd101014f66b79dffb20a2c2b5b7c9038401 + # glam-version: 0.30.7 + # target specs change again just a few commits later + # * fails: compiler too old, `proc-macro2` using `Literal::byte_character`. + # * resolution: want to support, can't be bothered to hack in old proc-macro2 versions + # - rust-gpu-version: bbb61f58b3d24f3f64745050eb214b90bf6dcce9 + # glam-version: 0.30.7 + + # testing rustc 1.5 months later + - rust-gpu-version: eea8998df9dc2fd8e7a65c5b5b7ae20c238a665a + glam-version: 0.29.3 + + # just after target specs v2 refactor, we updated to rustc 1.85 and needed to change them again + # before + - rust-gpu-version: a547c6e45266d613d9fec673e869d7a96181e47b + glam-version: =0.30.7 + # after + - rust-gpu-version: 2326b87fe1542eeb898065e36ac949307b55386d + glam-version: =0.30.7 + + # glam semver breakage due to vector type refactor + # before, glam was fixed to <=0.30.7 in this commit + - rust-gpu-version: f79c4181a5dc2d37303947b113f190930c6c1ce6 + glam-version: =0.30.7 + # after, glam >0.30.8 + - rust-gpu-version: e767f24f2565baf1a71bbaf84d453d181cab2417 + + # rustc 1.94.0 destabilised json target specs, requiring `-Ztarget-spec-json` + # see https://github.com/Rust-GPU/rust-gpu/pull/545 + # see https://github.com/rust-lang/rust/pull/150151 + # before + - rust-gpu-version: 30896871ba00e668029ccb724f1438202b284708 + # after + - rust-gpu-version: 877bd8697a15f3e6d09446a5e1807e6237ca1dac + runs-on: ubuntu-latest + env: + RUST_LOG: debug + steps: + - uses: actions/checkout@v6 + - if: ${{ matrix.glam-version }} + name: set glam version + run: cargo xtask set-dependency glam ${{ matrix.glam-version }} + - name: Run a full build + run: cargo xtask test-build --rust-gpu-version ${{ matrix.rust-gpu-version }} + # This allows us to have a single job we can branch protect on, rather than needing # to update the branch protection rules when the test matrix changes test_success: runs-on: ubuntu-latest - needs: [test, compiletest, difftest, android, lint, cargo-deny] + needs: [test, compiletest, difftest, android, lint, cargo-deny, cargo-gpu-os, cargo-gpu-backwards-compat] # Hack for buggy GitHub Actions behavior with skipped checks: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks if: ${{ always() }} steps: @@ -197,6 +301,8 @@ jobs: [[ "${{ needs.android.result }}" == "success" ]] || exit 1 [[ "${{ needs.lint.result }}" == "success" ]] || exit 1 [[ "${{ needs.cargo-deny.result }}" == "success" ]] || exit 1 + [[ "${{ needs.cargo-gpu-os.result }}" == "success" ]] || exit 1 + [[ "${{ needs.cargo-gpu-backwards-compat.result }}" == "success" ]] || exit 1 lint: name: Lint @@ -205,7 +311,7 @@ jobs: # Note that we are explicitly NOT checking out submodules, to validate # that we haven't accidentally enabled spirv-tools native compilation # and regressed CI times - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "false" - name: Install native dependencies @@ -244,7 +350,7 @@ jobs: cargo-deny: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: EmbarkStudios/cargo-deny-action@v2 defaults: diff --git a/Cargo.lock b/Cargo.lock index e0b1ed7a7bc..64ee62a4795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,6 +347,42 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cargo-gpu" +version = "0.9.0" +dependencies = [ + "anyhow", + "cargo-gpu-install", + "cargo_metadata", + "clap", + "dunce", + "env_logger", + "log", + "relative-path", + "semver", + "serde", + "serde_json", + "test-log", +] + +[[package]] +name = "cargo-gpu-install" +version = "0.9.0" +dependencies = [ + "anyhow", + "cargo-util-schemas", + "cargo_metadata", + "clap", + "crossterm", + "directories", + "expect-test", + "log", + "serde", + "spirv-builder", + "tempfile", + "test-log", +] + [[package]] name = "cargo-platform" version = "0.2.0" @@ -393,6 +429,7 @@ dependencies = [ "camino", "cargo-platform", "cargo-util-schemas", + "derive_builder", "semver", "serde", "serde_json", @@ -704,6 +741,32 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.10.0", + "crossterm_winapi", + "document-features", + "mio", + "parking_lot", + "rustix 1.1.3", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -716,6 +779,72 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -775,6 +904,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -785,6 +923,18 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.5.2", + "windows-sys 0.61.2", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -792,7 +942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.6", "winapi", ] @@ -813,6 +963,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dissimilar" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" + [[package]] name = "dlib" version = "0.5.2" @@ -843,6 +999,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" @@ -964,6 +1126,16 @@ dependencies = [ "spirv-builder", ] +[[package]] +name = "expect-test" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" +dependencies = [ + "dissimilar", + "once_cell", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1471,6 +1643,12 @@ dependencies = [ "rayon", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -2312,6 +2490,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "orbclient" version = "0.3.50" @@ -2647,6 +2831,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "reduce" version = "0.0.0" @@ -2677,12 +2872,27 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +[[package]] +name = "relative-path" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] + [[package]] name = "renderdoc-sys" version = "1.1.0" @@ -3006,6 +3216,37 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -3292,6 +3533,28 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test-log" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d53ac171c92a39e4769491c4b4dde7022c60042254b5fc044ae409d34a24d4" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be35209fd0781c5401458ab66e4f98accf63553e8fae7425503e92fdd319783b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tester" version = "0.9.1" @@ -3428,10 +3691,12 @@ version = "0.9.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" dependencies = [ + "indexmap 2.13.0", "serde_core", "serde_spanned 1.0.4", "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", + "toml_writer", "winnow", ] @@ -3494,6 +3759,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tracing" version = "0.1.44" @@ -4655,6 +4926,19 @@ version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "regex-lite", + "tempfile", + "toml 0.9.11+spec-1.1.0", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 1eb4422bb04..88a788506d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,9 @@ members = [ "crates/spirv-std", "crates/spirv-std/shared", "crates/spirv-std/macros", + "crates/cargo-gpu", + "crates/cargo-gpu-install", + "crates/xtask", "tests/compiletests", "tests/compiletests/deps-helper", @@ -31,12 +34,21 @@ members = [ "tests/difftests/lib", ] +exclude = [ + # This currently needs to be excluded because it depends on a version of `rust-gpu` that + # uses a toolchain whose Cargo version doesn't recognise version 4 of `Cargo.lock`. + "crates/shader-crate-template", + # Testing infra may copy the `shader-crate-template` into subdirs of target + "target", +] + [workspace.package] version = "0.9.0" authors = ["rust-gpu developers", "Embark "] edition = "2024" license = "MIT OR Apache-2.0" repository = "https://github.com/rust-gpu/rust-gpu" +keywords = ["gpu", "compiler", "rust-gpu"] [workspace.lints.rust] unexpected_cfgs = { level = "allow", check-cfg = ['cfg(target_arch, values("spirv"))'] } @@ -49,6 +61,27 @@ spirv-std-macros = { path = "./crates/spirv-std/macros", version = "=0.9.0" } spirv-tools = { version = "0.13.0", default-features = false } rustc_codegen_spirv = { path = "./crates/rustc_codegen_spirv", version = "=0.9.0", default-features = false } rustc_codegen_spirv-types = { path = "./crates/rustc_codegen_spirv-types", version = "=0.9.0" } +cargo-gpu-install = { path = "./crates/cargo-gpu-install" } + +# normal dependencies +anyhow = "1.0.98" +clap = { version = "4.5.41", features = ["derive"] } +crossterm = { version = "0.29.0", default-features = false, features = ["events", "windows"] } +directories = "6.0.0" +env_logger = "0.11.8" +log = "0.4" +relative-path = "2.0.1" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.141" +toml = "0.9.2" +tempfile = "3.22" +test-log = "0.2.18" +cargo_metadata = "0.21.0" +cargo-util-schemas = "0.8.2" +semver = "1.0.26" +dunce = "1.0.5" +expect-test = "1.5.1" +regex-lite = "0.1.9" # difftest libraries mirrored from difftest workspace difftest = { path = "tests/difftests/lib" } diff --git a/crates/cargo-gpu-install/Cargo.toml b/crates/cargo-gpu-install/Cargo.toml new file mode 100644 index 00000000000..233196acce6 --- /dev/null +++ b/crates/cargo-gpu-install/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "cargo-gpu-install" +version.workspace = true +edition.workspace = true +description = "Install rust-gpu and it's required toolchain automatically" +repository.workspace = true +readme.workspace = true +keywords.workspace = true +license.workspace = true + +[features] +clap = ["dep:clap", "spirv-builder/clap"] +watch = ["spirv-builder/watch"] +test = ["dep:tempfile"] +tty = ["dep:crossterm"] + +[dependencies] +cargo_metadata.workspace = true +anyhow.workspace = true +spirv-builder.workspace = true +clap = { workspace = true, optional = true } +directories.workspace = true +log.workspace = true +serde.workspace = true +crossterm = { workspace = true, optional = true } +tempfile = { workspace = true, optional = true } + +[dev-dependencies] +test-log.workspace = true +cargo_metadata = { workspace = true, features = ["builder"] } +cargo-util-schemas = "0.8.2" +expect-test = "1.5.1" + +[lints] +workspace = true diff --git a/crates/cargo-gpu-install/README.md b/crates/cargo-gpu-install/README.md new file mode 100644 index 00000000000..5068029c292 --- /dev/null +++ b/crates/cargo-gpu-install/README.md @@ -0,0 +1,43 @@ +# cargo-gpu-install + +`cargo-gpu-install` is the install action of `cargo-gpu`, separated into its own crate. It's intended to be used +by build scripts and other binaries that need automated installation of rust-gpu and it's required toolchain, +without having to pull all the other cli dependencies of the full `cargo-gpu` (like clap). + +## Example + +This is an example build script meant to be placed in your "main" std crate, to build a secondary no-std "shader" crate. +But you can just as well use this in your executable directly, with some minor adjustments. +```rust,no_run +# use std::path::PathBuf; +# use cargo_gpu_install::install::Install; +# use cargo_gpu_install::spirv_builder::SpirvMetadata; + +pub fn main() -> Result<(), Box> { + // path to your shader crate + let shader_crate = PathBuf::from("./shaders"); + + // install the toolchain and build the `rustc_codegen_spirv` codegen backend with it + let backend = Install::from_shader_crate(shader_crate.clone()).run()?; + + // build the shader crate + let mut builder = backend.to_spirv_builder(shader_crate, "spirv-unknown-vulkan1.2"); + // set to true when you're in a build script, false when outside one + builder.build_script.defaults = true; + // enable debug information in the shaders + builder.spirv_metadata = SpirvMetadata::Full; + let spv_result = builder.build()?; + let path_to_spv = spv_result.module.unwrap_single(); + + // emit path to the artifact into env var, use it anywhere in your crate like: + // > include_str!(env!("MY_SHADER_PATH")) + println!( + "cargo::rustc-env=MY_SHADER_PATH={}", + path_to_spv.display() + ); + + // you could also generate some rust source code into the `std::env::var("OUT_DIR")` dir + // and use `include!(concat!(env!("OUT_DIR"), "/shader_symbols.rs"));` to include it + Ok(()) +} +``` diff --git a/crates/cargo-gpu-install/src/install.rs b/crates/cargo-gpu-install/src/install.rs new file mode 100644 index 00000000000..c5061d3e6e8 --- /dev/null +++ b/crates/cargo-gpu-install/src/install.rs @@ -0,0 +1,372 @@ +//! Install a dedicated per-shader crate that has the `rust-gpu` compiler in it. + +use crate::spirv_source::{ + FindPackage as _, get_channel_from_rustc_codegen_spirv_build_script, query_metadata, +}; +use crate::{cache_dir, spirv_source::SpirvSource}; +use anyhow::Context as _; +use spirv_builder::SpirvBuilder; +use std::path::{Path, PathBuf}; + +/// Represents a functional backend installation, whether it was cached or just installed. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct InstalledBackend { + /// path to the `rustc_codegen_spirv` dylib + pub rustc_codegen_spirv_location: PathBuf, + /// toolchain channel name + pub toolchain_channel: String, + /// Whether this runs in a build script + pub build_script: bool, +} + +impl InstalledBackend { + /// Creates a new `SpirvBuilder` configured to use this installed backend. + #[expect( + clippy::unreachable, + reason = "it's unreachable, no need to return a Result" + )] + #[expect(clippy::impl_trait_in_params, reason = "forwarding spirv-builder API")] + #[inline] + pub fn to_spirv_builder( + &self, + path_to_crate: impl AsRef, + target: impl Into, + ) -> SpirvBuilder { + let mut builder = SpirvBuilder::new(path_to_crate, target); + self.configure_spirv_builder(&mut builder) + .unwrap_or_else(|_| unreachable!("we set target before calling this function")); + builder + } + + /// Configures the supplied [`SpirvBuilder`]. `SpirvBuilder.target` must be set and must not change after calling this function. + /// + /// # Errors + /// if `SpirvBuilder.target` is not set + #[inline] + pub fn configure_spirv_builder(&self, builder: &mut SpirvBuilder) -> anyhow::Result<()> { + builder.rustc_codegen_spirv_location = Some(self.rustc_codegen_spirv_location.clone()); + builder.toolchain_overwrite = Some(self.toolchain_channel.clone()); + builder.build_script.defaults = self.build_script; + Ok(()) + } +} + +/// Args for an install +#[expect( + clippy::struct_excessive_bools, + reason = "cmdline args have many bools" +)] +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[cfg_attr(feature = "clap", derive(clap::Parser))] +#[non_exhaustive] +pub struct Install { + /// Cargo target to compile. + /// + /// Conflicts with `--shader-crate`. + #[cfg_attr(feature = "clap", clap(short, long, conflicts_with("shader_crate")))] + pub package: Option, + + /// Directory containing the shader crate to compile. + /// + /// Conflicts with `--package` or `-p`. + #[cfg_attr( + feature = "clap", + clap(long, default_value = "./", conflicts_with("package")) + )] + pub shader_crate: PathBuf, + + #[expect( + clippy::doc_markdown, + reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" + )] + #[expect(rustdoc::bare_urls)] + /// Source of `spirv-builder` dependency + /// Eg: "https://github.com/Rust-GPU/rust-gpu" + #[cfg_attr(feature = "clap", clap(long))] + pub spirv_builder_source: Option, + + /// Version of `spirv-builder` dependency. + /// * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic + /// version such as "0.9.0". + /// * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such + /// as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. + #[cfg_attr(feature = "clap", clap(long, verbatim_doc_comment))] + pub spirv_builder_version: Option, + + /// Force `rustc_codegen_spirv` to be rebuilt. + #[cfg_attr(feature = "clap", clap(long))] + pub rebuild_codegen: bool, + + /// Assume "yes" to "Install Rust toolchain: [y/n]" prompt. + /// + /// Defaults to `false` in cli, `true` in [`Default`] + #[cfg_attr(feature = "clap", clap(long, action))] + pub auto_install_rust_toolchain: bool, + + /// Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about + /// 200MiB of disk space. + #[cfg_attr(feature = "clap", clap(long = "no-clear-target", default_value = "true", action = clap::ArgAction::SetFalse + ))] + pub clear_target: bool, + + /// Enables printing `cargo:rerun-if-changed` to stdout for build scripts, defaults to `false`. + /// + /// Will be forwarded to `spirv_builder.build_script.defaults`. On the cargo-gpu side, only used with `path`-based + /// `spirv-std` sources, so that any change in your local rust-gpu checkout reruns the build script to rebuild the + /// codegen backend and your shaders. + #[cfg_attr(feature = "clap", clap(skip))] + pub build_script: bool, + + /// There is a tricky situation where a shader crate that depends on workspace config can have + /// a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can + /// prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. + /// + /// The ideal way to resolve this would be to match the shader crate's toolchain with the + /// workspace's toolchain. However, that is not always possible. Another solution is to + /// `exclude = [...]` the problematic shader crate from the workspace. This also may not be a + /// suitable solution if there are a number of shader crates all sharing similar config and + /// you don't want to have to copy/paste and maintain that config across all the shaders. + /// + /// So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling + /// this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also + /// only overwrite versions for the duration of a build. It will attempt to return the versions + /// to their original values once the build is finished. However, of course, unexpected errors + /// can occur and the overwritten values can remain. Hence why this behaviour is not enabled by + /// default. + /// + /// This hack is possible because the change from v3 to v4 only involves a minor change to the + /// way source URLs are encoded. See these PRs for more details: + /// * + /// * + #[cfg_attr(feature = "clap", clap(long, action, verbatim_doc_comment))] + pub force_overwrite_lockfiles_v4_to_v3: bool, +} + +impl Install { + /// Create a default install for a shader crate of some path + #[inline] + #[must_use] + pub const fn from_shader_crate(shader_crate: PathBuf) -> Self { + Self { + package: None, + shader_crate, + spirv_builder_source: None, + spirv_builder_version: None, + rebuild_codegen: false, + auto_install_rust_toolchain: true, + clear_target: true, + build_script: false, + force_overwrite_lockfiles_v4_to_v3: false, + } + } + + /// Set `build_script = true` + #[inline] + pub fn within_build_script(self) -> Self { + Self { + build_script: true, + ..self + } + } + + /// Create the `rustc_codegen_spirv_dummy` crate that depends on `rustc_codegen_spirv` + fn write_source_files(source: &SpirvSource, checkout: &Path) -> anyhow::Result<()> { + // skip writing a dummy project if we use a local rust-gpu checkout + if source.is_path() { + return Ok(()); + } + log::debug!( + "writing `rustc_codegen_spirv_dummy` source files into '{}'", + checkout.display() + ); + + { + log::trace!("writing dummy lib.rs"); + let src = checkout.join("src"); + std::fs::create_dir_all(&src).context("creating 'src' directory")?; + std::fs::File::create(src.join("lib.rs")).context("creating 'src/lib.rs'")?; + }; + + { + log::trace!("writing dummy Cargo.toml"); + let version_spec = match &source { + SpirvSource::CratesIO(version) => { + format!("version = \"{version}\"") + } + SpirvSource::Git { url, rev } => format!("git = \"{url}\"\nrev = \"{rev}\""), + SpirvSource::Path { + rust_gpu_repo_root, + version, + } => { + // this branch is currently unreachable, as we just build `rustc_codegen_spirv` directly, + // since we don't need the `dummy` crate to make cargo download it for us + let mut new_path = rust_gpu_repo_root.to_owned(); + new_path.push("crates/spirv-builder"); + format!("path = \"{new_path}\"\nversion = \"{version}\"") + } + }; + let cargo_toml = format!( + r#" +[package] +name = "rustc_codegen_spirv_dummy" +version = "0.1.0" +edition = "2021" + +[dependencies.spirv-builder] +package = "rustc_codegen_spirv" +{version_spec} + "# + ); + std::fs::write(checkout.join("Cargo.toml"), cargo_toml) + .context("writing 'Cargo.toml'")?; + }; + Ok(()) + } + + /// Install the binary pair and return the [`InstalledBackend`], from which you can create [`SpirvBuilder`] instances. + /// + /// # Errors + /// If the installation somehow fails. + #[inline] + #[expect(clippy::too_many_lines, reason = "it's fine")] + pub fn run(&self) -> anyhow::Result { + // Ensure the cache dir exists + let cache_dir = cache_dir()?; + log::info!("cache directory is '{}'", cache_dir.display()); + std::fs::create_dir_all(&cache_dir).with_context(|| { + format!("could not create cache directory '{}'", cache_dir.display()) + })?; + + let source = SpirvSource::new( + &self.shader_crate, + self.spirv_builder_source.as_deref(), + self.spirv_builder_version.as_deref(), + )?; + let install_dir = source.install_dir()?; + + let dylib_filename = format!( + "{}rustc_codegen_spirv{}", + std::env::consts::DLL_PREFIX, + std::env::consts::DLL_SUFFIX + ); + + if self.build_script { + #[allow(clippy::print_stdout)] + if let SpirvSource::Path { + rust_gpu_repo_root, .. + } = &source + { + println!("cargo:rerun-if-changed={rust_gpu_repo_root}"); + } + } + + let (dest_dylib_path, skip_rebuild) = if source.is_path() { + ( + install_dir + .join("target") + .join("release") + .join(&dylib_filename), + // if `source` is a path, always rebuild + false, + ) + } else { + let dest_dylib_path = install_dir.join(&dylib_filename); + let artifacts_found = dest_dylib_path.is_file() + && install_dir.join("Cargo.toml").is_file() + && install_dir.join("src").join("lib.rs").is_file(); + if artifacts_found { + log::info!("cargo-gpu artifacts found in '{}'", install_dir.display()); + } + (dest_dylib_path, artifacts_found && !self.rebuild_codegen) + }; + + if skip_rebuild { + log::info!("...and so we are aborting the install step."); + } else { + Self::write_source_files(&source, &install_dir).context("writing source files")?; + } + + // TODO cache toolchain channel in a file? + log::debug!("resolving toolchain version to use"); + let dummy_metadata = query_metadata(&install_dir) + .context("resolving toolchain version: get `rustc_codegen_spirv_dummy` metadata")?; + let rustc_codegen_spirv = dummy_metadata.find_package("rustc_codegen_spirv").context( + "resolving toolchain version: expected a dependency on `rustc_codegen_spirv`", + )?; + let toolchain_channel = + get_channel_from_rustc_codegen_spirv_build_script(rustc_codegen_spirv).context( + "resolving toolchain version: read toolchain from `rustc_codegen_spirv`'s build.rs", + )?; + log::info!("selected toolchain channel `{toolchain_channel:?}`"); + + log::debug!("ensure_toolchain_and_components_exist"); + crate::install_toolchain::ensure_toolchain_and_components_exist( + &toolchain_channel, + self.auto_install_rust_toolchain, + ) + .context("ensuring toolchain and components exist")?; + + if !skip_rebuild { + // to prevent unsupported version errors when using older toolchains + if !source.is_path() { + log::debug!("remove Cargo.lock"); + std::fs::remove_file(install_dir.join("Cargo.lock")) + .context("remove Cargo.lock")?; + } + + crate::user_output!("Compiling `rustc_codegen_spirv` from source {}\n", source); + let mut cargo = spirv_builder::cargo_cmd::CargoCmd::new(); + // Make sure spirv-tools is build normally and does not skip C++ compile due to "being run in clippy" + // We add this only to our install and not generally to `CargoCmd` since we do want to forward clippy args + // to clippy running on the spirv target via e.g. `cargo gpu clippy`. + cargo.env_remove("CLIPPY_ARGS"); + cargo + .current_dir(&install_dir) + .arg(format!("+{toolchain_channel}")) + .args(["build", "--release"]); + if source.is_path() { + cargo.args(["-p", "rustc_codegen_spirv", "--lib"]); + } + + log::debug!("building artifacts with `{cargo:?}`"); + cargo + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .output() + .context("getting command output") + .and_then(|output| { + if output.status.success() { + Ok(output) + } else { + Err(anyhow::anyhow!("bad status {:?}", output.status)) + } + }) + .context("running build command")?; + + let target = install_dir.join("target"); + let dylib_path = target.join("release").join(&dylib_filename); + if dylib_path.is_file() { + log::info!("successfully built {}", dylib_path.display()); + if !source.is_path() { + std::fs::rename(&dylib_path, &dest_dylib_path) + .context("renaming dylib path")?; + + if self.clear_target { + log::warn!("clearing target dir {}", target.display()); + std::fs::remove_dir_all(&target).context("clearing target dir")?; + } + } + } else { + log::error!("could not find {}", dylib_path.display()); + anyhow::bail!("`rustc_codegen_spirv` build failed"); + } + } + + Ok(InstalledBackend { + rustc_codegen_spirv_location: dest_dylib_path, + toolchain_channel, + build_script: self.build_script, + }) + } +} diff --git a/crates/cargo-gpu-install/src/install_toolchain.rs b/crates/cargo-gpu-install/src/install_toolchain.rs new file mode 100644 index 00000000000..12e9255ba36 --- /dev/null +++ b/crates/cargo-gpu-install/src/install_toolchain.rs @@ -0,0 +1,232 @@ +//! toolchain installation logic + +use crate::user_output; +use anyhow::Context as _; +#[cfg(feature = "tty")] +use crossterm::tty::IsTty as _; +use std::collections::HashSet; +use std::process::Command; +use std::string::FromUtf8Error; + +/// list of required rustup components +pub const REQUIRED_COMPONENTS: &[&str] = [ + "cargo", + "rustc", + "rust-std", + "clippy", + "rust-src", + "rustc-dev", + "llvm-tools", +] +.as_slice(); + +/// Use `rustup` to install the toolchain and components, if not already installed. +/// +/// Pretty much runs: +/// +/// * rustup toolchain add nightly-2024-04-24 +/// * rustup component add --toolchain nightly-2024-04-24 rust-src rustc-dev llvm-tools +pub fn ensure_toolchain_and_components_exist( + channel: &str, + skip_toolchain_install_consent: bool, +) -> anyhow::Result<()> { + // While our channel may be `nightly-2024-04-24`, it'll be resolved to the full toolchain name of e.g. + // `nightly-2024-04-24-aarch64-unknown-linux-gnu` and that's also what `rustup toolchain list` will print. + // Only checking whether the toolchain starts with the channel name may incorrectly pass if you have a toolchain + // installed that you're not able to run on your system via `rustup toolchain install --force-non-host ...`. + // CMD: `rustc --print host-tuple` + // TODO: What if the user has no toolchain installed? You can't query this with rustup sady. + let (host_tuple, _) = run_cmd(Command::new("rustc").args(["--print", "host-tuple"]))?; + let host_tuple = host_tuple.trim_ascii(); + let toolchain = format!("{channel}-{host_tuple}"); + + if !is_toolchain_installed(&toolchain, host_tuple)? { + let message = format!( + "toolchain {channel} with components {}", + intersperse(", ", REQUIRED_COMPONENTS.iter().copied()) + ); + get_consent_for_toolchain_install( + format!("Install {message}").as_ref(), + skip_toolchain_install_consent, + )?; + user_output!("Installing {message}\n"); + + // component list may be out of sync + // CMD: `rustup toolchain install nightly-2024-04-24 -c clippy,rust-src,rustc-dev,llvm-tools` + run_cmd( + Command::new("rustup") + .args([ + "toolchain", + "install", + &toolchain, + "--profile", + "minimal", + "-c", + &intersperse(",", REQUIRED_COMPONENTS.iter().copied()), + ]) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()), + )?; + } + + Ok(()) +} + +/// Returns true if the toolchain and required components are installed. +fn is_toolchain_installed(toolchain: &str, host_tuple: &str) -> anyhow::Result { + // check if toolchain is installed + // CMD: `rustup toolchain list -q` + let (list_toolchains, _) = run_cmd(Command::new("rustup").args(["toolchain", "list", "-q"]))?; + if !list_toolchains + .split_ascii_whitespace() + .any(|s| s == toolchain) + { + log::info!("toolchain {toolchain} is not installed"); + return Ok(false); + } + + // check if required components are installed + // NOTE: checking for components will install the toolchain with the default profile, if not already installed! + // So we must check beforehand whether the toolchain is installed, to not accidentally install it here. + // Passing *just* `-q` will list available components, so add `--installed` for installed components. + // CMD: `rustup component list --toolchain nightly-2024-04-24-aarch64-unknown-linux-gnu -q --installed` + let (components, _) = run_cmd(Command::new("rustup").args([ + "component", + "list", + "--toolchain", + toolchain, + "-q", + "--installed", + ]))?; + + // components are listed as: + // * `llvm-tools-aarch64-unknown-linux-gnu` and we need to snippet off the host tuple from the end + // * `rust-src` since source code isn't target dependent + let component_host_suffix = format!("-{host_tuple}"); + let mut required = REQUIRED_COMPONENTS.iter().copied().collect::>(); + for component in components.split_ascii_whitespace() { + required.remove( + component + .strip_suffix(&component_host_suffix) + .unwrap_or(component), + ); + } + if !required.is_empty() { + log::info!("components {required:?} missing for toolchain {toolchain}"); + return Ok(false); + } + + log::info!("toolchain and required components are already installed"); + Ok(true) +} + +pub fn run_cmd(cmd: &mut Command) -> anyhow::Result<(String, String)> { + let output = cmd.output(); + let fmt_cmd = || { + intersperse( + " ", + std::iter::once(cmd.get_program()) + .chain(cmd.get_args()) + .map(|s| s.to_str().unwrap()), + ) + }; + let output = output.with_context(|| format!("Failed to launch cmd `{}`", fmt_cmd()))?; + + let utf8_error = |e: FromUtf8Error, kind: &str| { + anyhow::anyhow!( + "Command `{}` {} contains invalid UTF-8: {} \n {:?}", + kind, + fmt_cmd(), + e.utf8_error(), + e.into_bytes() + ) + }; + let stdout = String::from_utf8(output.stdout).map_err(|e| utf8_error(e, "stdout"))?; + let stderr = String::from_utf8(output.stderr).map_err(|e| utf8_error(e, "stderr"))?; + + if !output.status.success() { + anyhow::bail!( + "Command `{}` failed with {}:\n-- stdout\n{stdout}\n-- stderr\n{stderr}", + fmt_cmd(), + &output.status, + ); + } + Ok((stdout, stderr)) +} + +/// Folds an [`Iterator`] of `&str` into a [`String`] while interspersing some `&str` between each element +#[expect(clippy::string_add, reason = "Deliberately using String::add")] +fn intersperse<'a>(intersperse: &str, iter: impl Iterator) -> String { + let mut s = iter.fold(String::new(), |a, b| a + b + intersperse); + s.truncate(s.len() - intersperse.len()); + s +} + +#[cfg(not(feature = "tty"))] +/// Prompt user if they want to install a new Rust toolchain. +fn get_consent_for_toolchain_install( + _prompt: &str, + skip_toolchain_install_consent: bool, +) -> anyhow::Result<()> { + if skip_toolchain_install_consent { + Ok(()) + } else { + no_tty() + } +} + +#[cfg(feature = "tty")] +/// Prompt user if they want to install a new Rust toolchain. +fn get_consent_for_toolchain_install( + prompt: &str, + skip_toolchain_install_consent: bool, +) -> anyhow::Result<()> { + if skip_toolchain_install_consent { + return Ok(()); + } + + if !std::io::stdout().is_tty() { + no_tty() + } + + log::debug!("asking for consent to install the required toolchain"); + crossterm::terminal::enable_raw_mode().context("enabling raw mode")?; + user_output!("{prompt} [Y/n]: "); + let mut input = crossterm::event::read().context("reading crossterm event")?; + + if let crossterm::event::Event::Key(crossterm::event::KeyEvent { + code: crossterm::event::KeyCode::Enter, + kind: crossterm::event::KeyEventKind::Release, + .. + }) = input + { + // In Powershell, programs will potentially observe the Enter key release after they started + // (see crossterm#124). If that happens, re-read the input. + input = crossterm::event::read().context("re-reading crossterm event")?; + } + crossterm::terminal::disable_raw_mode().context("disabling raw mode")?; + + #[expect(clippy::print_stdout, reason = "need a newline after crossterm input")] + { + println!(); + } + + if let crossterm::event::Event::Key(crossterm::event::KeyEvent { + code: crossterm::event::KeyCode::Char('y') | crossterm::event::KeyCode::Enter, + .. + }) = input + { + Ok(()) + } else { + user_output!("Exiting...\n"); + #[expect(clippy::exit, reason = "user requested abort")] + std::process::exit(0); + } +} + +fn no_tty() -> ! { + user_output!("No TTY detected so can't ask for consent to install Rust toolchain."); + log::error!("Attempted to ask for consent when there's no TTY"); + #[expect(clippy::exit, reason = "can't ask for user consent if there's no TTY")] + std::process::exit(1); +} diff --git a/crates/cargo-gpu-install/src/lib.rs b/crates/cargo-gpu-install/src/lib.rs new file mode 100644 index 00000000000..19cc48bcf8f --- /dev/null +++ b/crates/cargo-gpu-install/src/lib.rs @@ -0,0 +1,73 @@ +#![expect(clippy::pub_use, reason = "pub use for build scripts")] +#![expect(missing_docs, reason = "crate docs are cfg'ed out")] +#![cfg_attr(doc, doc = include_str!("../README.md"))] + +pub mod install; +mod install_toolchain; +pub mod spirv_source; +pub mod test; + +pub use spirv_builder; + +/// Central function to write to the user. +#[macro_export] +#[cfg(feature = "tty")] +macro_rules! user_output { + ($($args: tt)*) => { { + #[allow( + clippy::allow_attributes, + clippy::useless_attribute, + unused_imports, + reason = "`std::io::Write` is only sometimes called??" + )] + use std::io::Write as _; + + #[expect( + clippy::non_ascii_literal, + reason = "CRAB GOOD. CRAB IMPORTANT." + )] + { + print!("🦀 "); + } + print!($($args)*); + std::io::stdout().flush().unwrap(); + } } +} + +/// Central function to write to the user. +#[macro_export] +#[cfg(not(feature = "tty"))] +macro_rules! user_output { + ($($args: tt)*) => {{}}; +} + +/// The central cache directory of cargo gpu +/// +/// # Errors +/// may fail if we can't find the user home directory +#[inline] +#[cfg(not(any(feature = "test", test)))] +#[expect(clippy::cfg_not_test, reason = "tests use different cache_dir")] +pub fn cache_dir() -> anyhow::Result { + use anyhow::Context as _; + Ok(directories::BaseDirs::new() + .with_context(|| "could not find the user home directory")? + .cache_dir() + .join("rust-gpu")) +} + +#[cfg(any(feature = "test", test))] +pub use test::test_cache_dir as cache_dir; + +/// Returns a string suitable to use as a directory. +/// +/// Created from the spirv-builder source dep and the rustc channel. +fn to_dirname(text: &str) -> String { + text.replace( + [std::path::MAIN_SEPARATOR, '\\', '/', '.', ':', '@', '='], + "_", + ) + .split(['{', '}', ' ', '\n', '"', '\'']) + .collect::>() + .concat() +} diff --git a/crates/cargo-gpu-install/src/spirv_source.rs b/crates/cargo-gpu-install/src/spirv_source.rs new file mode 100644 index 00000000000..9fc1e01222e --- /dev/null +++ b/crates/cargo-gpu-install/src/spirv_source.rs @@ -0,0 +1,351 @@ +//! Use the shader that we're compiling as the default source for which version of `rust-gpu` to use. +//! +//! We do this by calling `cargo tree` inside the shader's crate to get the defined `spirv-std` +//! version. Then with that we `git checkout` the `rust-gpu` repo that corresponds to that version. +//! From there we can look at the source code to get the required Rust toolchain. + +use anyhow::Context as _; +use cargo_metadata::camino::{Utf8Path, Utf8PathBuf}; +use cargo_metadata::semver::Version; +use cargo_metadata::{Metadata, MetadataCommand, Package}; +use std::fs; +use std::path::{Path, PathBuf}; + +#[expect( + clippy::doc_markdown, + reason = "The URL should appear literally like this. But Clippy wants a markdown clickable link" +)] +#[expect(rustdoc::bare_urls)] +/// The source and version of `rust-gpu`. +/// Eg: +/// * From crates.io with version "0.10.0" +/// * From Git with: +/// - a repo of "https://github.com/Rust-GPU/rust-gpu.git" +/// - a revision of "abc213" +/// * a local Path +#[non_exhaustive] +#[derive(Eq, PartialEq, Clone, Debug)] +pub enum SpirvSource { + /// If the shader specifies a simple version like `spirv-std = "0.9.0"` then the source of + /// `rust-gpu` is the conventional crates.io version. + CratesIO(Version), + /// If the shader specifies a version like: + /// `spirv-std = { git = "https://github.com..." ... }` + /// then the source of `rust-gpu` is `Git`. + Git { + /// URL of the repository + url: String, + /// Revision or "commitsh" + rev: String, + }, + /// If the shader specifies a version like: + /// `spirv-std = { path = "/path/to/rust-gpu" ... }` + /// then the source of `rust-gpu` is `Path`. + Path { + /// File path of rust-gpu repository + rust_gpu_repo_root: Utf8PathBuf, + /// Version of specified rust-gpu repository + version: Version, + }, +} + +impl core::fmt::Display for SpirvSource { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::CratesIO(version) => version.fmt(f), + Self::Git { url, rev } => { + // shorten rev to 8 chars, prevents windows compile errors due to too long paths... seriously + if let Some(short_rev) = rev.get(..8) { + write!(f, "{url}+{short_rev}") + } else { + write!(f, "{url}+{rev}") + } + } + Self::Path { + rust_gpu_repo_root, + version, + } => write!(f, "{rust_gpu_repo_root}+{version}"), + } + } +} + +impl SpirvSource { + /// Figures out which source of `rust-gpu` to use + /// + /// # Errors + /// Crate may not depend on `spirv-std` or is otherwise malformed + pub fn new( + shader_crate_path: &Path, + maybe_rust_gpu_source: Option<&str>, + maybe_rust_gpu_version: Option<&str>, + ) -> anyhow::Result { + let source = if let Some(rust_gpu_version) = maybe_rust_gpu_version { + if let Some(rust_gpu_source) = maybe_rust_gpu_source { + Self::Git { + url: rust_gpu_source.to_owned(), + rev: rust_gpu_version.to_owned(), + } + } else { + Self::CratesIO(Version::parse(rust_gpu_version)?) + } + } else { + Self::get_rust_gpu_deps_from_shader(shader_crate_path).with_context(|| { + format!( + "get spirv-std dependency from shader crate '{}'", + shader_crate_path.display() + ) + })? + }; + Ok(source) + } + + /// Look into the shader crate to get the version of `rust-gpu` it's using. + /// + /// # Errors + /// Crate may not depend on `spirv-std` or is otherwise malformed + pub fn get_rust_gpu_deps_from_shader(shader_crate_path: &Path) -> anyhow::Result { + let crate_metadata = query_metadata(shader_crate_path)?; + let spirv_std_package = crate_metadata.find_package("spirv-std")?; + let spirv_source = Self::parse_spirv_std_source_and_version(spirv_std_package)?; + log::debug!( + "Parsed `SpirvSource` from crate `{}`: \ + {spirv_source:?}", + shader_crate_path.display(), + ); + Ok(spirv_source) + } + + /// Convert the `SpirvSource` to a cache directory in which we can build it. + /// It needs to be dynamically created because an end-user might want to swap out the source, + /// maybe using their own fork for example. + /// + /// # Errors + /// [`crate::cache_dir`] may fail + pub fn install_dir(&self) -> anyhow::Result { + match self { + Self::Path { + rust_gpu_repo_root, .. + } => Ok(rust_gpu_repo_root.as_std_path().to_owned()), + Self::CratesIO { .. } | Self::Git { .. } => { + let dir = crate::to_dirname(self.to_string().as_ref()); + Ok(crate::cache_dir()?.join("codegen").join(dir)) + } + } + } + + /// Returns true if self is a Path + #[must_use] + pub const fn is_path(&self) -> bool { + matches!(self, Self::Path { .. }) + } + + /// Parse a string like: + /// `spirv-std v0.9.0 (https://github.com/Rust-GPU/rust-gpu?rev=54f6978c#54f6978c) (*)` + /// Which would return: + /// `SpirvSource::Git("https://github.com/Rust-GPU/rust-gpu", "54f6978c")` + fn parse_spirv_std_source_and_version(spirv_std_package: &Package) -> anyhow::Result { + log::trace!("parsing spirv-std source and version from package: '{spirv_std_package:?}'"); + + let result = if let Some(source) = &spirv_std_package.source { + let is_git = source.repr.starts_with("git+"); + let is_crates_io = source.is_crates_io(); + + match (is_git, is_crates_io) { + (true, true) => anyhow::bail!("parsed both git and crates.io?"), + (true, false) => { + let parse_git = || { + let link = &source.repr.get(4..)?; + let sharp_index = link.find('#')?; + let url_end = link.find('?').unwrap_or(sharp_index); + let url = link.get(..url_end)?.to_owned(); + let rev = link.get(sharp_index + 1..)?.to_owned(); + Some(Self::Git { url, rev }) + }; + parse_git() + .with_context(|| format!("Failed to parse git url {}", &source.repr))? + } + (false, true) => Self::CratesIO(spirv_std_package.version.clone()), + (false, false) => { + anyhow::bail!("Metadata of spirv-std package uses unknown url format!") + } + } + } else { + let rust_gpu_repo_root = spirv_std_package + .manifest_path // rust-gpu/crates/spirv-std/Cargo.toml + .parent() // rust-gpu/crates/spirv-std + .and_then(Utf8Path::parent) // rust-gpu/crates + .and_then(Utf8Path::parent) // rust-gpu + .context("selecting rust-gpu workspace root dir in local path")? + .to_owned(); + if !rust_gpu_repo_root.is_dir() { + anyhow::bail!("path {rust_gpu_repo_root} is not a directory"); + } + let version = spirv_std_package.version.clone(); + Self::Path { + rust_gpu_repo_root, + version, + } + }; + + log::debug!("Parsed `rust-gpu` source and version: {result:?}"); + + Ok(result) + } +} + +/// get the Package metadata from some crate +/// +/// # Errors +/// metadata query may fail +pub fn query_metadata(crate_path: &Path) -> anyhow::Result { + log::debug!("Running `cargo metadata` on `{}`", crate_path.display()); + let metadata = MetadataCommand::new() + .current_dir( + &crate_path + .canonicalize() + .context("could not get absolute path to shader crate")?, + ) + .exec()?; + Ok(metadata) +} + +/// implements [`Self::find_package`] +pub trait FindPackage { + /// Search for a package or return a nice error + /// + /// # Errors + /// package may not be found or crate may be malformed + fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package>; +} + +impl FindPackage for Metadata { + fn find_package(&self, crate_name: &str) -> anyhow::Result<&Package> { + if let Some(package) = self + .packages + .iter() + .find(|package| package.name.as_str() == crate_name) + { + log::trace!(" found `{}` version `{}`", package.name, package.version); + Ok(package) + } else { + anyhow::bail!( + "`{crate_name}` not found in `Cargo.toml` at `{:?}`", + self.workspace_root + ); + } + } +} + +/// Parse the `rust-toolchain.toml` in the working tree of the checked-out version of the `rust-gpu` repo. +/// +/// # Errors +/// parsing may fail +pub fn get_channel_from_rustc_codegen_spirv_build_script( + rustc_codegen_spirv_package: &Package, +) -> anyhow::Result { + let path = rustc_codegen_spirv_package + .manifest_path + .parent() + .context("finding `rustc_codegen_spirv` crate root")?; + let build_rs = path.join("build.rs"); + + log::debug!("Parsing `build.rs` at {build_rs:?} for the used toolchain"); + let contents = fs::read_to_string(&build_rs)?; + let channel_start = "channel = \""; + let channel_line = contents + .lines() + .find_map(|line| line.strip_prefix(channel_start)) + .context(format!("Can't find `{channel_start}` line in {build_rs:?}"))?; + let channel = channel_line + .get(..channel_line.find('"').context("ending \" missing")?) + .context("can't slice version")?; + Ok(channel.to_owned()) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::test::TestEnv; + use cargo_metadata::{PackageBuilder, PackageId, Source}; + use cargo_util_schemas::manifest::PackageName; + use expect_test::expect; + + #[test_log::test] + fn parsing_spirv_std_dep_for_shader_template() { + let shader_template_path = crate::test::shader_crate_template_path(); + let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); + expect![[r#" + Git { + url: "https://github.com/Rust-GPU/rust-gpu", + rev: "877bd8697a15f3e6d09446a5e1807e6237ca1dac", + }"#]] + .assert_eq(&format!("{source:#?}")); + } + + #[test_log::test] + fn cached_checkout_dir_sanity() { + let _env = TestEnv::new(); + let shader_template_path = crate::test::shader_crate_template_path(); + let source = SpirvSource::get_rust_gpu_deps_from_shader(&shader_template_path).unwrap(); + let dir = source.install_dir().unwrap(); + let name = dir + .file_name() + .unwrap() + .to_str() + .map(std::string::ToString::to_string) + .unwrap(); + expect!["https___github_com_Rust-GPU_rust-gpu+877bd869"].assert_eq(&name); + } + + #[test_log::test] + fn path_sanity() { + let path = std::path::PathBuf::from("./"); + assert!(path.is_relative()); + } + + #[test_log::test] + fn parse_git_with_rev() { + let source = parse_git( + "git+https://github.com/Rust-GPU/rust-gpu?rev=6a67e7b5954f37989ad540a555b5d6969073592e#86fc4803", + ); + assert_eq!( + source, + SpirvSource::Git { + url: "https://github.com/Rust-GPU/rust-gpu".to_owned(), + rev: "86fc4803".to_owned(), + } + ); + } + + #[test_log::test] + fn parse_git_no_question_mark() { + // taken directly from Graphite + let source = parse_git( + "git+https://github.com/Rust-GPU/rust-gpu.git#6e2c84d4fe64e32df4c060c5a7f3e35a32e45421", + ); + assert_eq!( + source, + SpirvSource::Git { + url: "https://github.com/Rust-GPU/rust-gpu.git".to_owned(), + rev: "6e2c84d4fe64e32df4c060c5a7f3e35a32e45421".to_owned(), + } + ); + } + + fn parse_git(source: &str) -> SpirvSource { + let package = PackageBuilder::new( + PackageName::new("spirv-std".to_owned()).unwrap(), + Version::new(0, 9, 0), + PackageId { + repr: String::new(), + }, + "", + ) + .source(Some(Source { + repr: source.to_owned(), + })) + .build() + .unwrap(); + SpirvSource::parse_spirv_std_source_and_version(&package).unwrap() + } +} diff --git a/crates/cargo-gpu-install/src/test.rs b/crates/cargo-gpu-install/src/test.rs new file mode 100644 index 00000000000..adbec0a527b --- /dev/null +++ b/crates/cargo-gpu-install/src/test.rs @@ -0,0 +1,117 @@ +//! utilities for tests +#![cfg(any(feature = "test", test))] + +use anyhow::Context; +use std::cell::RefCell; +use std::fs::File; +use std::io::Write as _; +use std::path::PathBuf; +use tempfile::TempDir; + +/// `TestEnv` sets up a temp dir in `./target/cargo-gpu-test/` that is used as the cache dir. Not initializing a +/// `TestEnv` and asking for the cache dir will panic, to ensure you set it up. Calling [`Self::setup_shader_crate`] +/// or [`Self::setup_shader_crate_with_cargo_toml`] will copy the `shader_crate_template` into the temp dir and return +/// you the path to it, so each test now has it's unique copy and won't race to change the template in the repo. +/// Dropping `TestEnv` will clean up the dir, except when panic unwinding, so you can debug failures. +#[must_use] +pub struct TestEnv(TempDir); + +impl TestEnv { + /// Create a new [`TestEnv`] + pub fn new() -> Self { + let target_dir = cargo_metadata::MetadataCommand::new() + .exec() + .unwrap() + .target_directory + .into_std_path_buf(); + let tests_dir = target_dir.join("cargo-gpu-test"); + std::fs::create_dir_all(&tests_dir).ok(); + let test_dir = TempDir::new_in(tests_dir).unwrap(); + + let had_old = TESTDIR + .replace(Some(test_dir.path().to_path_buf())) + .is_some(); + if had_old { + panic!("TestEnv is not reentrant!") + } + + TestEnv(test_dir) + } + + /// Copies the `shader_crate_template` to the temp dir and returns the path to the directory. + #[expect(clippy::unused_self)] + pub fn setup_shader_crate(&self) -> anyhow::Result { + let shader_crate_path = crate::cache_dir()?.join("shader_crate"); + copy_dir_all(shader_crate_template_path(), &shader_crate_path)?; + Ok(shader_crate_path) + } + + /// Like [`Self::setup_shader_crate`], copies the `shader_crate_template` to the temp dir and returns the path to + /// the directory. Additionally, takes a closure to allow you to overwrite the contents of the `Cargo.toml`. + /// This function will write the bare minimum for a valid crate, that is, give it a package name. + pub fn setup_shader_crate_with_cargo_toml( + &self, + f: impl FnOnce(&mut File) -> std::io::Result<()>, + ) -> anyhow::Result { + let shader_crate_path = self.setup_shader_crate()?; + let cargo_toml = shader_crate_path.join("Cargo.toml"); + let mut file = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(cargo_toml)?; + writeln!(file, "[package]")?; + writeln!(file, "name = \"test\"")?; + f(&mut file)?; + Ok(shader_crate_path) + } +} + +impl Default for TestEnv { + fn default() -> Self { + Self::new() + } +} + +impl Drop for TestEnv { + fn drop(&mut self) { + TESTDIR.replace(None).unwrap(); + // when a test fails, keep directory + if std::thread::panicking() { + self.0.disable_cleanup(true); + } + } +} + +thread_local! { + static TESTDIR: RefCell> = const { RefCell::new(None) }; +} + +/// [`crate::cache_dir`] for testing, see [`TestEnv`] +pub fn test_cache_dir() -> anyhow::Result { + TESTDIR.with_borrow(|a| a.clone()).context( + "TestEnv is not initialized! Add `let _env = TestEnv::new();` to the beginning of your test", + ) +} + +fn copy_dir_all( + src: impl AsRef, + dst: impl AsRef, +) -> anyhow::Result<()> { + std::fs::create_dir_all(&dst)?; + for maybe_entry in std::fs::read_dir(src)? { + let entry = maybe_entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + std::fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} + +/// Path to the `shader-crate-template` for copying or querying data +pub fn shader_crate_template_path() -> PathBuf { + let project_base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + project_base.join("../shader-crate-template") +} diff --git a/crates/cargo-gpu/Cargo.toml b/crates/cargo-gpu/Cargo.toml new file mode 100644 index 00000000000..951e88451a1 --- /dev/null +++ b/crates/cargo-gpu/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "cargo-gpu" +version.workspace = true +edition.workspace = true +description = "Generates shader .spv files from rust-gpu shader crates" +repository.workspace = true +readme.workspace = true +keywords.workspace = true +license.workspace = true +default-run = "cargo-gpu" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cargo_metadata.workspace = true +anyhow.workspace = true +cargo-gpu-install = { workspace = true, features = ["clap", "watch", "tty"] } +clap.workspace = true +env_logger.workspace = true +log.workspace = true +relative-path.workspace = true +serde.workspace = true +serde_json.workspace = true +semver.workspace = true +dunce.workspace = true + +[dev-dependencies] +cargo-gpu-install = { workspace = true, features = ["test"] } +test-log.workspace = true +cargo_metadata = { workspace = true, features = ["builder"] } + +[lints] +workspace = true diff --git a/crates/cargo-gpu/README.md b/crates/cargo-gpu/README.md new file mode 100644 index 00000000000..e37f4e210fa --- /dev/null +++ b/crates/cargo-gpu/README.md @@ -0,0 +1,351 @@ +# cargo-gpu + +`cargo-gpu` is an installation manager and command line tool for [rust-gpu](https://github.com/Rust-GPU/rust-gpu/). `cargo-gpu` is not an essential requirement, it should just make working with `rust-gpu` easier. + +There are 2 ways to use it: +1. Through a CLI, ie `cargo gpu ...` +2. As a crate included in your build scripts or executables + +## 1. CLI Quickstart + +To install the command line tool, ensure you are using `rustup`. Then run: + +``` +cargo install --git https://github.com/rust-gpu/cargo-gpu cargo-gpu +``` + +You can then use `cargo gpu` to compile your shader crates or use any of the other commands you're used to: + +``` +cargo gpu build +cargo gpu check +cargo gpu clippy +``` + +### Example project + +To create an example project from our [templates](https://github.com/Rust-GPU/rust-gpu-template), use the command below: +``` +cargo install cargo-generate +cargo generate --git https://github.com/Rust-GPU/rust-gpu-template +# choose any template you want, then select cargo-gpu cmdline integration +# you may have to adjust the crate name +cargo gpu build -p mygraphics-shaders +``` + +This plain invocation will compile the crate in the current directory and place the compiled shaders in the current directory. + +Use `cargo gpu help` to see more options :) + +## 2. Crate Quickstart + +Add `cargo-gpu-install` as a regular or build dependency to your project, and use it like this: + +```rust,no_run +let shader_crate = PathBuf::from("./shaders"); +let backend = cargo_gpu_install::Install::from_shader_crate(shader_crate.clone()).run()?; +let mut builder = backend.to_spirv_builder(shader_crate, "spirv-unknown-vulkan1.2"); +// configure the builder... +let spv_result = builder.build()?; +``` + +For more detail, see the [readme of `cargo-gpu-install`](crates/cargo-gpu-install/README.md) or use any of our [templates](https://github.com/Rust-GPU/rust-gpu-template) as reference and choosing the `cargo-gpu` integration. + +## How it works + +Behind the scenes `cargo gpu` compiles a custom [codegen backend](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/codegen-backend.html) +for `rustc` that allows emitting [SPIR-V](https://www.khronos.org/spir/) assembly, instead of the conventional LLVM assembly. SPIR-V is a dedicated +graphics language that is aimed to be open and portable so that it works with as many drivers and devices as possible. + +With the custom codegen backend (`rustc_codegen_spirv`) `cargo gpu` then compiles the shader it is pointed to. However, because custom codegen backends +are currently [an unstable feature](https://github.com/rust-lang/rust/issues/77933), `cargo gpu` also needs to install a "nightly" version of Rust. In +the usage instructions the backend and nightly Rust version are referred to as "artefacts" and can be explicitly managed with the arguments to the +`install` subcommand. + +> [!TIP] +> Whilst `cargo gpu` attempts to isolate shader compilation as much possible, if the shader crate is contained in a workspace then it's possible that +> the nightly version required by the shader is, ironically, older than the Rust/Cargo versions required by the workspace. Say for instance the +> workspace might use a newer `Cargo.lock` layout not supported by the pinned version of the shader crate's custom codegen backend. The solution to +> this is to either exclude the shader from the workspace, or upgrade the shader's `spirv-std` dependency to the latest. + +## CLI Usage + +All the following arguments for the `build` and `install` commands can also be set in the shader crate's `Cargo.toml` +file. In general usage that would be the recommended way to set config. See `crates/shader-crate-template/Cargo.toml` +for an example. + +```` + Commands: + install Install rust-gpu compiler artifacts + build Compile a shader crate to SPIR-V + show Show some useful values + help Print this message or the help of the given subcommand(s) + + Options: + -h, --help + Print help + + -V, --version + Print version + + +* Install + Install rust-gpu compiler artifacts + + Usage: cargo-gpu install [OPTIONS] + + Options: + --shader-crate + Directory containing the shader crate to compile + + [default: ./] + + --spirv-builder-source + Source of `spirv-builder` dependency Eg: "https://github.com/Rust-GPU/rust-gpu" + + --spirv-builder-version + Version of `spirv-builder` dependency. + * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic + version such as "0.9.0". + * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such + as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. + + --rebuild-codegen + Force `rustc_codegen_spirv` to be rebuilt + + --auto-install-rust-toolchain + Assume "yes" to "Install Rust toolchain: [y/n]" prompt + + --no-clear-target + Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about 200MiB of disk space + + --force-overwrite-lockfiles-v4-to-v3 + There is a tricky situation where a shader crate that depends on workspace config can have + a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can + prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. + + The ideal way to resolve this would be to match the shader crate's toolchain with the + workspace's toolchain. However, that is not always possible. Another solution is to + `exclude = [...]` the problematic shader crate from the workspace. This also may not be a + suitable solution if there are a number of shader crates all sharing similar config and + you don't want to have to copy/paste and maintain that config across all the shaders. + + So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling + this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also + only overwrite versions for the duration of a build. It will attempt to return the versions + to their original values once the build is finished. However, of course, unexpected errors + can occur and the overwritten values can remain. Hence why this behaviour is not enabled by + default. + + This hack is possible because the change from v3 to v4 only involves a minor change to the + way source URLs are encoded. See these PRs for more details: + * + * + + -h, --help + Print help (see a summary with '-h') + + +* Build + Compile a shader crate to SPIR-V + + Usage: cargo-gpu build [OPTIONS] + + Options: + --shader-crate + Directory containing the shader crate to compile + + [default: ./] + + --spirv-builder-source + Source of `spirv-builder` dependency Eg: "https://github.com/Rust-GPU/rust-gpu" + + --spirv-builder-version + Version of `spirv-builder` dependency. + * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic + version such as "0.9.0". + * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such + as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. + + --rebuild-codegen + Force `rustc_codegen_spirv` to be rebuilt + + --auto-install-rust-toolchain + Assume "yes" to "Install Rust toolchain: [y/n]" prompt + + --no-clear-target + Clear target dir of `rustc_codegen_spirv` build after a successful build, saves about 200MiB of disk space + + --force-overwrite-lockfiles-v4-to-v3 + There is a tricky situation where a shader crate that depends on workspace config can have + a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can + prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. + + The ideal way to resolve this would be to match the shader crate's toolchain with the + workspace's toolchain. However, that is not always possible. Another solution is to + `exclude = [...]` the problematic shader crate from the workspace. This also may not be a + suitable solution if there are a number of shader crates all sharing similar config and + you don't want to have to copy/paste and maintain that config across all the shaders. + + So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling + this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also + only overwrite versions for the duration of a build. It will attempt to return the versions + to their original values once the build is finished. However, of course, unexpected errors + can occur and the overwritten values can remain. Hence why this behaviour is not enabled by + default. + + This hack is possible because the change from v3 to v4 only involves a minor change to the + way source URLs are encoded. See these PRs for more details: + * + * + + -o, --output-dir + Path to the output directory for the compiled shaders + + [default: ./] + + -w, --watch + Watch the shader crate directory and automatically recompile on changes + + --debug + Build in release. Defaults to true + + --target + The target triple, eg. `spirv-unknown-vulkan1.2` + + [default: spirv-unknown-vulkan1.2] + + --no-default-features + Set --default-features for the target shader crate + + --features + Set --features for the target shader crate + + --deny-warnings + Deny any warnings, as they may never be printed when building within a build script. Defaults to false + + --multimodule + Splits the resulting SPIR-V file into one module per entry point. This is useful in cases where ecosystem tooling has bugs around multiple entry points per module - having all entry points bundled into a single file is the preferred system + + --spirv-metadata + Sets the level of metadata (primarily `OpName` and `OpLine`) included in the SPIR-V binary. Including metadata significantly increases binary size + + [default: none] + + Possible values: + - none: Strip all names and other debug information from SPIR-V output + - name-variables: Only include `OpName`s for public interface variables (uniforms and the like), to allow shader reflection + - full: Include all `OpName`s for everything, and `OpLine`s. Significantly increases binary size + + --capabilities + Adds a capability to the SPIR-V module. Checking if a capability is enabled in code can be done via `#[cfg(target_feature = "TheCapability")]` + + --extensions + Adds an extension to the SPIR-V module. Checking if an extension is enabled in code can be done via `#[cfg(target_feature = "ext:the_extension")]` + + --relax-struct-store + Record whether or not the validator should relax the rules on types for stores to structs. When relaxed, it will allow a type mismatch as long as the types are structs with the same layout. Two structs have the same layout if + + 1) the members of the structs are either the same type or are structs with same layout, and + + 2) the decorations that affect the memory layout are identical for both types. Other decorations are not relevant. + + --relax-logical-pointer + Records whether or not the validator should relax the rules on pointer usage in logical addressing mode. + + When relaxed, it will allow the following usage cases of pointers: 1) `OpVariable` allocating an object whose type is a pointer type 2) `OpReturnValue` returning a pointer value + + --relax-block-layout + Records whether the validator should use "relaxed" block layout rules. Relaxed layout rules are described by Vulkan extension `VK_KHR_relaxed_block_layout`, and they affect uniform blocks, storage blocks, and push constants. + + This is enabled by default when targeting Vulkan 1.1 or later. Relaxed layout is more permissive than the default rules in Vulkan 1.0. + + [default: false] + [possible values: true, false] + + --uniform-buffer-standard-layout + Records whether the validator should use standard block layout rules for uniform blocks + + --scalar-block-layout + Records whether the validator should use "scalar" block layout rules. Scalar layout rules are more permissive than relaxed block layout. + + See Vulkan extnesion `VK_EXT_scalar_block_layout`. The scalar alignment is defined as follows: - scalar alignment of a scalar is the scalar size - scalar alignment of a vector is the scalar alignment of its component - scalar alignment of a matrix is the scalar alignment of its component - scalar alignment of an array is the scalar alignment of its element - scalar alignment of a struct is the max scalar alignment among its members + + For a struct in Uniform, `StorageClass`, or `PushConstant`: - a member Offset must be a multiple of the member's scalar alignment - `ArrayStride` or `MatrixStride` must be a multiple of the array or matrix scalar alignment + + --skip-block-layout + Records whether or not the validator should skip validating standard uniform/storage block layout + + --preserve-bindings + Records whether all bindings within the module should be preserved + + -m, --manifest-file + Renames the manifest.json file to the given name + + [default: manifest.json] + + -h, --help + Print help (see a summary with '-h') + + +* Show + Show some useful values + + Usage: cargo-gpu show + + Commands: + cache-directory Displays the location of the cache directory + spirv-source The source location of spirv-std + commitsh The git commitsh of this cli tool + capabilities All the available SPIR-V capabilities that can be set with `--capabilities` + help Print this message or the help of the given subcommand(s) + + Options: + -h, --help + Print help + + + * Cache-directory + Displays the location of the cache directory + + Usage: cargo-gpu show cache-directory + + Options: + -h, --help + Print help + + + * Spirv-source + The source location of spirv-std + + Usage: cargo-gpu show spirv-source [OPTIONS] + + Options: + --shader-crate + The location of the shader-crate to inspect to determine its spirv-std dependency + + [default: ./] + + -h, --help + Print help + + + * Commitsh + The git commitsh of this cli tool + + Usage: cargo-gpu show commitsh + + Options: + -h, --help + Print help + + + * Capabilities + All the available SPIR-V capabilities that can be set with `--capabilities` + + Usage: cargo-gpu show capabilities + + Options: + -h, --help + Print help +```` diff --git a/crates/cargo-gpu/build.rs b/crates/cargo-gpu/build.rs new file mode 100644 index 00000000000..31a33575ce8 --- /dev/null +++ b/crates/cargo-gpu/build.rs @@ -0,0 +1,12 @@ +//! cargo-gpu build script. + +fn main() { + let git_hash = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .map_or_else( + |_| "unknown".to_owned(), + |output| String::from_utf8(output.stdout).unwrap_or_else(|_| "unknown".to_owned()), + ); + println!("cargo:rustc-env=GIT_HASH={git_hash}"); +} diff --git a/crates/cargo-gpu/src/build.rs b/crates/cargo-gpu/src/build.rs new file mode 100644 index 00000000000..ed1e9736f30 --- /dev/null +++ b/crates/cargo-gpu/src/build.rs @@ -0,0 +1,235 @@ +#![allow(clippy::shadow_reuse, reason = "let's not be silly")] +#![allow(clippy::unwrap_used, reason = "this is basically a test")] +//! `cargo gpu build`, analogous to `cargo build` + +use crate::install::Install; +use crate::linkage::Linkage; +use crate::lockfile::LockfileMismatchHandler; +use crate::spirv_builder::{CompileResult, ModuleResult, SpirvBuilder, SpirvBuilderError}; +use anyhow::Context as _; +use std::io::Write as _; +use std::path::PathBuf; + +/// Args for just a build +#[derive(clap::Parser, Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct BuildArgs { + /// Path to the output directory for the compiled shaders. + #[clap(long, short, default_value = "./")] + pub output_dir: PathBuf, + + /// Watch the shader crate directory and automatically recompile on changes. + #[clap(long, short, action)] + pub watch: bool, + + /// The flattened [`SpirvBuilder`] + #[clap(flatten)] + #[serde(flatten)] + pub spirv_builder: SpirvBuilder, + + /// Renames the `manifest.json` file to the given name + #[clap(long, short, default_value = "manifest.json")] + pub manifest_file: String, + + /// When building fails with [`SpirvBuilderError::NoArtifactProduced`], count it as a success anyway. + /// Used for e.g. `clippy`, which doesn't produce any artifacts. Defaults to false. + #[clap(skip)] + pub allow_no_artifacts: bool, +} + +impl Default for BuildArgs { + #[inline] + fn default() -> Self { + Self { + output_dir: PathBuf::from("./"), + watch: false, + spirv_builder: SpirvBuilder::default(), + manifest_file: String::from("manifest.json"), + allow_no_artifacts: false, + } + } +} + +/// `cargo build` subcommands +#[derive(Clone, clap::Parser, Debug, serde::Deserialize, serde::Serialize)] +pub struct Build { + /// CLI args for install the `rust-gpu` compiler and components + #[clap(flatten)] + pub install: Install, + + /// CLI args for configuring the build of the shader + #[clap(flatten)] + pub build: BuildArgs, +} + +impl Build { + /// Entrypoint + pub fn run(&mut self) -> anyhow::Result<()> { + let installed_backend = self.install.run()?; + let mut metadata = crate::metadata::MetadataCache::default(); + + if let Some(package) = self.install.package.as_ref() { + self.install.shader_crate = metadata.resolve_package_to_shader_crate(package)?; + } + + let _lockfile_mismatch_handler = LockfileMismatchHandler::new( + &self.install.shader_crate, + &installed_backend.toolchain_channel, + self.install.force_overwrite_lockfiles_v4_to_v3, + )?; + + let builder = &mut self.build.spirv_builder; + builder.path_to_crate = Some(self.install.shader_crate.clone()); + installed_backend.configure_spirv_builder(builder)?; + + // Ensure the shader output dir exists + log::debug!( + "ensuring output-dir '{}' exists", + self.build.output_dir.display() + ); + std::fs::create_dir_all(&self.build.output_dir)?; + let canonicalized = dunce::canonicalize(&self.build.output_dir)?; + log::debug!("canonicalized output dir: {}", canonicalized.display()); + self.build.output_dir = canonicalized; + + // Ensure the shader crate exists + self.install.shader_crate = dunce::canonicalize(&self.install.shader_crate)?; + anyhow::ensure!( + self.install.shader_crate.exists(), + "shader crate '{}' does not exist. (Current dir is '{}')", + self.install.shader_crate.display(), + std::env::current_dir()?.display() + ); + + if self.build.watch { + let mut watcher = self.build.spirv_builder.clone().watch()?; + loop { + // if the build fails "regularly", eg. `cargo build` fails and nothing else, just retry + crate::user_output!( + "Compiling shaders at {}...\n", + self.install.shader_crate.display() + ); + match watcher.recv() { + Ok(result) => { + self.parse_compilation_result(&result)?; + crate::user_output!("Build successful!\n"); + } + Err(SpirvBuilderError::BuildFailed) => crate::user_output!("Build failed!\n"), + Err(err) => return Err(anyhow::Error::from(err)), + } + } + } else { + crate::user_output!( + "Compiling shaders at {}...\n", + self.install.shader_crate.display() + ); + let result = self.build.spirv_builder.build(); + match result { + Ok(result) => { + self.parse_compilation_result(&result)?; + } + // conditionally ignore NoArtifactProduced + Err(SpirvBuilderError::NoArtifactProduced { .. }) + if self.build.allow_no_artifacts => {} + Err(err) => return Err(err.into()), + } + } + Ok(()) + } + + /// Parses compilation result from `SpirvBuilder` and writes it out to a file + fn parse_compilation_result(&self, result: &CompileResult) -> anyhow::Result<()> { + let shaders = match &result.module { + ModuleResult::MultiModule(modules) => { + anyhow::ensure!(!modules.is_empty(), "No shader modules were compiled"); + modules.iter().collect::>() + } + ModuleResult::SingleModule(filepath) => result + .entry_points + .iter() + .map(|entry| (entry, filepath)) + .collect::>(), + }; + let mut linkage: Vec = shaders + .into_iter() + .map(|(entry, filepath)| -> anyhow::Result { + use relative_path::PathExt as _; + let path = self.build.output_dir.join( + filepath + .file_name() + .context("Couldn't parse file name from shader module path")?, + ); + log::debug!("copying {} to {}", filepath.display(), path.display()); + std::fs::copy(filepath, &path)?; + log::debug!( + "linkage of {} relative to {}", + path.display(), + self.install.shader_crate.display() + ); + let spv_path = path + .relative_to(&self.install.shader_crate) + .map_or(path, |path_relative_to_shader_crate| { + path_relative_to_shader_crate.to_path("") + }); + Ok(Linkage::new(entry, spv_path)) + }) + .collect::>>()?; + // Sort the contents so the output is deterministic + linkage.sort(); + + // Write the shader manifest json file + let manifest_path = self.build.output_dir.join(&self.build.manifest_file); + let json = serde_json::to_string_pretty(&linkage)?; + let mut file = std::fs::File::create(&manifest_path).with_context(|| { + format!( + "could not create shader manifest file '{}'", + manifest_path.display(), + ) + })?; + file.write_all(json.as_bytes()).with_context(|| { + format!( + "could not write shader manifest file '{}'", + manifest_path.display(), + ) + })?; + + log::info!("wrote manifest to '{}'", manifest_path.display()); + Ok(()) + } +} + +#[cfg(test)] +mod test { + use clap::Parser as _; + + use crate::{Cli, Command}; + + #[test_log::test] + fn builder_from_params() { + let shader_crate_path = crate::test::shader_crate_template_path(); + let output_dir = shader_crate_path.join("shaders"); + + let args = [ + "target/debug/cargo-gpu", + "build", + "--shader-crate", + &format!("{}", shader_crate_path.display()), + "--output-dir", + &format!("{}", output_dir.display()), + ]; + if let Cli { + command: Command::Build(build), + } = Cli::parse_from(args) + { + assert_eq!(shader_crate_path, build.install.shader_crate); + assert_eq!(output_dir, build.build.output_dir); + + // TODO: + // For some reason running a full build (`build.run()`) inside tests fails on Windows. + // The error is in the `build.rs` step of compiling `spirv-tools-sys`. It is not clear + // from the logged error what the problem is. For now we'll just run a full build + // outside the tests environment, see `xtask`'s `test-build`. + } else { + panic!("was not a build command"); + } + } +} diff --git a/crates/cargo-gpu/src/config.rs b/crates/cargo-gpu/src/config.rs new file mode 100644 index 00000000000..37bcd84bf01 --- /dev/null +++ b/crates/cargo-gpu/src/config.rs @@ -0,0 +1,251 @@ +//! Manage and merge the various sources of config: shader crate's `Cargo.toml`(s) and CLI args. +use anyhow::Context as _; +use clap::Parser as _; + +use crate::metadata::MetadataCache; + +/// Config +pub struct Config; + +impl Config { + /// Get all the config defaults as JSON. + pub fn defaults_as_json() -> anyhow::Result { + let defaults_json = Self::cli_args_to_json(vec![String::new()])?; + Ok(defaults_json) + } + + /// Convert CLI args to their serde JSON representation. + fn cli_args_to_json(env_args: Vec) -> anyhow::Result { + Ok(serde_json::to_value(crate::build::Build::parse_from( + env_args, + ))?) + } + + /// Config for the `cargo gpu build` and `cargo gpu install` can be set in the shader crate's + /// `Cargo.toml`, so here we load that config first as the base config, and the CLI arguments can + /// then later override it. + pub fn clap_command_with_cargo_config( + shader_crate_path: &std::path::Path, + mut env_args: Vec, + metadata: &mut MetadataCache, + ) -> anyhow::Result { + let mut config = metadata.as_json(shader_crate_path)?; + + env_args.retain(|arg| { + !(arg == "build" || arg == "install" || arg == "check" || arg == "clippy") + }); + let cli_args_json = Self::cli_args_to_json(env_args)?; + Self::json_merge(&mut config, cli_args_json, None)?; + + let args = serde_json::from_value::(config)?; + Ok(args) + } + + /// Merge 2 JSON objects. But only if the incoming patch value isn't the default value. + /// Inspired by: + pub fn json_merge( + left_in: &mut serde_json::Value, + right_in: serde_json::Value, + maybe_pointer: Option<&String>, + ) -> anyhow::Result<()> { + let defaults = Self::defaults_as_json()?; + + match (left_in, right_in) { + (left @ &mut serde_json::Value::Object(_), serde_json::Value::Object(right)) => { + let left_as_object = left + .as_object_mut() + .context("Unreachable, we've already proved it's an object")?; + for (key, value) in right { + let new_pointer = maybe_pointer.as_ref().map_or_else( + || format!("/{}", key.clone()), + |pointer| format!("{}/{}", pointer, key.clone()), + ); + Self::json_merge( + left_as_object + .entry(key.clone()) + .or_insert(serde_json::Value::Null), + value, + Some(&new_pointer), + )?; + } + } + (left, right) => { + if let Some(pointer) = maybe_pointer { + let default = defaults.pointer(pointer).context(format!( + "Configuration option with path `{pointer}` was not found in the default configuration, \ + which is:\ndefaults: {defaults:#?}" + ))?; + if &right != default { + // Only overwrite if the new value differs from the defaults. + *left = right; + } + } + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::spirv_builder::Capability; + use crate::test::TestEnv; + use std::io::Write as _; + + #[test_log::test] + fn booleans_from_cli() { + let _env = TestEnv::new(); + let shader_crate_path = _env.setup_shader_crate().unwrap(); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![ + "gpu".to_owned(), + "build".to_owned(), + "--debug".to_owned(), + "--auto-install-rust-toolchain".to_owned(), + ], + &mut MetadataCache::default(), + ) + .unwrap(); + assert!(!args.build.spirv_builder.release); + assert!(args.install.auto_install_rust_toolchain); + } + + #[test_log::test] + fn booleans_from_cargo() { + let _env = TestEnv::new(); + let shader_crate_path = _env + .setup_shader_crate_with_cargo_toml(|file| { + file.write_all( + [ + "[package.metadata.rust-gpu.build]", + "release = false", + "[package.metadata.rust-gpu.install]", + "auto-install-rust-toolchain = true", + ] + .join("\n") + .as_bytes(), + ) + }) + .unwrap(); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![], + &mut MetadataCache::default(), + ) + .unwrap(); + assert!(!args.build.spirv_builder.release); + assert!(args.install.auto_install_rust_toolchain); + } + + fn update_cargo_output_dir(_env: &TestEnv) -> std::path::PathBuf { + _env.setup_shader_crate_with_cargo_toml(|file| { + file.write_all( + [ + "[package.metadata.rust-gpu.build]", + "output-dir = \"/the/moon\"", + ] + .join("\n") + .as_bytes(), + ) + }) + .unwrap() + } + + #[test_log::test] + fn string_from_cargo() { + let _env = TestEnv::new(); + let shader_crate_path = update_cargo_output_dir(&_env); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![], + &mut MetadataCache::default(), + ) + .unwrap(); + if cfg!(target_os = "windows") { + assert!( + args.build + .output_dir + .to_str() + .unwrap() + .ends_with("/the/moon"), + "Actual: {:?}", + args.build.output_dir + ); + } else { + assert_eq!(args.build.output_dir, std::path::Path::new("/the/moon")); + } + } + + #[test_log::test] + fn string_from_cargo_overwritten_by_cli() { + let _env = TestEnv::new(); + let shader_crate_path = update_cargo_output_dir(&_env); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![ + "gpu".to_owned(), + "build".to_owned(), + "--output-dir".to_owned(), + "/the/river".to_owned(), + ], + &mut MetadataCache::default(), + ) + .unwrap(); + assert_eq!(args.build.output_dir, std::path::Path::new("/the/river")); + } + + #[test_log::test] + fn arrays_from_cargo() { + let _env = TestEnv::new(); + let shader_crate_path = _env + .setup_shader_crate_with_cargo_toml(|file| { + file.write_all( + [ + "[package.metadata.rust-gpu.build]", + "capabilities = [\"AtomicStorage\", \"Matrix\"]", + ] + .join("\n") + .as_bytes(), + ) + }) + .unwrap(); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![], + &mut MetadataCache::default(), + ) + .unwrap(); + assert_eq!( + args.build.spirv_builder.capabilities, + vec![Capability::AtomicStorage, Capability::Matrix] + ); + } + + #[test_log::test] + fn rename_manifest_parse() { + let _env = TestEnv::new(); + let shader_crate_path = _env.setup_shader_crate().unwrap(); + + let args = Config::clap_command_with_cargo_config( + &shader_crate_path, + vec![ + "gpu".to_owned(), + "build".to_owned(), + "--manifest-file".to_owned(), + "mymanifest".to_owned(), + ], + &mut MetadataCache::default(), + ) + .unwrap(); + assert_eq!(args.build.manifest_file, "mymanifest".to_owned()); + } +} diff --git a/crates/cargo-gpu/src/dump_usage.rs b/crates/cargo-gpu/src/dump_usage.rs new file mode 100644 index 00000000000..4dd002af2ae --- /dev/null +++ b/crates/cargo-gpu/src/dump_usage.rs @@ -0,0 +1,51 @@ +//! Convenience function for internal use. Dumps all the CLI usage instructions. Useful for +//! updating the README. + +use crate::{Cli, user_output}; + +/// main dump usage function +pub fn dump_full_usage_for_readme() -> anyhow::Result<()> { + use clap::CommandFactory as _; + let mut command = Cli::command(); + + let mut buffer: Vec = Vec::default(); + command.build(); + + write_help(&mut buffer, &mut command, 0)?; + user_output!("{}", String::from_utf8(buffer)?); + + Ok(()) +} + +/// Recursive function to print the usage instructions for each subcommand. +fn write_help( + buffer: &mut impl std::io::Write, + cmd: &mut clap::Command, + depth: usize, +) -> anyhow::Result<()> { + if cmd.get_name() == "help" { + return Ok(()); + } + + let mut command = cmd.get_name().to_owned(); + let indent_depth = if depth == 0 || depth == 1 { 0 } else { depth }; + let indent = " ".repeat(indent_depth * 4); + writeln!( + buffer, + "\n{}* {}{}", + indent, + command.remove(0).to_uppercase(), + command + )?; + + for line in cmd.render_long_help().to_string().lines() { + writeln!(buffer, "{indent} {line}")?; + } + + for sub in cmd.get_subcommands_mut() { + writeln!(buffer)?; + write_help(buffer, sub, depth + 1)?; + } + + Ok(()) +} diff --git a/crates/cargo-gpu/src/lib.rs b/crates/cargo-gpu/src/lib.rs new file mode 100644 index 00000000000..eadde1c8d82 --- /dev/null +++ b/crates/cargo-gpu/src/lib.rs @@ -0,0 +1,162 @@ +#![expect(clippy::pub_use, reason = "reexports from cargo_gpu_install crate")] + +//! Rust GPU shader crate builder. +//! +//! This program and library allows you to easily compile your rust-gpu shaders, +//! without requiring you to fix your entire project to a specific toolchain. +//! +//! # How it works +//! +//! This program primarily manages installations of `rustc_codegen_spirv`, the +//! codegen backend of rust-gpu to generate SPIR-V shader binaries. The codegen +//! backend builds on internal, ever-changing interfaces of rustc, which requires +//! fixing a version of rust-gpu to a specific version of the rustc compiler. +//! Usually, this would require you to fix your entire project to that specific +//! toolchain, but this project loosens that requirement by managing installations +//! of `rustc_codegen_spirv` and their associated toolchains for you. +//! +//! We continue to use rust-gpu's `spirv_builder` crate to pass the many additional +//! parameters required to configure rustc and our codegen backend, but provide you +//! with a toolchain agnostic version that you may use from stable rustc. And a +//! `cargo gpu` cmdline utility to simplify shader building even more. +//! +//! ## Where the binaries are +//! +//! We store our prebuild `rustc_spirv_builder` binaries in the default cache +//! directory of your OS: +//! * Windows: `C:/users//AppData/Local/rust-gpu` +//! * Mac: `~/Library/Caches/rust-gpu` +//! * Linux: `~/.cache/rust-gpu` +//! +//! ## How we build the backend +//! +//! * retrieve the version of rust-gpu you want to use based on the version of the +//! `spirv-std` dependency in your shader crate. +//! * create a dummy project at `/codegen//` that depends on +//! `rustc_codegen_spirv` +//! * use `cargo metadata` to `cargo update` the dummy project, which downloads the +//! `rustc_codegen_spirv` crate into cargo's cache, and retrieve the path to the +//! download location. +//! * search for the required toolchain in `build.rs` of `rustc_codegen_spirv` +//! * build it with the required toolchain version +//! * copy out the binary and clean the target dir +//! +//! ## Building shader crates +//! +//! `cargo-gpu` takes a path to a shader crate to build, as well as a path to a directory +//! to put the compiled `spv` source files. It also takes a path to an output manifest +//! file where all shader entry points will be mapped to their `spv` source files. This +//! manifest file can be used by build scripts (`build.rs` files) to generate linkage or +//! conduct other post-processing, like converting the `spv` files into `wgsl` files, +//! for example. + +#[cfg(test)] +pub use cargo_gpu_install::test; +pub use cargo_gpu_install::{cache_dir, install, spirv_builder, spirv_source, user_output}; +pub use metadata::MetadataCache; + +mod build; +mod config; +mod dump_usage; +mod linkage; +mod lockfile; +mod metadata; +mod show; + +/// All of the available subcommands for `cargo gpu` +#[derive(clap::Subcommand)] +#[non_exhaustive] +pub enum Command { + /// Install rust-gpu compiler artifacts. + Install(Box), + + /// Compile a shader crate to SPIR-V. + Build(Box), + + /// Run `cargo check` on the shader crate with a SPIR-V target without building the actual shaders + Check(Box), + + /// Run clippy on a shader crate with a SPIR-V target + Clippy(Box), + + /// Show some useful values. + Show(show::Show), + + /// A hidden command that can be used to recursively print out all the subcommand help messages: + /// `cargo gpu dump-usage` + /// Useful for updating the README. + #[clap(hide(true))] + DumpUsage, +} + +impl Command { + /// Runs the command + /// + /// # Errors + /// Any errors during execution, usually printed to the user + #[inline] + pub fn run( + &self, + env_args: Vec, + metadata_cache: &mut metadata::MetadataCache, + ) -> anyhow::Result<()> { + match &self { + Self::Install(install) => { + let shader_crate_path = &install.shader_crate; + let command = config::Config::clap_command_with_cargo_config( + shader_crate_path, + env_args, + metadata_cache, + )?; + log::debug!( + "installing with final merged arguments: {:#?}", + command.install + ); + command.install.run()?; + } + Self::Build(build) | Self::Check(build) | Self::Clippy(build) => { + let shader_crate_path = &build.install.shader_crate; + let mut command = config::Config::clap_command_with_cargo_config( + shader_crate_path, + env_args, + metadata_cache, + )?; + #[expect(clippy::wildcard_enum_match_arm, reason = "unreachable")] + match self { + Self::Check(_) => { + command.build.spirv_builder.cargo_cmd = Some("check".into()); + command.build.allow_no_artifacts = true; + } + Self::Clippy(_) => { + command.build.spirv_builder.cargo_cmd = Some("clippy".into()); + command.build.allow_no_artifacts = true; + } + _ => {} + } + log::debug!("building with final merged arguments: {command:#?}"); + + if command.build.watch { + // When watching, do one normal run to setup the `manifest.json` file. + command.build.watch = false; + command.run()?; + command.build.watch = true; + } + command.run()?; + } + Self::Show(show) => show.run()?, + Self::DumpUsage => dump_usage::dump_full_usage_for_readme()?, + } + + Ok(()) + } +} + +/// the Cli struct representing the main cli +#[derive(clap::Parser)] +#[clap(author, version, about, subcommand_required = true)] +#[non_exhaustive] +pub struct Cli { + /// The command to run. + #[clap(subcommand)] + pub command: Command, +} diff --git a/crates/cargo-gpu/src/linkage.rs b/crates/cargo-gpu/src/linkage.rs new file mode 100644 index 00000000000..346bdaf32cc --- /dev/null +++ b/crates/cargo-gpu/src/linkage.rs @@ -0,0 +1,30 @@ +//! Mainly for the Linkage struct, which is written to a json file. + +/// Shader source and entry point that can be used to create shader linkage. +#[derive(serde::Serialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Linkage { + /// File path to the entry point's source file + pub source_path: String, + /// Name of the entry point for spirv and vulkan + pub entry_point: String, + /// Name of the entry point for wgsl, where `::` characters have been removed + pub wgsl_entry_point: String, +} + +impl Linkage { + /// Make a new `Linkage` from an entry point and source path + #[expect(clippy::impl_trait_in_params, reason = "just a struct new")] + pub fn new(entry_point: impl AsRef, source_path: impl AsRef) -> Self { + Self { + // Force a forward slash convention here so it works on all OSs + source_path: source_path + .as_ref() + .components() + .map(|comp| comp.as_os_str().to_string_lossy()) + .collect::>() + .join("/"), + wgsl_entry_point: entry_point.as_ref().replace("::", ""), + entry_point: entry_point.as_ref().to_owned(), + } + } +} diff --git a/crates/cargo-gpu/src/lockfile.rs b/crates/cargo-gpu/src/lockfile.rs new file mode 100644 index 00000000000..89aaad6f02a --- /dev/null +++ b/crates/cargo-gpu/src/lockfile.rs @@ -0,0 +1,274 @@ +//! Handles lockfile version conflicts and downgrades. Stable uses lockfile v4, but rust-gpu +//! v0.9.0 uses an old toolchain requiring v3 and will refuse to build with a v4 lockfile being +//! present. This module takes care of warning the user and potentially downgrading the lockfile. + +use crate::spirv_builder::query_rustc_version; +use anyhow::Context as _; +use semver::Version; +use std::io::Write as _; + +/// `Cargo.lock` manifest version 4 became the default in Rust 1.83.0. Conflicting manifest +/// versions between the workspace and the shader crate, can cause problems. +const RUST_VERSION_THAT_USES_V4_CARGO_LOCKS: Version = Version::new(1, 83, 0); + +/// Cargo dependency for `spirv-builder` and the rust toolchain channel. +#[derive(Debug, Clone)] +pub struct LockfileMismatchHandler { + /// `Cargo.lock`s that have had their manifest versions changed by us and need changing back. + pub cargo_lock_files_with_changed_manifest_versions: Vec, +} + +impl LockfileMismatchHandler { + /// Create instance + pub fn new( + shader_crate_path: &std::path::Path, + toolchain_channel: &str, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> anyhow::Result { + let mut cargo_lock_files_with_changed_manifest_versions = vec![]; + + let maybe_shader_crate_lock = + Self::ensure_workspace_rust_version_doesnt_conflict_with_shader( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("ensure_workspace_rust_version_doesnt_conflict_with_shader")?; + + if let Some(shader_crate_lock) = maybe_shader_crate_lock { + cargo_lock_files_with_changed_manifest_versions.push(shader_crate_lock); + } + + let maybe_workspace_crate_lock = + Self::ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks( + shader_crate_path, + toolchain_channel, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks")?; + + if let Some(workspace_crate_lock) = maybe_workspace_crate_lock { + cargo_lock_files_with_changed_manifest_versions.push(workspace_crate_lock); + } + + Ok(Self { + cargo_lock_files_with_changed_manifest_versions, + }) + } + + /// See docs for `force_overwrite_lockfiles_v4_to_v3` flag for why we do this. + fn ensure_workspace_rust_version_doesnt_conflict_with_shader( + shader_crate_path: &std::path::Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> anyhow::Result> { + log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from workspace Rust..."); + let workspace_rust_version = query_rustc_version(None).context("reading rustc version")?; + if workspace_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { + log::debug!( + "user's Rust is v{workspace_rust_version}, so no v3/v4 conflicts possible." + ); + return Ok(None); + } + + Self::handle_conflicting_cargo_lock_v4( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("handling v4/v3 conflict")?; + + if is_force_overwrite_lockfiles_v4_to_v3 { + Ok(Some(shader_crate_path.join("Cargo.lock"))) + } else { + Ok(None) + } + } + + /// See docs for `force_overwrite_lockfiles_v4_to_v3` flag for why we do this. + fn ensure_shader_rust_version_doesnt_conflict_with_any_cargo_locks( + shader_crate_path: &std::path::Path, + channel: &str, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> anyhow::Result> { + log::debug!("Ensuring no v3/v4 `Cargo.lock` conflicts from shader's Rust..."); + let shader_rust_version = + query_rustc_version(Some(channel)).context("getting rustc version")?; + if shader_rust_version >= RUST_VERSION_THAT_USES_V4_CARGO_LOCKS { + log::debug!("shader's Rust is v{shader_rust_version}, so no v3/v4 conflicts possible."); + return Ok(None); + } + + log::debug!( + "shader's Rust is v{shader_rust_version}, so checking both shader and workspace `Cargo.lock` manifest versions..." + ); + + if shader_crate_path.join("Cargo.lock").exists() { + // Note that we don't return the `Cargo.lock` here (so that it's marked for reversion + // after the build), because we can be sure that updating it now is actually updating it + // to the state it should have been all along. Therefore it doesn't need reverting once + // fixed. + Self::handle_conflicting_cargo_lock_v4( + shader_crate_path, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("handling v4/v3 conflict")?; + } + + if let Some(workspace_root) = + Self::get_workspace_root(shader_crate_path).context("reading workspace root")? + { + Self::handle_conflicting_cargo_lock_v4( + workspace_root, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("handling conflicting cargo v4")?; + return Ok(Some(workspace_root.join("Cargo.lock"))); + } + + Ok(None) + } + + /// Get the path to the shader crate's workspace, if it has one. We can't use the traditional + /// `cargo metadata` because if the workspace has a conflicting `Cargo.lock` manifest version + /// then that command won't work. Instead we do an old school recursive file tree walk. + fn get_workspace_root( + shader_crate_path: &std::path::Path, + ) -> anyhow::Result> { + let shader_cargo_toml = std::fs::read_to_string(shader_crate_path.join("Cargo.toml")) + .with_context(|| format!("reading Cargo.toml at {}", shader_crate_path.display()))?; + if !shader_cargo_toml.contains("workspace = true") { + return Ok(None); + } + + let mut current_path = shader_crate_path; + #[expect(clippy::default_numeric_fallback, reason = "It's just a loop")] + for _ in 0..15 { + if let Some(parent_path) = current_path.parent() { + if parent_path.join("Cargo.lock").exists() { + return Ok(Some(parent_path)); + } + current_path = parent_path; + } else { + break; + } + } + + Ok(None) + } + + /// When Rust < 1.83.0 is being used an error will occur if it tries to parse `Cargo.lock` + /// files that use lockfile manifest version 4. Here we check and handle that. + fn handle_conflicting_cargo_lock_v4( + folder: &std::path::Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> anyhow::Result<()> { + let shader_cargo_lock_path = folder.join("Cargo.lock"); + let shader_cargo_lock = std::fs::read_to_string(shader_cargo_lock_path.clone()) + .context("reading shader cargo lock")?; + let third_line = shader_cargo_lock.lines().nth(2).context("no third line")?; + if third_line.contains("version = 4") { + Self::handle_v3v4_conflict( + &shader_cargo_lock_path, + is_force_overwrite_lockfiles_v4_to_v3, + ) + .context("handling v4/v3 conflict")?; + return Ok(()); + } + if third_line.contains("version = 3") { + return Ok(()); + } + anyhow::bail!( + "Unrecognized `Cargo.lock` manifest version at: {}", + folder.display() + ) + } + + /// Handle conflicting `Cargo.lock` manifest versions by either overwriting the manifest + /// version or exiting with advice on how to handle the conflict. + fn handle_v3v4_conflict( + offending_cargo_lock: &std::path::Path, + is_force_overwrite_lockfiles_v4_to_v3: bool, + ) -> anyhow::Result<()> { + if !is_force_overwrite_lockfiles_v4_to_v3 { + Self::exit_with_v3v4_hack_suggestion(); + } + + Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "4", "3") + .context("replacing version 4 -> 3")?; + + Ok(()) + } + + /// Once all install and builds have completed put their manifest versions back to how they + /// were. + pub fn revert_cargo_lock_manifest_versions(&self) -> anyhow::Result<()> { + for offending_cargo_lock in &self.cargo_lock_files_with_changed_manifest_versions { + log::debug!("Reverting: {}", offending_cargo_lock.display()); + Self::replace_cargo_lock_manifest_version(offending_cargo_lock, "3", "4") + .context("replacing version 3 -> 4")?; + } + + Ok(()) + } + + /// Replace the manifest version, eg `version = 4`, in a `Cargo.lock` file. + fn replace_cargo_lock_manifest_version( + offending_cargo_lock: &std::path::Path, + from_version: &str, + to_version: &str, + ) -> anyhow::Result<()> { + log::warn!( + "Replacing manifest version 'version = {}' with 'version = {}' in: {}", + from_version, + to_version, + offending_cargo_lock.display() + ); + let old_contents = std::fs::read_to_string(offending_cargo_lock) + .context("reading offending Cargo.lock")?; + let new_contents = old_contents.replace( + &format!("\nversion = {from_version}\n"), + &format!("\nversion = {to_version}\n"), + ); + + let mut file = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(offending_cargo_lock) + .context("opening offending Cargo.lock")?; + file.write_all(new_contents.as_bytes())?; + + Ok(()) + } + + /// Exit and give the user advice on how to deal with the infamous v3/v4 Cargo lockfile version + /// problem. + #[expect(clippy::non_ascii_literal, reason = "It's CLI output")] + fn exit_with_v3v4_hack_suggestion() { + crate::user_output!( + "Conflicting `Cargo.lock` versions detected ⚠️\n\ + Because `cargo gpu` uses a dedicated Rust toolchain for compiling shaders\n\ + it's possible that the `Cargo.lock` manifest version of the shader crate\n\ + does not match the `Cargo.lock` manifest version of the workspace. This is\n\ + due to a change in the defaults introduced in Rust 1.83.0.\n\ + \n\ + One way to resolve this is to force the workspace to use the same version\n\ + of Rust as required by the shader. However that is not often ideal or even\n\ + possible. Another way is to exlude the shader from the workspace. This is\n\ + also not ideal if you have many shaders sharing config from the workspace.\n\ + \n\ + Therefore `cargo gpu build/install` offers a workaround with the argument:\n\ + --force-overwrite-lockfiles-v4-to-v3\n\ + \n\ + See `cargo gpu build --help` for more information.\n\ + " + ); + std::process::exit(1); + } +} + +impl Drop for LockfileMismatchHandler { + fn drop(&mut self) { + let result = self.revert_cargo_lock_manifest_versions(); + if let Err(error) = result { + log::error!("Couldn't revert some or all of the shader `Cargo.lock` files: {error}"); + } + } +} diff --git a/crates/cargo-gpu/src/main.rs b/crates/cargo-gpu/src/main.rs new file mode 100644 index 00000000000..cb094c76a1c --- /dev/null +++ b/crates/cargo-gpu/src/main.rs @@ -0,0 +1,41 @@ +//! main executable of cargo gpu +use cargo_gpu::Cli; +use clap::Parser as _; + +fn main() { + // Safety: always in single-threaded code + #[cfg(debug_assertions)] + unsafe { + std::env::set_var("RUST_BACKTRACE", "1"); + } + + env_logger::builder().init(); + + if let Err(error) = run() { + log::error!("{error:?}"); + + #[expect( + clippy::print_stderr, + reason = "Our central place for outputting error messages" + )] + { + eprintln!("Error: {error}"); + std::process::exit(1); + }; + } +} + +/// Wrappable "main" to catch errors. +fn run() -> anyhow::Result<()> { + let env_args = std::env::args() + .filter(|arg| { + // Calling our `main()` with the cargo subcommand `cargo gpu` passes "gpu" + // as the first parameter, so we want to ignore it. + arg != "gpu" + }) + .collect::>(); + log::trace!("CLI args: {env_args:#?}"); + let cli = Cli::parse_from(&env_args); + let mut metadata_cache = cargo_gpu::MetadataCache::default(); + cli.command.run(env_args, &mut metadata_cache) +} diff --git a/crates/cargo-gpu/src/metadata.rs b/crates/cargo-gpu/src/metadata.rs new file mode 100644 index 00000000000..00aeea13acc --- /dev/null +++ b/crates/cargo-gpu/src/metadata.rs @@ -0,0 +1,264 @@ +//! Get config from the shader crate's `Cargo.toml` `[*.metadata.rust-gpu.*]` + +use std::collections::HashMap; + +use anyhow::Context as _; +use cargo_metadata::MetadataCommand; +use serde_json::Value; + +/// A cache of metadata from various `Cargo.toml` files. +/// +/// "Metadata" refers to the `[metadata.*]` section of `Cargo.toml` that `cargo` formally +/// ignores so that packages can implement their own behaviour with it. +#[derive(Debug, Default)] +pub struct MetadataCache { + /// Cached result of `MetadataCommand::new().exec()`. + inner: HashMap, +} + +impl MetadataCache { + /// Return the cached cargo metadata for the Cargo.toml at the given path, + /// or find it, populate the cache with it and return it. + fn get_metadata( + &mut self, + maybe_path_to_manifest_dir: Option<&std::path::Path>, + ) -> anyhow::Result<&cargo_metadata::Metadata> { + let path = if let Some(path) = maybe_path_to_manifest_dir { + path.to_path_buf() + } else { + std::env::current_dir().context("cannot determine the current working directory")? + }; + + if !self.inner.contains_key(&path) { + let metadata = MetadataCommand::new().current_dir(&path).exec()?; + self.inner.insert(path.clone(), metadata); + } + + self.inner.get(&path).context("unreachable") + } + + /// Resolve a package name to a crate directory. + /// + /// ## Errors + /// * if fetching cargo metadata fails. + /// * if no packages are listed in the cargo metadata. + /// * if the manifest path has no parent. + pub fn resolve_package_to_shader_crate( + &mut self, + package: &str, + ) -> anyhow::Result { + log::debug!("resolving package '{package}' to shader crate"); + let metadata = self.get_metadata(None)?; + + let meta_package = metadata + .packages + .iter() + .find(|pkg| pkg.name.as_str() == package) + .context("Package not found in metadata")?; + let shader_crate_path: std::path::PathBuf = meta_package + .manifest_path + .parent() + .context("manifest is missing a parent directory")? + .to_path_buf() + .into(); + log::debug!( + " determined shader crate path to be '{}'", + shader_crate_path.display() + ); + Ok(shader_crate_path) + } + + /// Convert `rust-gpu`-specific sections in `Cargo.toml` to `clap`-compatible arguments. + /// The section in question is: `[package.metadata.rust-gpu.*]`. See the `shader-crate-template` + /// for an example. + /// + /// First we generate the CLI arg defaults as JSON. Then on top of those we merge any config + /// from the workspace `Cargo.toml`, then on top of those we merge any config from the shader + /// crate's `Cargo.toml`. + /// + /// ## Errors + /// Errors if cargo metadata cannot be found or if it cannot be operated on. + pub fn as_json(&mut self, path: &std::path::Path) -> anyhow::Result { + log::debug!("reading package metadata from {}", path.display()); + let cargo_json = self.get_cargo_toml_as_json(path)?; + let config = Self::merge_configs(&cargo_json, path)?; + Ok(config) + } + + /// Merge the various source of config: defaults, workspace and shader crate. + fn merge_configs( + cargo_json: &cargo_metadata::Metadata, + path: &std::path::Path, + ) -> anyhow::Result { + log::debug!("merging cargo metadata from {}", path.display()); + let mut metadata = crate::config::Config::defaults_as_json()?; + crate::config::Config::json_merge( + &mut metadata, + { + log::debug!("looking for workspace metadata"); + let ws_meta = Self::get_rust_gpu_from_metadata(&cargo_json.workspace_metadata); + log::trace!("workspace_metadata: {ws_meta:#?}"); + ws_meta + }, + None, + )?; + crate::config::Config::json_merge( + &mut metadata, + { + log::debug!("looking for crate metadata"); + let mut crate_meta = Self::get_crate_metadata(cargo_json, path)?; + log::trace!("crate_metadata: {crate_meta:#?}"); + if let Some(output_path) = crate_meta.pointer_mut("/build/output_dir") { + log::debug!("found output-dir path in crate metadata: {output_path:?}"); + if let Some(output_dir) = output_path.clone().as_str() { + let new_output_path = path.join(output_dir); + *output_path = Value::String(format!("{}", new_output_path.display())); + log::debug!( + "setting that to be relative to the Cargo.toml it was found in: {}", + new_output_path.display() + ); + } + } + crate_meta + }, + None, + )?; + + Ok(metadata) + } + + /// Convert a `Cargo.toml` to JSON + fn get_cargo_toml_as_json( + &mut self, + path: &std::path::Path, + ) -> anyhow::Result { + self.get_metadata(Some(path)).cloned() + } + + /// Get any `rust-gpu` metadata set in the crate's `Cargo.toml` + fn get_crate_metadata( + json: &cargo_metadata::Metadata, + path: &std::path::Path, + ) -> anyhow::Result { + let shader_crate_path = std::fs::canonicalize(path)?.join("Cargo.toml"); + + for package in &json.packages { + let manifest_path = std::fs::canonicalize(package.manifest_path.as_std_path())?; + log::debug!( + "Matching shader crate path with manifest path: '{}' == '{}'?", + shader_crate_path.display(), + manifest_path.display() + ); + if manifest_path == shader_crate_path { + log::debug!("...matches! Getting metadata"); + return Ok(Self::get_rust_gpu_from_metadata(&package.metadata)); + } + } + Ok(serde_json::json!({})) + } + + /// Get `rust-gpu` value from some metadata + fn get_rust_gpu_from_metadata(metadata: &Value) -> Value { + Self::keys_to_snake_case( + metadata + .pointer("/rust-gpu") + .cloned() + .unwrap_or(Value::Null), + ) + } + + /// Convert JSON keys from kebab case to snake case. Eg: `a-b` to `a_b`. + /// + /// Detection of keys for serde deserialization must match the case in the Rust structs. + /// However clap defaults to detecting CLI args in kebab case. So here we do the conversion. + #[expect(clippy::wildcard_enum_match_arm, reason = "we only want objects")] + fn keys_to_snake_case(json: Value) -> Value { + match json { + Value::Object(object) => Value::Object( + object + .into_iter() + .map(|(key, value)| (key.replace('-', "_"), Self::keys_to_snake_case(value))) + .collect(), + ), + other => other, + } + } +} + +#[expect( + clippy::indexing_slicing, + reason = "We don't need to be so strict in tests" +)] +#[cfg(test)] +mod test { + use super::*; + use std::path::Path; + + #[test_log::test] + fn generates_defaults() { + let mut metadata = MetadataCommand::new() + .current_dir(env!("CARGO_MANIFEST_DIR")) + .exec() + .unwrap(); + metadata.packages.first_mut().unwrap().metadata = serde_json::json!({}); + let configs = MetadataCache::merge_configs(&metadata, Path::new("./")).unwrap(); + assert_eq!(configs["build"]["release"], Value::Bool(true)); + assert_eq!( + configs["install"]["auto_install_rust_toolchain"], + Value::Bool(false) + ); + } + + #[test_log::test] + fn can_override_config_from_workspace_toml() { + let mut metadata = MetadataCommand::new() + .current_dir(env!("CARGO_MANIFEST_DIR")) + .exec() + .unwrap(); + metadata.workspace_metadata = serde_json::json!({ + "rust-gpu": { + "build": { + "release": false + }, + "install": { + "auto-install-rust-toolchain": true + } + } + }); + let configs = MetadataCache::merge_configs(&metadata, Path::new("./")).unwrap(); + assert_eq!(configs["build"]["release"], Value::Bool(false)); + assert_eq!( + configs["install"]["auto_install_rust_toolchain"], + Value::Bool(true) + ); + } + + #[test_log::test] + fn can_override_config_from_crate_toml() { + let mut metadata = MetadataCommand::new() + .current_dir(env!("CARGO_MANIFEST_DIR")) + .exec() + .unwrap(); + let cargo_gpu = metadata + .packages + .iter_mut() + .find(|package| package.name.contains("cargo-gpu")) + .unwrap(); + cargo_gpu.metadata = serde_json::json!({ + "rust-gpu": { + "build": { + "release": false + }, + "install": { + "auto-install-rust-toolchain": true + } + } + }); + let configs = MetadataCache::merge_configs(&metadata, Path::new(".")).unwrap(); + assert_eq!(configs["build"]["release"], Value::Bool(false)); + assert_eq!( + configs["install"]["auto_install_rust_toolchain"], + Value::Bool(true) + ); + } +} diff --git a/crates/cargo-gpu/src/show.rs b/crates/cargo-gpu/src/show.rs new file mode 100644 index 00000000000..491550cd944 --- /dev/null +++ b/crates/cargo-gpu/src/show.rs @@ -0,0 +1,80 @@ +//! Display various information about `cargo gpu`, eg its cache directory. + +use crate::cache_dir; +use crate::spirv_builder::Capability; +use crate::spirv_source::SpirvSource; + +/// Show the computed source of the spirv-std dependency. +#[derive(Clone, Debug, clap::Parser)] +pub struct SpirvSourceDep { + /// The location of the shader-crate to inspect to determine its spirv-std dependency. + #[clap(long, default_value = "./")] + pub shader_crate: std::path::PathBuf, +} + +/// Different tidbits of information that can be queried at the command line. +#[derive(Clone, Debug, clap::Subcommand)] +pub enum Info { + /// Displays the location of the cache directory + CacheDirectory, + /// The source location of spirv-std + SpirvSource(SpirvSourceDep), + /// The git commitsh of this cli tool. + Commitsh, + /// All the available SPIR-V capabilities that can be set with `--capabilities` + Capabilities, +} + +/// `cargo gpu show` +#[derive(clap::Parser)] +pub struct Show { + /// Display information about rust-gpu + #[clap(subcommand)] + command: Info, +} + +impl Show { + /// Entrypoint + pub fn run(&self) -> anyhow::Result<()> { + log::info!("{:?}: ", self.command); + + #[expect( + clippy::print_stdout, + reason = "The output of this command could potentially be used in a script, \ + so we _don't_ want to use `crate::user_output`, as that prefixes a crab." + )] + match &self.command { + Info::CacheDirectory => { + println!("{}\n", cache_dir()?.display()); + } + Info::SpirvSource(SpirvSourceDep { shader_crate }) => { + let rust_gpu_source = SpirvSource::get_rust_gpu_deps_from_shader(shader_crate)?; + println!("{rust_gpu_source}\n"); + } + Info::Commitsh => { + println!("{}", env!("GIT_HASH")); + } + Info::Capabilities => { + println!("All available options to the `cargo gpu build --capabilities` argument:"); + #[expect( + clippy::use_debug, + reason = "It's easier to just use `Debug` formatting than implementing `Display`" + )] + for capability in Self::capability_variants_iter() { + println!(" {capability:?}"); + } + } + } + + Ok(()) + } + + /// Iterator over all `Capability` variants. + fn capability_variants_iter() -> impl Iterator { + // Since spirv::Capability is repr(u32) we can iterate over + // u32s until some maximum + #[expect(clippy::as_conversions, reason = "We know all variants are repr(u32)")] + let last_capability = Capability::CacheControlsINTEL as u32; + (0..=last_capability).filter_map(Capability::from_u32) + } +} diff --git a/crates/shader-crate-template/.gitignore b/crates/shader-crate-template/.gitignore new file mode 100644 index 00000000000..d2cea260095 --- /dev/null +++ b/crates/shader-crate-template/.gitignore @@ -0,0 +1,4 @@ +/target +/shaders +manifest.json +rust_gpu_shader_crate_template.spv diff --git a/crates/shader-crate-template/Cargo.lock b/crates/shader-crate-template/Cargo.lock new file mode 100644 index 00000000000..d0c753c3145 --- /dev/null +++ b/crates/shader-crate-template/Cargo.lock @@ -0,0 +1,112 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "glam" +version = "0.30.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12d847aeb25f41be4c0ec9587d624e9cd631bc007a8fd7ce3f5851e064c6460" +dependencies = [ + "libm", +] + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rust-gpu-shader-crate-template" +version = "0.1.0" +dependencies = [ + "glam", + "spirv-std", +] + +[[package]] +name = "spirv-std" +version = "0.9.0" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=877bd8697a15f3e6d09446a5e1807e6237ca1dac#877bd8697a15f3e6d09446a5e1807e6237ca1dac" +dependencies = [ + "bitflags", + "glam", + "libm", + "num-traits", + "spirv-std-macros", + "spirv-std-types", +] + +[[package]] +name = "spirv-std-macros" +version = "0.9.0" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=877bd8697a15f3e6d09446a5e1807e6237ca1dac#877bd8697a15f3e6d09446a5e1807e6237ca1dac" +dependencies = [ + "proc-macro2", + "quote", + "spirv-std-types", + "syn", +] + +[[package]] +name = "spirv-std-types" +version = "0.9.0" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=877bd8697a15f3e6d09446a5e1807e6237ca1dac#877bd8697a15f3e6d09446a5e1807e6237ca1dac" + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" diff --git a/crates/shader-crate-template/Cargo.toml b/crates/shader-crate-template/Cargo.toml new file mode 100644 index 00000000000..26a9b82d75a --- /dev/null +++ b/crates/shader-crate-template/Cargo.toml @@ -0,0 +1,108 @@ +[package] +name = "rust-gpu-shader-crate-template" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unexpected_cfgs = { level = "allow", check-cfg = ['cfg(target_arch, values("spirv"))'] } + +# Dependencies for CPU and GPU code +[dependencies] +# TODO: use a simple crate version once v0.10.0 is released +spirv-std = { git = "https://github.com/Rust-GPU/rust-gpu", rev = "877bd8697a15f3e6d09446a5e1807e6237ca1dac" } +glam = { version = "0.30.8", default-features = false } + +[package.metadata.rust-gpu.build] +# Where to output the compiled shader. Defaults to where `cargo gpu` is called from. +# TODO: Should it default to the root of the shader crate? +output-dir = "./" +# Set shader crate's cargo default-features +default_features = true +# Set shader crate's cargo features. +features = [] +# The compile target. +# TODO: `cargo gpu show targets` for all available options. +target = "spirv-unknown-vulkan1.2" +# Treat warnings as errors during compilation. +deny-warnings = false +# Compile shaders in debug mode. +release = true +# Enables the provided SPIR-V capabilities. +# See: `impl core::str::FromStr for spirv_builder::Capability`. +# TODO: `cargo gpu show capabilities` for all available options. +capabilities = [] +# Enables the provided SPIR-V extensions. +# See https://github.com/KhronosGroup/SPIRV-Registry for all extensions +# TODO: `cargo gpu show extensions` for all available options. +extensions = [] +# Compile one .spv file per shader entry point. +multimodule = false +# Set the level of metadata included in the SPIR-V binary. +# Options: "None", "NameVariables", "Full". +spirv-metadata = "None" +# Allow store from one struct type to a different type with compatible layout and members. +relax-struct-store = false +# Allow allocating an object of a pointer type and returning a pointer value from a function +# in logical addressing mode. +relax-logical-pointer = false +# Enable VK_KHR_relaxed_block_layout when checking standard uniform, storage buffer, and push +# constant layouts. +# This is the default when targeting Vulkan 1.1 or later. +relax-block-layout = false +# Enable VK_KHR_uniform_buffer_standard_layout when checking standard uniform buffer layouts. +uniform-buffer-standard-layout = false +# Enable `VK_EXT_scalar_block_layout` when checking standard uniform, storage buffer, and push +# constant layouts. +# Scalar layout rules are more permissive than relaxed block layout so in effect this will +# override the `relax_block_layout` option. +scalar-block-layout = false +# Skip checking standard uniform/storage buffer layout. +# Overrides `relax_block_layout` and `scalar_block_layout`. +skip-block-layout = false +# Preserve unused descriptor bindings. Useful for reflection. +preserve-bindings = false +# Renames the manifest.json file to the given string. Useful if you collect all your SPIR-V fragments +# in one place. +manifest-file = "manifest.json" + +[package.metadata.rust-gpu.install] +# Source of `spirv-builder` dependency +# Eg: "https://github.com/Rust-GPU/rust-gpu" +# spirv_builder_source = "" + +# Version of `spirv-builder` dependency. +# * If `--spirv-builder-source` is not set, then this is assumed to be a crates.io semantic +# version such as "0.9.0". +# * If `--spirv-builder-source` is set, then this is assumed to be a Git "commitsh", such +# as a Git commit hash or a Git tag, therefore anything that `git checkout` can resolve. +# spirv_builder_version = "" + +# Whether to assume "yes" to the "Install Rust toolchain: [y/n]" prompt. +auto-install-rust-toolchain = false +# Force `rustc_codegen_spirv` to be rebuilt. +rebuild_codegen = false +# There is a tricky situation where a shader crate that depends on workspace config can have +# a different `Cargo.lock` lockfile version from the the workspace's `Cargo.lock`. This can +# prevent builds when an old Rust toolchain doesn't recognise the newer lockfile version. +# +# The ideal way to resolve this would be to match the shader crate's toolchain with the +# workspace's toolchain. However, that is not always possible. Another solution is to +# `exclude = [...]` the problematic shader crate from the workspace. This also may not be a +# suitable solution if there are a number of shader crates all sharing similar config and +# you don't want to have to copy/paste and maintain that config across all the shaders. +# +# So a somewhat hacky workaround is to have `cargo gpu` overwrite lockfile versions. Enabling +# this flag will only come into effect if there are a mix of v3/v4 lockfiles. It will also +# only overwrite versions for the duration of a build. It will attempt to return the versions +# to their original values once the build is finished. However, of course, unexpected errors +# can occur and the overwritten values can remain. Hence why this behaviour is not enabled by +# default. +# +# This hack is possible because the change from v3 to v4 only involves a minor change to the +# way source URLs are encoded. See these PRs for more details: +# * https://github.com/rust-lang/cargo/pull/12280 +# * https://github.com/rust-lang/cargo/pull/14595 +force-overwrite-lockfiles-v4-to-v3 = false diff --git a/crates/shader-crate-template/README.md b/crates/shader-crate-template/README.md new file mode 100644 index 00000000000..847967e4842 --- /dev/null +++ b/crates/shader-crate-template/README.md @@ -0,0 +1,5 @@ +# shader-crate + +This is a shader crate that can be compiled to SPIR-V using `rust-gpu`. + +This crate can also be used from CPU Rust code, just like any other crate. diff --git a/crates/shader-crate-template/src/lib.rs b/crates/shader-crate-template/src/lib.rs new file mode 100644 index 00000000000..eb547f4719d --- /dev/null +++ b/crates/shader-crate-template/src/lib.rs @@ -0,0 +1,41 @@ +//! Shader entry points. +//! +//! Contains an example vertex shader, fragment shader. +#![no_std] +use spirv_std::glam::{Vec2, Vec4}; +use spirv_std::spirv; + +pub const CLIP_SPACE_COORD_QUAD_CCW: [Vec4; 6] = { + let tl = Vec4::new(-1.0, 1.0, 0.5, 1.0); + let tr = Vec4::new(1.0, 1.0, 0.5, 1.0); + let bl = Vec4::new(-1.0, -1.0, 0.5, 1.0); + let br = Vec4::new(1.0, -1.0, 0.5, 1.0); + [bl, br, tr, tr, tl, bl] +}; + +pub const UV_COORD_QUAD_CCW: [Vec2; 6] = { + let tl = Vec2::new(0.0, 0.0); + let tr = Vec2::new(1.0, 0.0); + let bl = Vec2::new(0.0, 1.0); + let br = Vec2::new(1.0, 1.0); + [bl, br, tr, tr, tl, bl] +}; + +/// Vertex shader that renders an implicit quad. +#[spirv(vertex)] +pub fn vertex( + #[spirv(vertex_index)] vertex_id: u32, + out_uv: &mut Vec2, + #[spirv(position)] clip_pos: &mut Vec4, +) { + let index = vertex_id as usize % 6; + *out_uv = UV_COORD_QUAD_CCW[index]; + *clip_pos = CLIP_SPACE_COORD_QUAD_CCW[index]; +} + +/// Fragment shader that uses UV coords passed in from the vertex shader +/// to render a simple gradient. +#[spirv(fragment)] +pub fn fragment(in_uv: Vec2, frag_color: &mut Vec4) { + *frag_color = Vec4::new(in_uv.x, in_uv.y, 0.0, 1.0); +} diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml new file mode 100644 index 00000000000..979a50f06fc --- /dev/null +++ b/crates/xtask/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "xtask" +version = "0.1.0" +edition.workspace = true +repository.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +clap.workspace = true +env_logger.workspace = true +log.workspace = true +tempfile.workspace = true +toml.workspace = true +regex-lite.workspace = true + +[lints] +workspace = true diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs new file mode 100644 index 00000000000..5f5c3cd2552 --- /dev/null +++ b/crates/xtask/src/main.rs @@ -0,0 +1,359 @@ +//! Project/repository utilities. +#![allow( + clippy::shadow_reuse, + clippy::unwrap_used, + clippy::unwrap_in_result, + reason = "This is just a workflow tool" +)] + +use anyhow::Context as _; +use clap::Parser as _; +use std::borrow::Cow; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +/// Path to the shader crate +const SHADER_CRATE_PATH: &str = "crates/shader-crate-template"; + +/// Our xtask commands. +#[derive(Debug, clap::Parser)] +enum Cli { + /// Run a test build of the shader-crate-template project. + TestBuild { + /// Build using the specified version of `spirv-std`. + #[clap(long)] + rust_gpu_version: Option, + /// The version of glam to use + #[clap(long)] + glam_version: Option, + }, + /// Set a dependency in the shader-crate-template to some version + SetDependency { + /// the dependency to modify + package: String, + /// the version to set it to + version: String, + /// the git repo to use, if version is a commit rev + #[clap(long)] + git: Option, + }, + UpdateExpect, + RustGpuRev { + rev: String, + }, +} + +/// run some cmd +fn cmd(args: impl IntoIterator>) -> anyhow::Result<()> { + let mut args = args.into_iter(); + let status = std::process::Command::new(args.next().context("no args")?.as_ref()) + .args(args) + .status() + .context("cmd failed")?; + anyhow::ensure!(status.success()); + Ok(()) +} + +/// Overwrites a toml file's output-dir field, and reverts that on drop. +struct ShaderCrateTemplateCargoTomlWriter { + /// Original string + original_shader_crate_template_str: String, + /// Original lockfile + original_shader_crate_lock_file: String, + /// Parsed toml table + table: toml::Table, + /// false will reset Cargo.toml when this is dropped + persistent: bool, +} + +impl Drop for ShaderCrateTemplateCargoTomlWriter { + fn drop(&mut self) { + if self.persistent { + return; + } + + log::info!("reverting overwrite of Cargo.toml"); + std::fs::write( + format!("{SHADER_CRATE_PATH}/Cargo.toml"), + &self.original_shader_crate_template_str, + ) + .unwrap(); + log::info!("reverting overwrite of Cargo.lock"); + std::fs::write( + format!("{SHADER_CRATE_PATH}/Cargo.lock"), + &self.original_shader_crate_lock_file, + ) + .unwrap(); + } +} + +impl Default for ShaderCrateTemplateCargoTomlWriter { + fn default() -> Self { + Self::new(false) + } +} + +impl ShaderCrateTemplateCargoTomlWriter { + /// Create a new one + fn new(persistent: bool) -> Self { + let original_shader_crate_template_str = + std::fs::read_to_string(format!("{SHADER_CRATE_PATH}/Cargo.toml")).unwrap(); + let table = toml::from_str::(&original_shader_crate_template_str).unwrap(); + let original_shader_crate_lock_file = + std::fs::read_to_string(format!("{SHADER_CRATE_PATH}/Cargo.lock")).unwrap(); + Self { + original_shader_crate_template_str, + original_shader_crate_lock_file, + table, + persistent, + } + } + + /// Get the `[dependencies]` section of the shader's `Cargo.toml`. + fn get_cargo_dependencies_table(&mut self) -> &mut toml::Table { + self.table + .get_mut("dependencies") + .unwrap() + .as_table_mut() + .unwrap() + } + + /// Get the `[package.metadata.rust-gpu.build]` section of the shader's `Cargo.toml`. + fn get_rust_gpu_table(&mut self) -> &mut toml::Table { + let package = self + .table + .get_mut("package") + .unwrap() + .as_table_mut() + .unwrap(); + let metadata = package.get_mut("metadata").unwrap().as_table_mut().unwrap(); + metadata + .get_mut("rust-gpu") + .unwrap() + .as_table_mut() + .unwrap() + } + + /// Write any temporary changes to the shader crate's `Cargo.toml` that are needed to run e2e + /// tests. + fn write_shader_crate_cargo_toml_changes(&self) -> anyhow::Result<()> { + std::fs::write( + format!("{SHADER_CRATE_PATH}/Cargo.toml"), + toml::to_string_pretty(&self.table).context("could not serialize")?, + ) + .context("could not overwrite path")?; + Ok(()) + } + + /// Replace the output-dir + fn replace_output_dir(&mut self, path: impl AsRef) -> anyhow::Result<()> { + let rust_gpu = self.get_rust_gpu_table(); + let build = rust_gpu.get_mut("build").unwrap().as_table_mut().unwrap(); + let output_dir = build.get_mut("output-dir").unwrap(); + *output_dir = toml::Value::String(format!("{}", path.as_ref().display())); + self.write_shader_crate_cargo_toml_changes()?; + Ok(()) + } + + /// Add or replace a dependency in the shader-crate-template + fn set_dependency(&mut self, package: &str, version: &DependencyVersion) -> anyhow::Result<()> { + let dependencies = self.get_cargo_dependencies_table(); + if let Some(value) = dependencies.get_mut(package) { + version.modify_toml(value); + self.write_shader_crate_cargo_toml_changes()?; + Ok(()) + } else { + anyhow::bail!("Crate `{package}` not found") + } + } + + /// Replace the `spirv-std` dependency version + fn set_spirv_std_version(&mut self, version: &str) -> anyhow::Result<()> { + self.set_dependency( + "spirv-std", + &DependencyVersion::parse( + version.into(), + Some("https://github.com/Rust-GPU/rust-gpu".into()), + )?, + ) + } + + /// Replace the `glam` dependency version + fn set_dependency_glam(&mut self, version: &str) -> anyhow::Result<()> { + self.set_dependency( + "glam", + &DependencyVersion::parse( + version.into(), + Some("https://github.com/bitshifter/glam-rs".into()), + )?, + ) + } +} + +/// The version of a dependency +#[non_exhaustive] +pub enum DependencyVersion { + /// Don't change anything, don't replace the dependency nor add it when it's not there. + Latest, + /// A version dependency for crates.io + Crates(String), + /// A git dependency on a specific rev + Git { + /// git repo + git: String, + /// git commit revision + rev: String, + }, +} + +impl DependencyVersion { + /// Try to parse a version from a string + /// + /// # Errors + /// if `version` is a commit rev, `git` must be specified + pub fn parse(version: String, git: Option) -> anyhow::Result { + if version == "latest" { + Ok(Self::Latest) + } else if version.contains('.') { + Ok(Self::Crates(version)) + } else { + Ok(Self::Git { + git: git.context("specifying a revision requires a git repo")?, + rev: version, + }) + } + } + + /// Convert this version to a toml value, may fail if we want the latest version + #[must_use] + pub fn to_toml(&self) -> Option { + match self { + Self::Latest => None, + Self::Crates(version) => Some(toml::Table::from_iter([( + "version".to_owned(), + toml::Value::String(version.clone()), + )])), + Self::Git { git, rev } => Some(toml::Table::from_iter([ + ("git".to_owned(), toml::Value::String(git.clone())), + ("rev".to_owned(), toml::Value::String(rev.clone())), + ])), + } + } + + /// Convert this version to a toml value, may fail if we want the latest version + pub fn modify_toml(&self, toml: &mut toml::Value) { + if let Some(mut table) = self.to_toml() { + let mut copy = |key: &str| { + if let Some(src_table) = toml.as_table_mut() + && let Some(value) = src_table.remove(key) + { + table.insert(key.to_owned(), value); + } + }; + copy("default-features"); + copy("features"); + *toml = toml::Value::Table(table); + } + } +} + +/// Run the xtask. +fn main() -> anyhow::Result<()> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .init(); + Cli::parse().run() +} + +impl Cli { + fn run(&self) -> anyhow::Result<()> { + match &self { + Cli::TestBuild { + rust_gpu_version, + glam_version, + } => { + log::info!("installing cargo gpu"); + cmd(["cargo", "install", "--path", "crates/cargo-gpu"])?; + + log::info!("setup project"); + let mut overwriter = ShaderCrateTemplateCargoTomlWriter::default(); + let dir = tempfile::TempDir::with_prefix("test-shader-output")?; + overwriter.replace_output_dir(dir.path())?; + if let Some(rust_gpu_version) = rust_gpu_version.as_ref() { + overwriter.set_spirv_std_version(rust_gpu_version)?; + } + if let Some(glam_version) = glam_version.as_ref() { + overwriter.set_dependency_glam(glam_version)?; + } + + log::info!("building with auto-install"); + cmd([ + "cargo", + "gpu", + "build", + "--shader-crate", + SHADER_CRATE_PATH, + "--auto-install-rust-toolchain", + "--rebuild-codegen", + "--force-overwrite-lockfiles-v4-to-v3", + ])?; + + cmd(["ls", "-lah", dir.path().to_str().unwrap()])?; + //NOTE: manifest.json is the default value here, which should be valid + cmd(["cat", dir.path().join("manifest.json").to_str().unwrap()])?; + } + Cli::SetDependency { + package, + version, + git, + } => { + let mut overwriter = ShaderCrateTemplateCargoTomlWriter::new(true); + overwriter.set_dependency( + package, + &DependencyVersion::parse(version.clone(), git.clone())?, + )?; + } + Cli::UpdateExpect => { + let status = std::process::Command::new("cargo") + .args(["nextest", "run"]) + .env("UPDATE_EXPECT", "1") + .status()?; + anyhow::ensure!(status.success()); + } + Cli::RustGpuRev { rev } => { + let root = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/../..")); + let rev_regex = regex_lite::Regex::new(r#"rev\s*=\s*"[0-9a-f]*""#)?; + let rev_replace = format!("rev = \"{rev}\""); + let replace_rev = |file: &Path, dep: &str| -> anyhow::Result<()> { + log::info!("patching file `{}` dep `{dep}`", file.display()); + let content = std::fs::read_to_string(file)?; + let content = content + // unlike `.lines()`, includes the `\n` or `\r\n` at the end of the line + .split_inclusive("\n") + .map(|line| { + if line.starts_with(dep) { + let replace = rev_regex.replace(line, &rev_replace); + assert!( + matches!(replace, Cow::Owned(..)), + "rev not found in line:\n{line}" + ); + replace + } else { + Cow::Borrowed(line) + } + }) + .collect::(); + std::fs::write(file, content.as_bytes())?; + Ok(()) + }; + replace_rev(&root.join("Cargo.toml"), "spirv-builder")?; + replace_rev( + &root.join("crates/shader-crate-template/Cargo.toml"), + "spirv-std", + )?; + Cli::UpdateExpect.run()?; + } + } + Ok(()) + } +} diff --git a/deny.toml b/deny.toml index e4feda2e211..4012eba98ac 100644 --- a/deny.toml +++ b/deny.toml @@ -39,6 +39,7 @@ allow = [ "BSD-2-Clause", "BSD-3-Clause", "ISC", + "MPL-2.0", "Zlib",