From 5d6ac65150a9c52822f154ec9371d41c31914dc0 Mon Sep 17 00:00:00 2001 From: Nripesh Niketan Date: Sun, 21 Jun 2026 17:52:33 +0100 Subject: [PATCH] Emit optional binary SPIR-V (.spv) via SPIRV-Tools When SPIRV-Tools is available (detected like the optional MLIR path), the Vulkan backend assembles the generated .spvasm text in-process (spvTextToBinary, SPV_ENV_VULKAN_1_2 matching the vulkan1.2/SPIR-V 1.5 target) and writes a sibling .spv binary. Fully optional: builds without SPIRV-Tools succeed and emit only .spvasm. Adds SPIRVAssembler (.h/.cpp) and a spirv-val-gated smoke test. --- CMakeLists.txt | 39 +++++++++ include/crossgl/Backend/SPIRVAssembler.h | 31 +++++++ src/Backend/SPIRVAssembler.cpp | 65 +++++++++++++++ src/Backend/VulkanBackend.cpp | 57 +++++++++++++ .../cmake/CrossGLVulkanNativeBuildTests.cmake | 29 +++++++ tests/cmake/VulkanSpirvToolsBinarySmoke.cmake | 82 +++++++++++++++++++ 6 files changed, 303 insertions(+) create mode 100644 include/crossgl/Backend/SPIRVAssembler.h create mode 100644 src/Backend/SPIRVAssembler.cpp create mode 100644 tests/cmake/VulkanSpirvToolsBinarySmoke.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 076e08a53..4cab7222e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,11 +20,44 @@ option(CROSSGL_ENABLE_MLIR_EXPERIMENTAL find_package(LLVM CONFIG QUIET) find_package(MLIR CONFIG QUIET) +find_package(SPIRV-Tools CONFIG QUIET) +if(NOT SPIRV-Tools_FOUND) + find_package(SPIRV-Tools-tools CONFIG QUIET) +endif() if(LLVM_FOUND) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}: ${LLVM_DIR}") endif() +# Optional SPIRV-Tools: when present, the Vulkan backend can assemble the +# generated SPIR-V text (.spvasm) into a real binary (.spv) in-process via the +# SPIRV-Tools C API. This mirrors the optional MLIR path -- a build WITHOUT +# SPIRV-Tools still succeeds and simply does not emit the sibling .spv. +if(SPIRV-Tools_FOUND) + if(TARGET SPIRV-Tools) + set(CROSSGL_SPIRV_TOOLS_TARGET SPIRV-Tools) + elseif(TARGET SPIRV-Tools-shared) + set(CROSSGL_SPIRV_TOOLS_TARGET SPIRV-Tools-shared) + elseif(TARGET SPIRV-Tools-static) + set(CROSSGL_SPIRV_TOOLS_TARGET SPIRV-Tools-static) + else() + set(CROSSGL_SPIRV_TOOLS_TARGET "") + endif() +endif() + +if(SPIRV-Tools_FOUND AND CROSSGL_SPIRV_TOOLS_TARGET) + message(STATUS + "Found SPIRV-Tools: ${SPIRV-Tools_DIR}; " + "binary .spv emission enabled via ${CROSSGL_SPIRV_TOOLS_TARGET}; " + "CROSSGL_HAVE_SPIRV_TOOLS=1") +else() + message(STATUS + "SPIRV-Tools not found; binary .spv emission disabled; " + "the .spvasm text artifact is still produced; " + "CROSSGL_HAVE_SPIRV_TOOLS=0") + set(CROSSGL_SPIRV_TOOLS_TARGET "") +endif() + if(MLIR_FOUND) message(STATUS "Found MLIR: ${MLIR_DIR}") set(CROSSGL_MLIR_FOUND_TEXT TRUE) @@ -128,6 +161,7 @@ add_library(crossgl_compiler src/Backend/MetalBackend.cpp src/Backend/OpenGLBackend.cpp src/Backend/ResourceArrays.cpp + src/Backend/SPIRVAssembler.cpp src/Backend/SPIRVModule.cpp src/Backend/StorageBufferSupport.cpp src/Backend/Target.cpp @@ -202,6 +236,11 @@ else() target_compile_definitions(crossgl_compiler PUBLIC CROSSGL_HAS_MLIR=0) endif() +if(SPIRV-Tools_FOUND AND CROSSGL_SPIRV_TOOLS_TARGET) + target_compile_definitions(crossgl_compiler PRIVATE CROSSGL_HAVE_SPIRV_TOOLS=1) + target_link_libraries(crossgl_compiler PRIVATE ${CROSSGL_SPIRV_TOOLS_TARGET}) +endif() + if(CROSSGL_ENABLE_MLIR_EXPERIMENTAL) target_compile_definitions(crossgl_compiler PUBLIC CROSSGL_ENABLE_MLIR_EXPERIMENTAL=1) diff --git a/include/crossgl/Backend/SPIRVAssembler.h b/include/crossgl/Backend/SPIRVAssembler.h new file mode 100644 index 000000000..b22b6e254 --- /dev/null +++ b/include/crossgl/Backend/SPIRVAssembler.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace crossgl { + +// Optional in-process SPIR-V assembly backed by the SPIRV-Tools C API. +// +// When the project is built WITH SPIRV-Tools (CROSSGL_HAVE_SPIRV_TOOLS=1), the +// Vulkan backend can assemble the SPIR-V text it generates (.spvasm) into a real +// binary module (.spv) without shelling out to the spirv-as CLI. When SPIRV-Tools +// is absent these functions compile to stubs: spirvToolsAssemblyAvailable() +// returns false and assembleVulkanSpirvText() returns std::nullopt, so callers +// build and behave correctly either way. + +// Returns true when SPIRV-Tools was linked in at build time. +bool spirvToolsAssemblyAvailable(); + +// Assembles SPIR-V assembly text into binary words for the Vulkan native target +// environment (SPV_ENV_VULKAN_1_2 -> SPIR-V 1.5, matching kVulkanNativeTargetEnv +// / kVulkanNativeSpirvVersion). On success returns the assembled words; on +// assembly failure (or when SPIRV-Tools is unavailable) returns std::nullopt and, +// when provided, writes a human-readable diagnostic to *error. +std::optional> +assembleVulkanSpirvText(std::string_view assembly, std::string *error = nullptr); + +} // namespace crossgl diff --git a/src/Backend/SPIRVAssembler.cpp b/src/Backend/SPIRVAssembler.cpp new file mode 100644 index 000000000..33b8954e4 --- /dev/null +++ b/src/Backend/SPIRVAssembler.cpp @@ -0,0 +1,65 @@ +#include "crossgl/Backend/SPIRVAssembler.h" + +#ifdef CROSSGL_HAVE_SPIRV_TOOLS +#include + +#include +#endif + +namespace crossgl { + +#ifdef CROSSGL_HAVE_SPIRV_TOOLS + +bool spirvToolsAssemblyAvailable() { return true; } + +std::optional> +assembleVulkanSpirvText(std::string_view assembly, std::string *error) { + // The Vulkan native target environment is vulkan1.2, which SPIRV-Tools maps to + // SPIR-V 1.5 -- matching kVulkanNativeTargetEnv / kVulkanNativeSpirvVersion. + spv_context context = spvContextCreate(SPV_ENV_VULKAN_1_2); + if (context == nullptr) { + if (error != nullptr) { + *error = "failed to create SPIRV-Tools context"; + } + return std::nullopt; + } + + spv_binary binary = nullptr; + spv_diagnostic diagnostic = nullptr; + const spv_result_t status = + spvTextToBinary(context, assembly.data(), assembly.size(), &binary, + &diagnostic); + + std::optional> result; + if (status == SPV_SUCCESS && binary != nullptr) { + result.emplace(binary->code, binary->code + binary->wordCount); + } else if (error != nullptr) { + if (diagnostic != nullptr && diagnostic->error != nullptr) { + *error = diagnostic->error; + } else { + *error = "spvTextToBinary failed with status " + std::to_string(status); + } + } + + spvBinaryDestroy(binary); + spvDiagnosticDestroy(diagnostic); + spvContextDestroy(context); + return result; +} + +#else + +bool spirvToolsAssemblyAvailable() { return false; } + +std::optional> +assembleVulkanSpirvText(std::string_view assembly, std::string *error) { + (void)assembly; + if (error != nullptr) { + *error = "SPIRV-Tools support was not compiled in"; + } + return std::nullopt; +} + +#endif + +} // namespace crossgl diff --git a/src/Backend/VulkanBackend.cpp b/src/Backend/VulkanBackend.cpp index 15b3a7578..1c4e7b5bb 100644 --- a/src/Backend/VulkanBackend.cpp +++ b/src/Backend/VulkanBackend.cpp @@ -4,6 +4,7 @@ #include "crossgl/Backend/BackendPlan.h" #include "crossgl/Backend/BackendHIR.h" #include "crossgl/Backend/ResourceArrays.h" +#include "crossgl/Backend/SPIRVAssembler.h" #include "crossgl/Backend/SPIRVModule.h" #include "crossgl/Backend/TargetLegalization.h" #include "crossgl/Backend/TextureCompare.h" @@ -18,6 +19,7 @@ #include #include +#include #include #include #include @@ -16360,6 +16362,52 @@ bool writeTextFile(const std::filesystem::path &path, std::string_view text, return true; } +bool writeSpirvWordsFile(const std::filesystem::path &path, + const std::vector &words, + DiagnosticEngine &diagnostics, std::string_view code) { + std::ofstream output(path, std::ios::binary); + if (!output) { + diagnostics.error(std::string(code), + "failed to write '" + path.string() + "'"); + return false; + } + // SPIR-V words are emitted in the host byte order, matching the convention + // used by spirv-as when producing a module for the host toolchain. + output.write(reinterpret_cast(words.data()), + static_cast(words.size() * sizeof(std::uint32_t))); + return output.good(); +} + +// Optionally assemble the generated SPIR-V text into a real binary module using +// the in-process SPIRV-Tools library. This is an opt-in feature: builds without +// SPIRV-Tools compile to a stub that returns false here, leaving the .spvasm +// text artifact as the only SPIR-V output. When the library is present we write a +// sibling .spv next to the .spvasm so the binary is available even when +// the spirv-as CLI is not on PATH. The CLI native path, when available, later +// rewrites the same file (byte-identical, same target environment). +bool emitOptionalSpirvBinaryFromAssembly( + const std::string &assembly, const std::filesystem::path &spvPath, + DiagnosticEngine &diagnostics) { + if (!spirvToolsAssemblyAvailable()) { + return false; + } + std::string assembleError; + const std::optional> words = + assembleVulkanSpirvText(assembly, &assembleError); + if (!words) { + // The generated .spvasm did not assemble. Surface the exact SPIRV-Tools + // diagnostic rather than silently dropping the binary; this reveals any + // pre-existing .spvasm validity gap instead of masking it. + diagnostics.error("vulkan.spirv-tools-assemble-failed", + "SPIRV-Tools failed to assemble generated Vulkan " + "prototype assembly: " + + assembleError); + return false; + } + return writeSpirvWordsFile(spvPath, *words, diagnostics, + "artifact.write-vulkan-spirv-binary"); +} + } // namespace std::vector @@ -17166,6 +17214,15 @@ VulkanBuildResult buildVulkanPrototypeBinary( return result; } + // Optionally assemble the .spvasm into a real binary .spv in-process via the + // SPIRV-Tools library. No-op (and no diagnostics) when SPIRV-Tools was not + // compiled in; the spirv-as CLI path below remains the canonical producer. + if (spirvToolsAssemblyAvailable() && + !emitOptionalSpirvBinaryFromAssembly(assembly, result.spvPath, + diagnostics)) { + return result; + } + const std::vector assembleCommand{ "spirv-as", "--target-env", kVulkanNativeTargetEnv, result.assemblyPath.string(), "-o", result.spvPath.string()}; diff --git a/tests/cmake/CrossGLVulkanNativeBuildTests.cmake b/tests/cmake/CrossGLVulkanNativeBuildTests.cmake index f58eed6fa..d58d3a7ce 100644 --- a/tests/cmake/CrossGLVulkanNativeBuildTests.cmake +++ b/tests/cmake/CrossGLVulkanNativeBuildTests.cmake @@ -3031,3 +3031,32 @@ if(NOT CROSSGL_HAS_VULKAN_SPIRV_DIS) REASON "optional Vulkan spirv-dis sidecar smoke requires spirv-dis" REQUIRED_VARS CROSSGL_SPIRV_DIS) endif() + +# Optional in-process SPIRV-Tools binary (.spv) emission. Registered only when +# the compiler was built WITH SPIRV-Tools (CROSSGL_HAVE_SPIRV_TOOLS=1) and +# spirv-val is present to validate the assembled binary; otherwise a skip +# sentinel is registered so the suite documents why it was unavailable. +if(SPIRV-Tools_FOUND) + set(CROSSGL_HAS_SPIRV_TOOLS_LIBRARY TRUE) +else() + set(CROSSGL_HAS_SPIRV_TOOLS_LIBRARY FALSE) +endif() +if(CROSSGL_HAS_SPIRV_TOOLS_LIBRARY AND CROSSGL_SPIRV_VAL) + add_test(NAME cglc_build_vulkan_spirv_tools_binary_smoke + COMMAND ${CMAKE_COMMAND} + -DCGLC=$ + -DINPUT=${CROSSGL_STORAGE_BUFFER_COMPUTE_SHADER} + -DOUTPUT=${CMAKE_CURRENT_BINARY_DIR}/test-vulkan-spirv-tools-binary.cglb + -DEXPECTED_MODULE=StorageBufferComputeShader + "-DSPIRV_VAL=${CROSSGL_SPIRV_VAL}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/cmake/VulkanSpirvToolsBinarySmoke.cmake) + crossgl_label_optional_native_test(cglc_build_vulkan_spirv_tools_binary_smoke + vulkan) +else() + crossgl_add_optional_native_skip_test( + NAME cglc_build_vulkan_spirv_tools_binary_unavailable + TARGET vulkan + REASON "optional in-process SPIRV-Tools .spv emission requires the \ +SPIRV-Tools library at build time (CROSSGL_HAVE_SPIRV_TOOLS) and spirv-val" + REQUIRED_VARS CROSSGL_HAS_SPIRV_TOOLS_LIBRARY CROSSGL_SPIRV_VAL) +endif() diff --git a/tests/cmake/VulkanSpirvToolsBinarySmoke.cmake b/tests/cmake/VulkanSpirvToolsBinarySmoke.cmake new file mode 100644 index 000000000..eb2903acd --- /dev/null +++ b/tests/cmake/VulkanSpirvToolsBinarySmoke.cmake @@ -0,0 +1,82 @@ +# Validates the optional in-process SPIRV-Tools binary (.spv) emission path. +# +# This script is registered only when the compiler was built WITH SPIRV-Tools +# (CROSSGL_HAVE_SPIRV_TOOLS=1) and spirv-val is available. It builds a fixture to +# the Vulkan target, confirms the sibling .spv binary exists, carries the +# SPIR-V magic word (proving a real assembled binary was written, not the .spvasm +# text), and that spirv-val accepts it for vulkan1.2 / SPIR-V 1.5. + +if(POLICY CMP0054) + cmake_policy(SET CMP0054 NEW) +endif() + +foreach(required_var IN ITEMS CGLC INPUT OUTPUT EXPECTED_MODULE SPIRV_VAL) + if(NOT DEFINED ${required_var} OR "${${required_var}}" STREQUAL "") + message(FATAL_ERROR "Vulkan SPIRV-Tools binary smoke requires ${required_var}") + endif() +endforeach() + +function(crossgl_spv_smoke_run result_var stdout_var stderr_var) + execute_process( + COMMAND ${ARGN} + RESULT_VARIABLE command_result + OUTPUT_VARIABLE command_stdout + ERROR_VARIABLE command_stderr) + set(${result_var} "${command_result}" PARENT_SCOPE) + set(${stdout_var} "${command_stdout}" PARENT_SCOPE) + set(${stderr_var} "${command_stderr}" PARENT_SCOPE) +endfunction() + +file(REMOVE_RECURSE "${OUTPUT}") + +crossgl_spv_smoke_run( + build_result build_stdout build_stderr + "${CGLC}" build "${INPUT}" --target vulkan --output "${OUTPUT}" --debug-ir) +if(NOT "${build_result}" STREQUAL "0") + message(FATAL_ERROR + "Vulkan SPIRV-Tools binary smoke failed while building ${INPUT}.\n" + "package: ${OUTPUT}\n" + "stdout:\n${build_stdout}\n" + "stderr:\n${build_stderr}") +endif() + +set(vulkan_binary "${OUTPUT}/backend/vulkan/${EXPECTED_MODULE}.spv") +set(vulkan_assembly "${OUTPUT}/backend/vulkan/${EXPECTED_MODULE}.spvasm") + +if(NOT EXISTS "${vulkan_assembly}") + message(FATAL_ERROR "expected generated SPIR-V assembly at ${vulkan_assembly}") +endif() +if(NOT EXISTS "${vulkan_binary}") + message(FATAL_ERROR "expected assembled SPIR-V binary at ${vulkan_binary}") +endif() +file(SIZE "${vulkan_binary}" binary_size) +if(binary_size LESS 20) + message(FATAL_ERROR + "expected a non-trivial SPIR-V binary at ${vulkan_binary}, " + "got ${binary_size} bytes") +endif() + +# Confirm the first word is the SPIR-V magic number (0x07230203). The binary is +# little-endian on this host, so the leading bytes are 03 02 23 07. +file(READ "${vulkan_binary}" magic_hex LIMIT 4 HEX) +if(NOT magic_hex STREQUAL "03022307") + message(FATAL_ERROR + "expected SPIR-V magic 0x07230203 (bytes 03022307) at the start of " + "${vulkan_binary}, got 0x${magic_hex}") +endif() + +crossgl_spv_smoke_run( + validate_result validate_stdout validate_stderr + "${SPIRV_VAL}" --target-env vulkan1.2 "${vulkan_binary}") +if(NOT "${validate_result}" STREQUAL "0") + message(FATAL_ERROR + "spirv-val rejected the SPIRV-Tools-assembled binary.\n" + "spirv-val: ${SPIRV_VAL}\n" + "binary: ${vulkan_binary}\n" + "stdout:\n${validate_stdout}\n" + "stderr:\n${validate_stderr}") +endif() + +message(STATUS + "SPIRV-Tools binary smoke assembled ${vulkan_binary} " + "(${binary_size} bytes) and validated it with ${SPIRV_VAL}")